From 17216316f65718d4d178e8d65f460eb95179bf94 Mon Sep 17 00:00:00 2001 From: Greyson Parrelli Date: Fri, 28 Mar 2025 14:19:16 -0400 Subject: [PATCH] Dynamically compute MediaName and MediaId. --- .../database/AttachmentTableTest_deduping.kt | 11 +- .../database/BackupMediaSnapshotTableTest.kt | 131 ++++++++++-- ...ageProcessorTest_synchronizeDeleteForMe.kt | 3 - .../attachments/DatabaseAttachment.kt | 21 +- .../securesms/backup/v2/BackupRepository.kt | 45 ++-- ...ns.kt => DatabaseAttachmentArchiveUtil.kt} | 49 ++++- .../v2/exporters/ChatItemArchiveExporter.kt | 2 +- .../v2/util/ArchiveConverterExtensions.kt | 8 +- .../InternalBackupPlaygroundViewModel.kt | 20 +- .../securesms/database/AttachmentTable.kt | 111 ++++------ .../database/BackupMediaSnapshotTable.kt | 161 +++++++++++--- .../securesms/database/MediaTable.kt | 2 - .../securesms/database/MessageTable.kt | 2 - .../helpers/SignalDatabaseMigrations.kt | 6 +- .../V269_BackupMediaSnapshotChanges.kt | 26 +++ .../jobs/ArchiveThumbnailUploadJob.kt | 16 +- .../securesms/jobs/AttachmentDownloadJob.kt | 14 +- .../jobs/BackupMediaSnapshotSyncJob.kt | 199 +++++++++++++++--- .../securesms/jobs/BackupMessagesJob.kt | 12 +- .../securesms/jobs/JobManagerFactories.java | 1 - .../securesms/jobs/RestoreAttachmentJob.kt | 10 +- .../jobs/RestoreAttachmentThumbnailJob.kt | 12 +- .../securesms/jobs/SyncArchivedMediaJob.kt | 113 ---------- .../ui/restore/RemoteRestoreViewModel.kt | 2 - .../service/MessageBackupListener.kt | 5 +- .../securesms/video/exo/PartDataSource.java | 11 +- app/src/main/protowire/JobData.proto | 1 + .../v2/ArchivedMediaObjectIteratorTest.kt | 16 +- .../sms/UploadDependencyGraphTest.kt | 3 - .../securesms/database/FakeMessageRecords.kt | 3 - .../main/java/org/signal/core/util/SqlUtil.kt | 2 +- .../signalservice/api/NetworkResult.kt | 5 + .../signalservice/api/archive/ArchiveApi.kt | 2 +- .../signalservice/api/backup/MediaName.kt | 12 ++ 34 files changed, 641 insertions(+), 396 deletions(-) rename app/src/main/java/org/thoughtcrime/securesms/backup/v2/{database/DatabaseAttachmentArchiveExtensions.kt => DatabaseAttachmentArchiveUtil.kt} (73%) create mode 100644 app/src/main/java/org/thoughtcrime/securesms/database/helpers/migration/V269_BackupMediaSnapshotChanges.kt delete mode 100644 app/src/main/java/org/thoughtcrime/securesms/jobs/SyncArchivedMediaJob.kt diff --git a/app/src/androidTest/java/org/thoughtcrime/securesms/database/AttachmentTableTest_deduping.kt b/app/src/androidTest/java/org/thoughtcrime/securesms/database/AttachmentTableTest_deduping.kt index 907de22406..0c111cf5ee 100644 --- a/app/src/androidTest/java/org/thoughtcrime/securesms/database/AttachmentTableTest_deduping.kt +++ b/app/src/androidTest/java/org/thoughtcrime/securesms/database/AttachmentTableTest_deduping.kt @@ -18,7 +18,6 @@ import org.signal.core.util.update import org.thoughtcrime.securesms.attachments.AttachmentId import org.thoughtcrime.securesms.attachments.Cdn import org.thoughtcrime.securesms.attachments.PointerAttachment -import org.thoughtcrime.securesms.backup.v2.BackupRepository.getMediaName import org.thoughtcrime.securesms.database.AttachmentTable.TransformProperties import org.thoughtcrime.securesms.keyvalue.SignalStore import org.thoughtcrime.securesms.mms.MediaStream @@ -30,7 +29,6 @@ import org.thoughtcrime.securesms.recipients.Recipient import org.thoughtcrime.securesms.util.MediaUtil import org.thoughtcrime.securesms.util.Util import org.whispersystems.signalservice.api.attachment.AttachmentUploadResult -import org.whispersystems.signalservice.api.backup.MediaId import org.whispersystems.signalservice.api.messages.SignalServiceAttachmentRemoteId import org.whispersystems.signalservice.api.push.ServiceId import org.whispersystems.signalservice.internal.crypto.PaddingInputStream @@ -734,12 +732,9 @@ class AttachmentTableTest_deduping { SignalDatabase.attachments.finalizeAttachmentAfterUpload(attachmentId, createUploadResult(attachmentId, uploadTimestamp)) val attachment = SignalDatabase.attachments.getAttachment(attachmentId)!! - SignalDatabase.attachments.setArchiveData( + SignalDatabase.attachments.setArchiveCdn( attachmentId = attachmentId, - archiveCdn = Cdn.CDN_3.cdnNumber, - archiveMediaName = attachment.getMediaName().name, - archiveThumbnailMediaId = MediaId(Util.getSecretBytes(15)).encode(), - archiveMediaId = MediaId(Util.getSecretBytes(15)).encode() + archiveCdn = Cdn.CDN_3.cdnNumber ) } @@ -861,8 +856,6 @@ class AttachmentTableTest_deduping { val rhsAttachment = SignalDatabase.attachments.getAttachment(rhs)!! assertEquals(lhsAttachment.archiveCdn, rhsAttachment.archiveCdn) - assertEquals(lhsAttachment.archiveMediaName, rhsAttachment.archiveMediaName) - assertEquals(lhsAttachment.archiveMediaId, rhsAttachment.archiveMediaId) } fun assertDoesNotHaveRemoteFields(attachmentId: AttachmentId) { diff --git a/app/src/androidTest/java/org/thoughtcrime/securesms/database/BackupMediaSnapshotTableTest.kt b/app/src/androidTest/java/org/thoughtcrime/securesms/database/BackupMediaSnapshotTableTest.kt index 8572d1c569..9ae96352ec 100644 --- a/app/src/androidTest/java/org/thoughtcrime/securesms/database/BackupMediaSnapshotTableTest.kt +++ b/app/src/androidTest/java/org/thoughtcrime/securesms/database/BackupMediaSnapshotTableTest.kt @@ -1,5 +1,6 @@ package org.thoughtcrime.securesms.database +import androidx.media3.common.util.Util import androidx.test.ext.junit.runners.AndroidJUnit4 import assertk.assertThat import assertk.assertions.isEqualTo @@ -9,6 +10,7 @@ import org.junit.runner.RunWith import org.signal.core.util.count import org.signal.core.util.readToSingleInt import org.thoughtcrime.securesms.backup.v2.ArchivedMediaObject +import org.thoughtcrime.securesms.database.BackupMediaSnapshotTable.ArchiveMediaItem import org.thoughtcrime.securesms.testing.SignalActivityRule @RunWith(AndroidJUnit4::class) @@ -16,6 +18,7 @@ class BackupMediaSnapshotTableTest { companion object { private const val SEQUENCE_COUNT = 100 + private const val SEQUENCE_COUNT_WITH_THUMBNAILS = 200 } @get:Rule @@ -24,7 +27,7 @@ class BackupMediaSnapshotTableTest { @Test fun givenAnEmptyTable_whenIWriteToTable_thenIExpectEmptyTable() { val pendingSyncTime = 1L - SignalDatabase.backupMediaSnapshots.writePendingMediaObjects(generateArchiveObjectSequence(), pendingSyncTime) + SignalDatabase.backupMediaSnapshots.writePendingMediaObjects(generateArchiveMediaItemSequence(), pendingSyncTime) val count = getSyncedItemCount(pendingSyncTime) @@ -34,22 +37,23 @@ class BackupMediaSnapshotTableTest { @Test fun givenAnEmptyTable_whenIWriteToTableAndCommit_thenIExpectFilledTable() { val pendingSyncTime = 1L - SignalDatabase.backupMediaSnapshots.writePendingMediaObjects(generateArchiveObjectSequence(), pendingSyncTime) + SignalDatabase.backupMediaSnapshots.writePendingMediaObjects(generateArchiveMediaItemSequence(), pendingSyncTime) SignalDatabase.backupMediaSnapshots.commitPendingRows() val count = getSyncedItemCount(pendingSyncTime) - assertThat(count).isEqualTo(SEQUENCE_COUNT) + assertThat(count).isEqualTo(SEQUENCE_COUNT_WITH_THUMBNAILS) } @Test fun givenAFilledTable_whenIInsertSimilarIds_thenIExpectUncommittedOverrides() { - SignalDatabase.backupMediaSnapshots.writePendingMediaObjects(generateArchiveObjectSequence(), 1L) + SignalDatabase.backupMediaSnapshots.writePendingMediaObjects(generateArchiveMediaItemSequence(), 1L) SignalDatabase.backupMediaSnapshots.commitPendingRows() val newPendingTime = 2L val newObjectCount = 50 - SignalDatabase.backupMediaSnapshots.writePendingMediaObjects(generateArchiveObjectSequence(newObjectCount), newPendingTime) + val newObjectCountWithThumbnails = newObjectCount * 2 + SignalDatabase.backupMediaSnapshots.writePendingMediaObjects(generateArchiveMediaItemSequence(newObjectCount), newPendingTime) val count = SignalDatabase.backupMediaSnapshots.readableDatabase.count() .from(BackupMediaSnapshotTable.TABLE_NAME) @@ -57,17 +61,18 @@ class BackupMediaSnapshotTableTest { .run() .readToSingleInt(-1) - assertThat(count).isEqualTo(50) + assertThat(count).isEqualTo(newObjectCountWithThumbnails) } @Test fun givenAFilledTable_whenIInsertSimilarIdsAndCommit_thenIExpectCommittedOverrides() { - SignalDatabase.backupMediaSnapshots.writePendingMediaObjects(generateArchiveObjectSequence(), 1L) + SignalDatabase.backupMediaSnapshots.writePendingMediaObjects(generateArchiveMediaItemSequence(), 1L) SignalDatabase.backupMediaSnapshots.commitPendingRows() val newPendingTime = 2L val newObjectCount = 50 - SignalDatabase.backupMediaSnapshots.writePendingMediaObjects(generateArchiveObjectSequence(newObjectCount), newPendingTime) + val newObjectCountWithThumbnails = newObjectCount * 2 + SignalDatabase.backupMediaSnapshots.writePendingMediaObjects(generateArchiveMediaItemSequence(newObjectCount), newPendingTime) SignalDatabase.backupMediaSnapshots.commitPendingRows() val count = SignalDatabase.backupMediaSnapshots.readableDatabase.count() @@ -78,18 +83,19 @@ class BackupMediaSnapshotTableTest { val total = getTotalItemCount() - assertThat(count).isEqualTo(50) - assertThat(total).isEqualTo(SEQUENCE_COUNT) + assertThat(count).isEqualTo(newObjectCountWithThumbnails) + assertThat(total).isEqualTo(SEQUENCE_COUNT_WITH_THUMBNAILS) } @Test fun givenAFilledTable_whenIInsertSimilarIdsAndCommitThenDelete_thenIExpectOnlyCommittedOverrides() { - SignalDatabase.backupMediaSnapshots.writePendingMediaObjects(generateArchiveObjectSequence(), 1L) + SignalDatabase.backupMediaSnapshots.writePendingMediaObjects(generateArchiveMediaItemSequence(), 1L) SignalDatabase.backupMediaSnapshots.commitPendingRows() val newPendingTime = 2L val newObjectCount = 50 - SignalDatabase.backupMediaSnapshots.writePendingMediaObjects(generateArchiveObjectSequence(newObjectCount), newPendingTime) + val newObjectCountWithThumbnails = newObjectCount * 2 + SignalDatabase.backupMediaSnapshots.writePendingMediaObjects(generateArchiveMediaItemSequence(newObjectCount), newPendingTime) SignalDatabase.backupMediaSnapshots.commitPendingRows() val page = SignalDatabase.backupMediaSnapshots.getPageOfOldMediaObjects(currentSyncTime = newPendingTime, pageSize = 100) @@ -97,7 +103,86 @@ class BackupMediaSnapshotTableTest { val total = getTotalItemCount() - assertThat(total).isEqualTo(50) + assertThat(total).isEqualTo(newObjectCountWithThumbnails) + } + + @Test + fun getMediaObjectsWithNonMatchingCdn_noMismatches() { + val localData = listOf( + createArchiveMediaItem(seed = 1, cdn = 1), + createArchiveMediaItem(seed = 2, cdn = 2) + ) + + val remoteData = listOf( + createArchiveMediaObject(seed = 1, cdn = 1), + createArchiveMediaObject(seed = 2, cdn = 2) + ) + + SignalDatabase.backupMediaSnapshots.writePendingMediaObjects(localData.asSequence(), 1L) + SignalDatabase.backupMediaSnapshots.commitPendingRows() + + val mismatches = SignalDatabase.backupMediaSnapshots.getMediaObjectsWithNonMatchingCdn(remoteData) + assertThat(mismatches.size).isEqualTo(0) + } + + @Test + fun getMediaObjectsWithNonMatchingCdn_oneMismatch() { + val localData = listOf( + createArchiveMediaItem(seed = 1, cdn = 1), + createArchiveMediaItem(seed = 2, cdn = 2) + ) + + val remoteData = listOf( + createArchiveMediaObject(seed = 1, cdn = 1), + createArchiveMediaObject(seed = 2, cdn = 99) + ) + + SignalDatabase.backupMediaSnapshots.writePendingMediaObjects(localData.asSequence(), 1L) + SignalDatabase.backupMediaSnapshots.commitPendingRows() + + val mismatches = SignalDatabase.backupMediaSnapshots.getMediaObjectsWithNonMatchingCdn(remoteData) + assertThat(mismatches.size).isEqualTo(1) + assertThat(mismatches.get(0).cdn).isEqualTo(99) + assertThat(mismatches.get(0).digest).isEqualTo(localData.get(1).digest) + } + + @Test + fun getMediaObjectsThatCantBeFound_allFound() { + val localData = listOf( + createArchiveMediaItem(seed = 1, cdn = 1), + createArchiveMediaItem(seed = 2, cdn = 2) + ) + + val remoteData = listOf( + createArchiveMediaObject(seed = 1, cdn = 1), + createArchiveMediaObject(seed = 2, cdn = 2) + ) + + SignalDatabase.backupMediaSnapshots.writePendingMediaObjects(localData.asSequence(), 1L) + SignalDatabase.backupMediaSnapshots.commitPendingRows() + + val notFound = SignalDatabase.backupMediaSnapshots.getMediaObjectsThatCantBeFound(remoteData) + assertThat(notFound.size).isEqualTo(0) + } + + @Test + fun getMediaObjectsThatCantBeFound_oneMissing() { + val localData = listOf( + createArchiveMediaItem(seed = 1, cdn = 1), + createArchiveMediaItem(seed = 2, cdn = 2) + ) + + val remoteData = listOf( + createArchiveMediaObject(seed = 1, cdn = 1), + createArchiveMediaObject(seed = 3, cdn = 2) + ) + + SignalDatabase.backupMediaSnapshots.writePendingMediaObjects(localData.asSequence(), 1L) + SignalDatabase.backupMediaSnapshots.commitPendingRows() + + val notFound = SignalDatabase.backupMediaSnapshots.getMediaObjectsThatCantBeFound(remoteData) + assertThat(notFound.size).isEqualTo(1) + assertThat(notFound.first().cdn).isEqualTo(2) } private fun getTotalItemCount(): Int { @@ -112,8 +197,24 @@ class BackupMediaSnapshotTableTest { .readToSingleInt(-1) } - private fun generateArchiveObjectSequence(count: Int = SEQUENCE_COUNT): Sequence { + private fun generateArchiveMediaItemSequence(count: Int = SEQUENCE_COUNT): Sequence { return generateSequence(0) { seed -> if (seed < (count - 1)) seed + 1 else null } - .map { ArchivedMediaObject(mediaId = "media_id_$it", 0) } + .map { createArchiveMediaItem(it) } + } + + private fun createArchiveMediaItem(seed: Int, cdn: Int = 0): ArchiveMediaItem { + return ArchiveMediaItem( + mediaId = "media_id_$seed", + thumbnailMediaId = "thumbnail_media_id_$seed", + cdn = cdn, + digest = Util.toByteArray(seed) + ) + } + + private fun createArchiveMediaObject(seed: Int, cdn: Int = 0): ArchivedMediaObject { + return ArchivedMediaObject( + mediaId = "media_id_$seed", + cdn = cdn + ) } } diff --git a/app/src/androidTest/java/org/thoughtcrime/securesms/messages/SyncMessageProcessorTest_synchronizeDeleteForMe.kt b/app/src/androidTest/java/org/thoughtcrime/securesms/messages/SyncMessageProcessorTest_synchronizeDeleteForMe.kt index a40d09fd0d..d58e5ea9cb 100644 --- a/app/src/androidTest/java/org/thoughtcrime/securesms/messages/SyncMessageProcessorTest_synchronizeDeleteForMe.kt +++ b/app/src/androidTest/java/org/thoughtcrime/securesms/messages/SyncMessageProcessorTest_synchronizeDeleteForMe.kt @@ -677,7 +677,6 @@ class SyncMessageProcessorTest_synchronizeDeleteForMe { mmsId = this.mmsId, hasData = this.hasData, hasThumbnail = false, - hasArchiveThumbnail = false, contentType = this.contentType, transferProgress = this.transferState, size = this.size, @@ -705,8 +704,6 @@ class SyncMessageProcessorTest_synchronizeDeleteForMe { uploadTimestamp = this.uploadTimestamp, dataHash = this.dataHash, archiveCdn = this.archiveCdn, - archiveMediaName = this.archiveMediaName, - archiveMediaId = this.archiveMediaId, thumbnailRestoreState = this.thumbnailRestoreState, archiveTransferState = this.archiveTransferState, uuid = uuid diff --git a/app/src/main/java/org/thoughtcrime/securesms/attachments/DatabaseAttachment.kt b/app/src/main/java/org/thoughtcrime/securesms/attachments/DatabaseAttachment.kt index bb5c6f2f0f..fb2a92f43a 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/attachments/DatabaseAttachment.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/attachments/DatabaseAttachment.kt @@ -29,19 +29,12 @@ class DatabaseAttachment : Attachment { @JvmField val archiveCdn: Int - @JvmField - val archiveMediaName: String? - - @JvmField - val archiveMediaId: String? - @JvmField val thumbnailRestoreState: AttachmentTable.ThumbnailRestoreState @JvmField val archiveTransferState: AttachmentTable.ArchiveTransferState - private val hasArchiveThumbnail: Boolean private val hasThumbnail: Boolean val displayOrder: Int @@ -50,7 +43,6 @@ class DatabaseAttachment : Attachment { mmsId: Long, hasData: Boolean, hasThumbnail: Boolean, - hasArchiveThumbnail: Boolean, contentType: String?, transferProgress: Int, size: Long, @@ -78,8 +70,6 @@ class DatabaseAttachment : Attachment { uploadTimestamp: Long, dataHash: String?, archiveCdn: Int, - archiveMediaName: String?, - archiveMediaId: String?, thumbnailRestoreState: AttachmentTable.ThumbnailRestoreState, archiveTransferState: AttachmentTable.ArchiveTransferState, uuid: UUID? @@ -114,11 +104,8 @@ class DatabaseAttachment : Attachment { this.hasData = hasData this.dataHash = dataHash this.hasThumbnail = hasThumbnail - this.hasArchiveThumbnail = hasArchiveThumbnail this.displayOrder = displayOrder this.archiveCdn = archiveCdn - this.archiveMediaName = archiveMediaName - this.archiveMediaId = archiveMediaId this.thumbnailRestoreState = thumbnailRestoreState this.archiveTransferState = archiveTransferState } @@ -131,9 +118,6 @@ class DatabaseAttachment : Attachment { mmsId = parcel.readLong() displayOrder = parcel.readInt() archiveCdn = parcel.readInt() - archiveMediaName = parcel.readString() - archiveMediaId = parcel.readString() - hasArchiveThumbnail = ParcelUtil.readBoolean(parcel) thumbnailRestoreState = AttachmentTable.ThumbnailRestoreState.deserialize(parcel.readInt()) archiveTransferState = AttachmentTable.ArchiveTransferState.deserialize(parcel.readInt()) } @@ -147,9 +131,6 @@ class DatabaseAttachment : Attachment { dest.writeLong(mmsId) dest.writeInt(displayOrder) dest.writeInt(archiveCdn) - dest.writeString(archiveMediaName) - dest.writeString(archiveMediaId) - ParcelUtil.writeBoolean(dest, hasArchiveThumbnail) dest.writeInt(thumbnailRestoreState.value) dest.writeInt(archiveTransferState.value) } @@ -169,7 +150,7 @@ class DatabaseAttachment : Attachment { } override val thumbnailUri: Uri? - get() = if (hasArchiveThumbnail) { + get() = if (thumbnailRestoreState == AttachmentTable.ThumbnailRestoreState.FINISHED) { PartAuthority.getAttachmentThumbnailUri(attachmentId) } else { null diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/BackupRepository.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/BackupRepository.kt index a6ebfc2ccb..7064d612ba 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/BackupRepository.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/BackupRepository.kt @@ -28,7 +28,7 @@ import org.signal.core.util.getAllTriggerDefinitions import org.signal.core.util.getForeignKeyViolations import org.signal.core.util.logging.Log import org.signal.core.util.requireInt -import org.signal.core.util.requireNonNullString +import org.signal.core.util.requireNonNullBlob import org.signal.core.util.stream.NonClosingOutputStream import org.signal.core.util.urlEncode import org.signal.core.util.withinTransaction @@ -59,6 +59,7 @@ import org.thoughtcrime.securesms.components.settings.app.subscription.Recurring import org.thoughtcrime.securesms.crypto.AttachmentSecretProvider import org.thoughtcrime.securesms.crypto.DatabaseSecretProvider import org.thoughtcrime.securesms.database.AttachmentTable +import org.thoughtcrime.securesms.database.BackupMediaSnapshotTable.ArchiveMediaItem import org.thoughtcrime.securesms.database.KeyValueDatabase import org.thoughtcrime.securesms.database.SearchTable import org.thoughtcrime.securesms.database.SignalDatabase @@ -357,6 +358,7 @@ object BackupRepository { Log.d(TAG, "Disabling backups.") SignalStore.backup.disableBackups() + SignalDatabase.attachments.clearAllArchiveData() true } catch (e: Exception) { Log.w(TAG, "Failed to turn off backups.", e) @@ -1100,7 +1102,7 @@ object BackupRepository { fun copyThumbnailToArchive(thumbnailAttachment: Attachment, parentAttachment: DatabaseAttachment): NetworkResult { return initBackupAndFetchAuth() .then { credential -> - val request = thumbnailAttachment.toArchiveMediaRequest(parentAttachment.getThumbnailMediaName(), credential.mediaBackupAccess.backupKey) + val request = thumbnailAttachment.toArchiveMediaRequest(parentAttachment.requireThumbnailMediaName(), credential.mediaBackupAccess.backupKey) SignalNetwork.archive.copyAttachmentToArchive( aci = SignalStore.account.requireAci(), @@ -1116,7 +1118,7 @@ object BackupRepository { fun copyAttachmentToArchive(attachment: DatabaseAttachment): NetworkResult { return initBackupAndFetchAuth() .then { credential -> - val mediaName = attachment.getMediaName() + val mediaName = attachment.requireMediaName() val request = attachment.toArchiveMediaRequest(mediaName, credential.mediaBackupAccess.backupKey) SignalNetwork.archive .copyAttachmentToArchive( @@ -1124,12 +1126,9 @@ object BackupRepository { archiveServiceAccess = credential.mediaBackupAccess, item = request ) - .map { credential to Triple(mediaName, request.mediaId, it) } } - .map { (credential, triple) -> - val (mediaName, mediaId, response) = triple - val thumbnailId = credential.mediaBackupAccess.backupKey.deriveMediaId(attachment.getThumbnailMediaName()).encode() - SignalDatabase.attachments.setArchiveData(attachmentId = attachment.attachmentId, archiveCdn = response.cdn, archiveMediaName = mediaName.name, archiveMediaId = mediaId, archiveThumbnailMediaId = thumbnailId) + .map { response -> + SignalDatabase.attachments.setArchiveCdn(attachmentId = attachment.attachmentId, archiveCdn = response.cdn) } .also { Log.i(TAG, "archiveMediaResult: $it") } } @@ -1142,7 +1141,7 @@ object BackupRepository { val attachmentIdToMediaName = mutableMapOf() databaseAttachments.forEach { - val mediaName = it.getMediaName() + val mediaName = it.requireMediaName() val request = it.toArchiveMediaRequest(mediaName, credential.mediaBackupAccess.backupKey) requests += request mediaIdToAttachmentId[request.mediaId] = it.attachmentId @@ -1164,7 +1163,7 @@ object BackupRepository { val attachmentId = result.mediaIdToAttachmentId(it.mediaId) val mediaName = result.attachmentIdToMediaName(attachmentId) val thumbnailId = credential.mediaBackupAccess.backupKey.deriveMediaId(MediaName.forThumbnailFromMediaName(mediaName = mediaName)).encode() - SignalDatabase.attachments.setArchiveData(attachmentId = attachmentId, archiveCdn = it.cdn!!, archiveMediaName = mediaName, archiveMediaId = it.mediaId, thumbnailId) + SignalDatabase.attachments.setArchiveCdn(attachmentId = attachmentId, archiveCdn = it.cdn!!) } result } @@ -1172,12 +1171,14 @@ object BackupRepository { } fun deleteArchivedMedia(attachments: List): NetworkResult { + val mediaRootBackupKey = SignalStore.backup.mediaRootBackupKey + val mediaToDelete = attachments - .filter { it.archiveMediaId != null } + .filter { it.archiveTransferState == AttachmentTable.ArchiveTransferState.FINISHED } .map { DeleteArchivedMediaRequest.ArchivedMediaObject( cdn = it.archiveCdn, - mediaId = it.archiveMediaId!! + mediaId = it.requireMediaName().toMediaId(mediaRootBackupKey).encode() ) } @@ -1538,14 +1539,6 @@ object BackupRepository { val profileKey: ProfileKey ) - fun DatabaseAttachment.getMediaName(): MediaName { - return MediaName.fromDigest(remoteDigest!!) - } - - fun DatabaseAttachment.getThumbnailMediaName(): MediaName { - return MediaName.fromDigestForThumbnail(remoteDigest!!) - } - private fun Attachment.toArchiveMediaRequest(mediaName: MediaName, mediaRootBackupKey: MediaRootBackupKey): ArchiveMediaRequest { val mediaSecrets = mediaRootBackupKey.deriveMediaSecrets(mediaName) @@ -1646,7 +1639,7 @@ sealed class ImportResult { * // Cursor is closed after use block. * ``` */ -class ArchivedMediaObjectIterator(private val cursor: Cursor) : Iterator { +class ArchiveMediaItemIterator(private val cursor: Cursor) : Iterator { init { cursor.moveToFirst() @@ -1654,10 +1647,14 @@ class ArchivedMediaObjectIterator(private val cursor: Cursor) : Iterator?, attachments: List?): ViewOnceMessage { - val attachment: DatabaseAttachment? = attachments?.firstOrNull()?.takeUnless { !it.hasData && it.size == 0L && it.archiveMediaId == null && it.width == 0 && it.height == 0 && it.blurHash == null } + val attachment: DatabaseAttachment? = attachments?.firstOrNull()?.takeUnless { !it.hasData && it.size == 0L && it.remoteDigest == null && it.width == 0 && it.height == 0 && it.blurHash == null } return ViewOnceMessage( attachment = attachment?.toRemoteMessageAttachment(mediaArchiveEnabled), diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/util/ArchiveConverterExtensions.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/util/ArchiveConverterExtensions.kt index 1b5330370d..cfc2e3c5d9 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/util/ArchiveConverterExtensions.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/util/ArchiveConverterExtensions.kt @@ -8,6 +8,7 @@ package org.thoughtcrime.securesms.backup.v2.util import okio.ByteString import okio.ByteString.Companion.toByteString import org.signal.core.util.Base64 +import org.signal.core.util.emptyIfNull import org.signal.core.util.nullIfBlank import org.signal.core.util.orNull import org.thoughtcrime.securesms.attachments.ArchivedAttachment @@ -16,8 +17,8 @@ import org.thoughtcrime.securesms.attachments.Cdn import org.thoughtcrime.securesms.attachments.DatabaseAttachment import org.thoughtcrime.securesms.attachments.PointerAttachment import org.thoughtcrime.securesms.attachments.TombstoneAttachment -import org.thoughtcrime.securesms.backup.v2.BackupRepository.getMediaName import org.thoughtcrime.securesms.backup.v2.ImportState +import org.thoughtcrime.securesms.backup.v2.getMediaName import org.thoughtcrime.securesms.backup.v2.proto.FilePointer import org.thoughtcrime.securesms.conversation.colors.AvatarColor import org.thoughtcrime.securesms.database.AttachmentTable @@ -150,10 +151,11 @@ fun DatabaseAttachment.toRemoteFilePointer(mediaArchiveEnabled: Boolean, content if (mediaArchiveEnabled && !pending) { val transitCdnKey = this.remoteLocation?.nullIfBlank() val transitCdnNumber = this.cdn.cdnNumber.takeIf { transitCdnKey != null } + val archiveMediaName = this.getMediaName()?.toString() builder.backupLocator = FilePointer.BackupLocator( - mediaName = this.archiveMediaName ?: this.getMediaName().toString(), - cdnNumber = this.archiveCdn.takeIf { this.archiveMediaName != null }, + mediaName = archiveMediaName.emptyIfNull(), + cdnNumber = this.archiveCdn.takeIf { archiveMediaName != null }, key = Base64.decode(remoteKey).toByteString(), size = this.size.toInt(), digest = this.remoteDigest.toByteString(), diff --git a/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/backup/InternalBackupPlaygroundViewModel.kt b/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/backup/InternalBackupPlaygroundViewModel.kt index 3495e1012d..d6f7435451 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/backup/InternalBackupPlaygroundViewModel.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/backup/InternalBackupPlaygroundViewModel.kt @@ -41,6 +41,7 @@ import org.thoughtcrime.securesms.backup.v2.local.LocalArchiver import org.thoughtcrime.securesms.backup.v2.local.LocalArchiver.FailureCause import org.thoughtcrime.securesms.backup.v2.local.SnapshotFileSystem import org.thoughtcrime.securesms.backup.v2.stream.EncryptedBackupReader.Companion.MAC_SIZE +import org.thoughtcrime.securesms.database.AttachmentTable import org.thoughtcrime.securesms.database.MessageType import org.thoughtcrime.securesms.database.SignalDatabase import org.thoughtcrime.securesms.dependencies.AppDependencies @@ -53,13 +54,11 @@ import org.thoughtcrime.securesms.jobs.CopyAttachmentToArchiveJob import org.thoughtcrime.securesms.jobs.RestoreAttachmentJob import org.thoughtcrime.securesms.jobs.RestoreAttachmentThumbnailJob import org.thoughtcrime.securesms.jobs.RestoreLocalAttachmentJob -import org.thoughtcrime.securesms.jobs.SyncArchivedMediaJob import org.thoughtcrime.securesms.keyvalue.SignalStore import org.thoughtcrime.securesms.mms.IncomingMessage import org.thoughtcrime.securesms.providers.BlobProvider import org.thoughtcrime.securesms.recipients.Recipient import org.whispersystems.signalservice.api.NetworkResult -import org.whispersystems.signalservice.api.backup.MediaName import org.whispersystems.signalservice.api.backup.MessageBackupKey import org.whispersystems.signalservice.api.push.ServiceId.ACI import java.io.FileOutputStream @@ -351,7 +350,6 @@ class InternalBackupPlaygroundViewModel : ViewModel() { AppDependencies .jobManager .startChain(BackupRestoreJob()) - .then(SyncArchivedMediaJob()) .then(BackupRestoreMediaJob()) .enqueueAndBlockUntilCompletion(120.seconds.inWholeMilliseconds) } @@ -585,24 +583,16 @@ class InternalBackupPlaygroundViewModel : ViewModel() { attachments: List = this.attachments, inProgress: Set = this.inProgressMediaIds ): MediaState { - val backupKey = SignalStore.backup.messageBackupKey - val mediaRootBackupKey = SignalStore.backup.mediaRootBackupKey - val updatedAttachments = attachments.map { val state = if (inProgress.contains(it.dbAttachment.attachmentId)) { BackupAttachment.State.IN_PROGRESS - } else if (it.dbAttachment.archiveMediaName != null) { - if (it.dbAttachment.remoteDigest != null) { - val mediaId = mediaRootBackupKey.deriveMediaId(MediaName(it.dbAttachment.archiveMediaName)).encode() - if (it.dbAttachment.archiveMediaId == mediaId) { - BackupAttachment.State.UPLOADED_FINAL - } else { - BackupAttachment.State.UPLOADED_UNDOWNLOADED - } + } else if (it.dbAttachment.archiveTransferState == AttachmentTable.ArchiveTransferState.FINISHED) { + if (it.dbAttachment.transferState == AttachmentTable.TRANSFER_PROGRESS_DONE) { + BackupAttachment.State.UPLOADED_FINAL } else { BackupAttachment.State.UPLOADED_UNDOWNLOADED } - } else if (it.dbAttachment.dataHash == null) { + } else if (it.dbAttachment.remoteLocation != null) { BackupAttachment.State.ATTACHMENT_CDN } else { BackupAttachment.State.LOCAL_ONLY diff --git a/app/src/main/java/org/thoughtcrime/securesms/database/AttachmentTable.kt b/app/src/main/java/org/thoughtcrime/securesms/database/AttachmentTable.kt index 70de72dadf..234047c7a9 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/database/AttachmentTable.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/database/AttachmentTable.kt @@ -64,6 +64,7 @@ import org.signal.core.util.stream.LimitedInputStream import org.signal.core.util.stream.NullOutputStream import org.signal.core.util.toInt import org.signal.core.util.update +import org.signal.core.util.updateAll import org.signal.core.util.withinTransaction import org.thoughtcrime.securesms.attachments.ArchivedAttachment import org.thoughtcrime.securesms.attachments.Attachment @@ -97,7 +98,6 @@ import org.thoughtcrime.securesms.util.StorageUtil import org.thoughtcrime.securesms.util.Util import org.thoughtcrime.securesms.video.EncryptedMediaDataSource import org.whispersystems.signalservice.api.attachment.AttachmentUploadResult -import org.whispersystems.signalservice.api.backup.MediaId import org.whispersystems.signalservice.api.crypto.AttachmentCipherOutputStream import org.whispersystems.signalservice.api.util.UuidUtil import org.whispersystems.signalservice.internal.crypto.PaddingInputStream @@ -164,9 +164,6 @@ class AttachmentTable( const val DISPLAY_ORDER = "display_order" const val UPLOAD_TIMESTAMP = "upload_timestamp" const val ARCHIVE_CDN = "archive_cdn" - const val ARCHIVE_MEDIA_NAME = "archive_media_name" - const val ARCHIVE_MEDIA_ID = "archive_media_id" - const val ARCHIVE_THUMBNAIL_MEDIA_ID = "archive_thumbnail_media_id" const val ARCHIVE_TRANSFER_FILE = "archive_transfer_file" const val ARCHIVE_TRANSFER_STATE = "archive_transfer_state" const val THUMBNAIL_RESTORE_STATE = "thumbnail_restore_state" @@ -224,8 +221,6 @@ class AttachmentTable( DATA_HASH_START, DATA_HASH_END, ARCHIVE_CDN, - ARCHIVE_MEDIA_NAME, - ARCHIVE_MEDIA_ID, ARCHIVE_TRANSFER_FILE, THUMBNAIL_FILE, THUMBNAIL_RESTORE_STATE, @@ -270,11 +265,8 @@ class AttachmentTable( $DATA_HASH_START TEXT DEFAULT NULL, $DATA_HASH_END TEXT DEFAULT NULL, $ARCHIVE_CDN INTEGER DEFAULT 0, - $ARCHIVE_MEDIA_NAME TEXT DEFAULT NULL, - $ARCHIVE_MEDIA_ID TEXT DEFAULT NULL, $ARCHIVE_TRANSFER_FILE TEXT DEFAULT NULL, $ARCHIVE_TRANSFER_STATE INTEGER DEFAULT ${ArchiveTransferState.NONE.value}, - $ARCHIVE_THUMBNAIL_MEDIA_ID TEXT DEFAULT NULL, $THUMBNAIL_FILE TEXT DEFAULT NULL, $THUMBNAIL_RANDOM BLOB DEFAULT NULL, $THUMBNAIL_RESTORE_STATE INTEGER DEFAULT ${ThumbnailRestoreState.NONE.value}, @@ -294,8 +286,8 @@ class AttachmentTable( "CREATE INDEX IF NOT EXISTS attachment_data_hash_start_index ON $TABLE_NAME ($DATA_HASH_START);", "CREATE INDEX IF NOT EXISTS attachment_data_hash_end_index ON $TABLE_NAME ($DATA_HASH_END);", "CREATE INDEX IF NOT EXISTS $DATA_FILE_INDEX ON $TABLE_NAME ($DATA_FILE);", - "CREATE INDEX IF NOT EXISTS attachment_archive_media_id_index ON $TABLE_NAME ($ARCHIVE_MEDIA_ID);", - "CREATE INDEX IF NOT EXISTS attachment_archive_transfer_state ON $TABLE_NAME ($ARCHIVE_TRANSFER_STATE);" + "CREATE INDEX IF NOT EXISTS attachment_archive_transfer_state ON $TABLE_NAME ($ARCHIVE_TRANSFER_STATE);", + "CREATE INDEX IF NOT EXISTS attachment_remote_digest_index ON $TABLE_NAME ($REMOTE_DIGEST);" ) @JvmStatic @@ -403,11 +395,15 @@ class AttachmentTable( } } - fun getMediaIdCursor(): Cursor { + /** + * Returns a cursor (with just the digest+archive_cdn) for all attachments that are eligible for archive upload. + * In practice, this means that the attachments have a digest and have not hit a permanent archive upload failure. + */ + fun getAttachmentsEligibleForArchiveUpload(): Cursor { return readableDatabase - .select(ARCHIVE_MEDIA_ID, ARCHIVE_CDN) + .select(REMOTE_DIGEST, ARCHIVE_CDN) .from(TABLE_NAME) - .where("$ARCHIVE_MEDIA_ID IS NOT NULL") + .where("$REMOTE_DIGEST IS NOT NULL AND $ARCHIVE_TRANSFER_STATE != ${ArchiveTransferState.PERMANENT_FAILURE.value}") .run() } @@ -742,9 +738,12 @@ class AttachmentTable( """ SELECT SUM($DATA_SIZE) FROM ( - SELECT DISTINCT $ARCHIVE_MEDIA_ID, $DATA_SIZE + SELECT DISTINCT $REMOTE_DIGEST, $DATA_SIZE FROM $TABLE_NAME - WHERE $ARCHIVE_TRANSFER_STATE NOT IN (${ArchiveTransferState.FINISHED.value}, ${ArchiveTransferState.PERMANENT_FAILURE.value}) + WHERE + $DATA_FILE NOT NULL AND + $REMOTE_DIGEST NOT NULL AND + $ARCHIVE_TRANSFER_STATE NOT IN (${ArchiveTransferState.FINISHED.value}, ${ArchiveTransferState.PERMANENT_FAILURE.value}) ) """.trimIndent() ) @@ -1158,7 +1157,7 @@ class AttachmentTable( // We don't look at hash_start here because that could result in us matching on a file that got compressed down to something smaller, effectively lowering // the quality of the attachment we received. val hashMatch: DataFileInfo? = readableDatabase - .select(ID, DATA_FILE, DATA_SIZE, DATA_RANDOM, DATA_HASH_START, DATA_HASH_END, TRANSFORM_PROPERTIES, UPLOAD_TIMESTAMP, ARCHIVE_CDN, ARCHIVE_MEDIA_NAME, ARCHIVE_MEDIA_ID) + .select(ID, DATA_FILE, DATA_SIZE, DATA_RANDOM, DATA_HASH_START, DATA_HASH_END, TRANSFORM_PROPERTIES, UPLOAD_TIMESTAMP, ARCHIVE_CDN) .from(TABLE_NAME) .where("$DATA_HASH_END = ? AND $DATA_HASH_END NOT NULL AND $TRANSFER_STATE = $TRANSFER_PROGRESS_DONE AND $DATA_FILE NOT NULL", fileWriteResult.hash) .run() @@ -1175,8 +1174,6 @@ class AttachmentTable( values.put(DATA_HASH_START, hashMatch.hashEnd) values.put(DATA_HASH_END, hashMatch.hashEnd) values.put(ARCHIVE_CDN, hashMatch.archiveCdn) - values.put(ARCHIVE_MEDIA_NAME, hashMatch.archiveMediaName) - values.put(ARCHIVE_MEDIA_ID, hashMatch.archiveMediaId) } else { values.put(DATA_FILE, fileWriteResult.file.absolutePath) values.put(DATA_SIZE, fileWriteResult.length) @@ -1252,7 +1249,7 @@ class AttachmentTable( } @Throws(IOException::class) - fun finalizeAttachmentThumbnailAfterDownload(attachmentId: AttachmentId, archiveMediaId: String, inputStream: InputStream, transferFile: File) { + fun finalizeAttachmentThumbnailAfterDownload(attachmentId: AttachmentId, digest: ByteArray, inputStream: InputStream, transferFile: File) { Log.i(TAG, "[finalizeAttachmentThumbnailAfterDownload] Finalizing downloaded data for $attachmentId.") val fileWriteResult: DataFileWriteResult = writeToDataFile(newDataFile(context), inputStream, TransformProperties.empty()) @@ -1265,7 +1262,7 @@ class AttachmentTable( db.update(TABLE_NAME) .values(values) - .where("$ARCHIVE_MEDIA_ID = ?", archiveMediaId) + .where("$REMOTE_DIGEST = ?", digest) .run() } @@ -1277,10 +1274,12 @@ class AttachmentTable( } } + /** + * Updates the state around archive thumbnail uploads, and ensures that all attachments sharing the same digest remain in sync. + */ fun finalizeAttachmentThumbnailAfterUpload( attachmentId: AttachmentId, - archiveMediaId: String, - archiveThumbnailMediaId: MediaId, + attachmentDigest: ByteArray, data: ByteArray ) { Log.i(TAG, "[finalizeAttachmentThumbnailAfterUpload] Finalizing archive data for $attachmentId thumbnail.") @@ -1290,13 +1289,12 @@ class AttachmentTable( val values = contentValuesOf( THUMBNAIL_FILE to fileWriteResult.file.absolutePath, THUMBNAIL_RANDOM to fileWriteResult.random, - THUMBNAIL_RESTORE_STATE to ThumbnailRestoreState.FINISHED.value, - ARCHIVE_THUMBNAIL_MEDIA_ID to archiveThumbnailMediaId.encode() + THUMBNAIL_RESTORE_STATE to ThumbnailRestoreState.FINISHED.value ) db.update(TABLE_NAME) .values(values) - .where("$ARCHIVE_MEDIA_ID = ? OR $ID = ?", archiveMediaId, attachmentId) + .where("$ID = ? OR $REMOTE_DIGEST = ?", attachmentId, attachmentDigest) .run() } } @@ -1601,10 +1599,7 @@ class AttachmentTable( $DATA_RANDOM, $DATA_HASH_START, $DATA_HASH_END, - $ARCHIVE_MEDIA_ID, - $ARCHIVE_MEDIA_NAME, $ARCHIVE_CDN, - $ARCHIVE_THUMBNAIL_MEDIA_ID, $THUMBNAIL_RESTORE_STATE ) SELECT @@ -1631,10 +1626,7 @@ class AttachmentTable( $DATA_RANDOM, $DATA_HASH_START, $DATA_HASH_END, - "${attachment.archiveMediaId}", - "${attachment.archiveMediaName}", ${attachment.archiveCdn}, - $ARCHIVE_THUMBNAIL_MEDIA_ID, ${if (forThumbnail) ThumbnailRestoreState.NEEDS_RESTORE.value else ThumbnailRestoreState.NONE.value} FROM $TABLE_NAME WHERE $ID = ${attachment.attachmentId.id} @@ -1733,7 +1725,7 @@ class AttachmentTable( fun getDataFileInfo(attachmentId: AttachmentId): DataFileInfo? { return readableDatabase - .select(ID, DATA_FILE, DATA_SIZE, DATA_RANDOM, DATA_HASH_START, DATA_HASH_END, TRANSFORM_PROPERTIES, UPLOAD_TIMESTAMP, ARCHIVE_CDN, ARCHIVE_MEDIA_NAME, ARCHIVE_MEDIA_ID) + .select(ID, DATA_FILE, DATA_SIZE, DATA_RANDOM, DATA_HASH_START, DATA_HASH_END, TRANSFORM_PROPERTIES, UPLOAD_TIMESTAMP, ARCHIVE_CDN) .from(TABLE_NAME) .where("$ID = ?", attachmentId.id) .run() @@ -1934,9 +1926,6 @@ class AttachmentTable( uploadTimestamp = jsonObject.getLong(UPLOAD_TIMESTAMP), dataHash = jsonObject.getString(DATA_HASH_END), archiveCdn = jsonObject.getInt(ARCHIVE_CDN), - archiveMediaName = jsonObject.getString(ARCHIVE_MEDIA_NAME), - archiveMediaId = jsonObject.getString(ARCHIVE_MEDIA_ID), - hasArchiveThumbnail = !TextUtils.isEmpty(jsonObject.getString(THUMBNAIL_FILE)), thumbnailRestoreState = ThumbnailRestoreState.deserialize(jsonObject.getInt(THUMBNAIL_RESTORE_STATE)), archiveTransferState = ArchiveTransferState.deserialize(jsonObject.getInt(ARCHIVE_TRANSFER_STATE)), uuid = UuidUtil.parseOrNull(jsonObject.getString(ATTACHMENT_UUID)) @@ -1982,7 +1971,7 @@ class AttachmentTable( /** * Sets the archive data for the specific attachment, as well as for any attachments that use the same underlying file. */ - fun setArchiveData(attachmentId: AttachmentId, archiveCdn: Int, archiveMediaName: String, archiveMediaId: String, archiveThumbnailMediaId: String) { + fun setArchiveCdn(attachmentId: AttachmentId, archiveCdn: Int) { writableDatabase.withinTransaction { db -> val dataFile = db .select(DATA_FILE) @@ -1999,9 +1988,6 @@ class AttachmentTable( db.update(TABLE_NAME) .values( ARCHIVE_CDN to archiveCdn, - ARCHIVE_MEDIA_ID to archiveMediaId, - ARCHIVE_MEDIA_NAME to archiveMediaName, - ARCHIVE_THUMBNAIL_MEDIA_ID to archiveThumbnailMediaId, ARCHIVE_TRANSFER_STATE to ArchiveTransferState.FINISHED.value ) .where("$DATA_FILE = ?", dataFile) @@ -2009,14 +1995,15 @@ class AttachmentTable( } } - fun updateArchiveCdnByMediaId(archiveMediaId: String, archiveCdn: Int): Int { - return writableDatabase.rawQuery( - "UPDATE $TABLE_NAME SET " + - "$ARCHIVE_CDN = CASE WHEN $ARCHIVE_MEDIA_ID = ? THEN ? ELSE $ARCHIVE_CDN END " + - "WHERE $ARCHIVE_MEDIA_ID = ? OR $ARCHIVE_THUMBNAIL_MEDIA_ID = ? " + - "RETURNING $ARCHIVE_CDN", - SqlUtil.buildArgs(archiveMediaId, archiveCdn, archiveMediaId, archiveMediaId) - ).count + /** + * Updates all attachments that share the same digest with the given archive CDN. + */ + fun setArchiveCdnByDigest(digest: ByteArray, archiveCdn: Int) { + writableDatabase + .update(TABLE_NAME) + .values(ARCHIVE_CDN to archiveCdn) + .where("$REMOTE_DIGEST = ?", digest) + .run() } fun clearArchiveData(attachmentIds: List) { @@ -2025,9 +2012,7 @@ class AttachmentTable( writableDatabase .update(TABLE_NAME) .values( - ARCHIVE_CDN to 0, - ARCHIVE_MEDIA_ID to null, - ARCHIVE_MEDIA_NAME to null + ARCHIVE_CDN to 0 ) .where(query.where, query.whereArgs) .run() @@ -2036,13 +2021,11 @@ class AttachmentTable( fun clearAllArchiveData() { writableDatabase - .update(TABLE_NAME) + .updateAll(TABLE_NAME) .values( ARCHIVE_CDN to 0, - ARCHIVE_MEDIA_ID to null, - ARCHIVE_MEDIA_NAME to null + ARCHIVE_TRANSFER_STATE to ArchiveTransferState.NONE.value ) - .where("$ARCHIVE_CDN > 0 OR $ARCHIVE_MEDIA_ID IS NOT NULL OR $ARCHIVE_MEDIA_NAME IS NOT NULL") .run() } @@ -2332,9 +2315,6 @@ class AttachmentTable( put(CAPTION, attachment.caption) put(UPLOAD_TIMESTAMP, attachment.uploadTimestamp) put(ARCHIVE_CDN, attachment.archiveCdn) - put(ARCHIVE_MEDIA_NAME, attachment.archiveMediaName) - put(ARCHIVE_MEDIA_ID, attachment.archiveMediaId) - put(ARCHIVE_THUMBNAIL_MEDIA_ID, attachment.archiveThumbnailMediaId) put(ARCHIVE_TRANSFER_STATE, ArchiveTransferState.FINISHED.value) put(THUMBNAIL_RESTORE_STATE, ThumbnailRestoreState.NEEDS_RESTORE.value) put(ATTACHMENT_UUID, attachment.uuid?.toString()) @@ -2399,7 +2379,7 @@ class AttachmentTable( // First we'll check if our file hash matches the starting or ending hash of any other attachments and has compatible transform properties. // We'll prefer the match with the most recent upload timestamp. val hashMatch: DataFileInfo? = readableDatabase - .select(ID, DATA_FILE, DATA_SIZE, DATA_RANDOM, DATA_HASH_START, DATA_HASH_END, TRANSFORM_PROPERTIES, UPLOAD_TIMESTAMP, ARCHIVE_CDN, ARCHIVE_MEDIA_NAME, ARCHIVE_MEDIA_ID) + .select(ID, DATA_FILE, DATA_SIZE, DATA_RANDOM, DATA_HASH_START, DATA_HASH_END, TRANSFORM_PROPERTIES, UPLOAD_TIMESTAMP, ARCHIVE_CDN) .from(TABLE_NAME) .where("$DATA_FILE NOT NULL AND ($DATA_HASH_START = ? OR $DATA_HASH_END = ?)", fileWriteResult.hash, fileWriteResult.hash) .run() @@ -2435,8 +2415,6 @@ class AttachmentTable( contentValues.put(DATA_HASH_START, fileWriteResult.hash) contentValues.put(DATA_HASH_END, hashMatch.hashEnd) contentValues.put(ARCHIVE_CDN, hashMatch.archiveCdn) - contentValues.put(ARCHIVE_MEDIA_NAME, hashMatch.archiveMediaName) - contentValues.put(ARCHIVE_MEDIA_ID, hashMatch.archiveMediaId) if (hashMatch.transformProperties.skipTransform) { Log.i(TAG, "[insertAttachmentWithData] The hash match has a DATA_HASH_END and skipTransform=true, so skipping transform of the new file as well. (MessageId: $messageId, ${attachment.uri})") @@ -2597,9 +2575,6 @@ class AttachmentTable( uploadTimestamp = cursor.requireLong(UPLOAD_TIMESTAMP), dataHash = cursor.requireString(DATA_HASH_END), archiveCdn = cursor.requireInt(ARCHIVE_CDN), - archiveMediaName = cursor.requireString(ARCHIVE_MEDIA_NAME), - archiveMediaId = cursor.requireString(ARCHIVE_MEDIA_ID), - hasArchiveThumbnail = !cursor.isNull(THUMBNAIL_FILE), thumbnailRestoreState = ThumbnailRestoreState.deserialize(cursor.requireInt(THUMBNAIL_RESTORE_STATE)), archiveTransferState = ArchiveTransferState.deserialize(cursor.requireInt(ARCHIVE_TRANSFER_STATE)), uuid = UuidUtil.parseOrNull(cursor.requireString(ATTACHMENT_UUID)) @@ -2627,9 +2602,7 @@ class AttachmentTable( hashEnd = this.requireString(DATA_HASH_END), transformProperties = TransformProperties.parse(this.requireString(TRANSFORM_PROPERTIES)), uploadTimestamp = this.requireLong(UPLOAD_TIMESTAMP), - archiveCdn = this.requireInt(ARCHIVE_CDN), - archiveMediaName = this.requireString(ARCHIVE_MEDIA_NAME), - archiveMediaId = this.requireString(ARCHIVE_MEDIA_ID) + archiveCdn = this.requireInt(ARCHIVE_CDN) ) } @@ -2693,9 +2666,7 @@ class AttachmentTable( val hashEnd: String?, val transformProperties: TransformProperties, val uploadTimestamp: Long, - val archiveCdn: Int, - val archiveMediaName: String?, - val archiveMediaId: String? + val archiveCdn: Int ) @VisibleForTesting diff --git a/app/src/main/java/org/thoughtcrime/securesms/database/BackupMediaSnapshotTable.kt b/app/src/main/java/org/thoughtcrime/securesms/database/BackupMediaSnapshotTable.kt index 8d53b9a24b..778fd265b5 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/database/BackupMediaSnapshotTable.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/database/BackupMediaSnapshotTable.kt @@ -10,15 +10,20 @@ import androidx.annotation.VisibleForTesting import androidx.core.content.contentValuesOf import org.signal.core.util.SqlUtil import org.signal.core.util.delete -import org.signal.core.util.exists import org.signal.core.util.readToList +import org.signal.core.util.readToSet import org.signal.core.util.requireInt +import org.signal.core.util.requireNonNullBlob import org.signal.core.util.requireNonNullString import org.signal.core.util.select +import org.signal.core.util.toInt import org.thoughtcrime.securesms.backup.v2.ArchivedMediaObject /** - * Helper table for attachment deletion sync + * When we delete attachments locally, we can't immediately delete them from the archive CDN. This is because there is still a backup that exists that + * references that attachment -- at least until a new backup is made. + * + * So, this table maintains a snapshot of the media present in the last backup, so that we know what we can and can't delete from the archive CDN. */ class BackupMediaSnapshotTable(context: Context, database: SignalDatabase) : DatabaseTable(context, database) { companion object { @@ -50,72 +55,160 @@ class BackupMediaSnapshotTable(context: Context, database: SignalDatabase) : Dat @VisibleForTesting const val PENDING_SYNC_TIME = "pending_sync_time" + /** + * Whether or not this entry is for a thumbnail. + */ + const val IS_THUMBNAIL = "is_thumbnail" + + /** + * The remote digest for the media object. This is used to find matching attachments in the attachment table when necessary. + */ + const val REMOTE_DIGEST = "remote_digest" + val CREATE_TABLE = """ CREATE TABLE $TABLE_NAME ( $ID INTEGER PRIMARY KEY, $MEDIA_ID TEXT UNIQUE, $CDN INTEGER, $LAST_SYNC_TIME INTEGER DEFAULT 0, - $PENDING_SYNC_TIME INTEGER + $PENDING_SYNC_TIME INTEGER, + $IS_THUMBNAIL INTEGER DEFAULT 0, + $REMOTE_DIGEST BLOB NOT NULL ) """.trimIndent() - - private const val ON_MEDIA_ID_CONFLICT = """ - ON CONFLICT($MEDIA_ID) DO UPDATE SET - $PENDING_SYNC_TIME = EXCLUDED.$PENDING_SYNC_TIME, - $CDN = EXCLUDED.$CDN - """ } /** - * Creates the temporary table if it doesn't exist, clears it, then inserts the media objects into it. + * Writes the set of media items that are slated to be referenced in the next backup, updating their pending sync time. + * Will insert multiple rows per object -- one for the main item, and one for the thumbnail. */ - fun writePendingMediaObjects(mediaObjects: Sequence, pendingSyncTime: Long) { - mediaObjects.chunked(999) + fun writePendingMediaObjects(mediaObjects: Sequence, pendingSyncTime: Long) { + mediaObjects + .chunked(SqlUtil.MAX_QUERY_ARGS) .forEach { chunk -> - writePendingMediaObjectsChunk(chunk, pendingSyncTime) - } - } + writePendingMediaObjectsChunk( + chunk.map { MediaEntry(it.mediaId, it.cdn, it.digest, isThumbnail = false) }, + pendingSyncTime + ) - private fun writePendingMediaObjectsChunk(chunk: List, pendingSyncTime: Long) { - SqlUtil.buildBulkInsert( - TABLE_NAME, - arrayOf(MEDIA_ID, CDN, PENDING_SYNC_TIME), - chunk.map { - contentValuesOf(MEDIA_ID to it.mediaId, CDN to it.cdn, PENDING_SYNC_TIME to pendingSyncTime) + writePendingMediaObjectsChunk( + chunk.map { MediaEntry(it.thumbnailMediaId, it.cdn, it.digest, isThumbnail = true) }, + pendingSyncTime + ) } - ).forEach { - writableDatabase.execSQL("${it.where} $ON_MEDIA_ID_CONFLICT", it.whereArgs) - } } /** - * Copies all entries from the temporary table to the persistent table, then deletes the temporary table. + * Commits the pending sync time to the last sync time. This is called once a backup has been successfully uploaded. */ fun commitPendingRows() { writableDatabase.execSQL("UPDATE $TABLE_NAME SET $LAST_SYNC_TIME = $PENDING_SYNC_TIME") } - fun getPageOfOldMediaObjects(currentSyncTime: Long, pageSize: Int): List { + fun getPageOfOldMediaObjects(currentSyncTime: Long, pageSize: Int): Set { return readableDatabase.select(MEDIA_ID, CDN) .from(TABLE_NAME) .where("$LAST_SYNC_TIME < ? AND $LAST_SYNC_TIME = $PENDING_SYNC_TIME", currentSyncTime) .limit(pageSize) .run() - .readToList { + .readToSet { ArchivedMediaObject(mediaId = it.requireNonNullString(MEDIA_ID), cdn = it.requireInt(CDN)) } } - fun deleteMediaObjects(mediaObjects: List) { - SqlUtil.buildCollectionQuery(MEDIA_ID, mediaObjects.map { it.mediaId }).forEach { - writableDatabase.delete(TABLE_NAME) - .where(it.where, it.whereArgs) - .run() + fun deleteMediaObjects(mediaObjects: Collection) { + val query = SqlUtil.buildFastCollectionQuery(MEDIA_ID, mediaObjects.map { it.mediaId }) + + writableDatabase.delete(TABLE_NAME) + .where(query.where, query.whereArgs) + .run() + } + + /** + * Given a list of media objects, find the ones that we have no knowledge of in our local store. + */ + fun getMediaObjectsThatCantBeFound(objects: List): Set { + val query = SqlUtil.buildSingleCollectionQuery( + column = MEDIA_ID, + values = objects.map { it.mediaId }, + collectionOperator = SqlUtil.CollectionOperator.NOT_IN, + prefix = "$IS_THUMBNAIL = 0 AND " + ) + + return readableDatabase + .select(MEDIA_ID, CDN) + .from(TABLE_NAME) + .where(query.where, query.whereArgs) + .run() + .readToSet { + ArchivedMediaObject( + mediaId = it.requireNonNullString(MEDIA_ID), + cdn = it.requireInt(CDN) + ) + } + } + + /** + * Given a list of media objects, find the ones that we have no knowledge of in our local store. + */ + fun getMediaObjectsWithNonMatchingCdn(objects: List): List { + val inputValues = objects.joinToString(separator = ", ") { "('${it.mediaId}', ${it.cdn})" } + return readableDatabase.rawQuery( + """ + WITH input_pairs($MEDIA_ID, $CDN) AS (VALUES $inputValues) + SELECT a.$REMOTE_DIGEST, b.$CDN + FROM $TABLE_NAME a + JOIN input_pairs b ON a.$MEDIA_ID = b.$MEDIA_ID + WHERE a.$CDN != b.$CDN AND a.$IS_THUMBNAIL = 0 + """ + ).readToList { cursor -> + CdnMismatchResult( + digest = cursor.requireNonNullBlob(REMOTE_DIGEST), + cdn = cursor.requireInt(CDN) + ) } } - fun hasOldMediaObjects(currentSyncTime: Long): Boolean { - return readableDatabase.exists(TABLE_NAME).where("$LAST_SYNC_TIME > ? AND $LAST_SYNC_TIME = $PENDING_SYNC_TIME", currentSyncTime).run() + private fun writePendingMediaObjectsChunk(chunk: List, pendingSyncTime: Long) { + val values = chunk.map { + contentValuesOf( + MEDIA_ID to it.mediaId, + CDN to it.cdn, + REMOTE_DIGEST to it.digest, + IS_THUMBNAIL to it.isThumbnail.toInt(), + PENDING_SYNC_TIME to pendingSyncTime + ) + } + + val query = SqlUtil.buildSingleBulkInsert(TABLE_NAME, arrayOf(MEDIA_ID, CDN, REMOTE_DIGEST, IS_THUMBNAIL, PENDING_SYNC_TIME), values) + + writableDatabase.execSQL( + """ + ${query.where} + ON CONFLICT($MEDIA_ID) DO UPDATE SET + $PENDING_SYNC_TIME = EXCLUDED.$PENDING_SYNC_TIME, + $CDN = EXCLUDED.$CDN + """, + query.whereArgs + ) } + + class ArchiveMediaItem( + val mediaId: String, + val thumbnailMediaId: String, + val cdn: Int, + val digest: ByteArray + ) + + class CdnMismatchResult( + val digest: ByteArray, + val cdn: Int + ) + + private data class MediaEntry( + val mediaId: String, + val cdn: Int, + val digest: ByteArray, + val isThumbnail: Boolean + ) } diff --git a/app/src/main/java/org/thoughtcrime/securesms/database/MediaTable.kt b/app/src/main/java/org/thoughtcrime/securesms/database/MediaTable.kt index 12448a8588..4674bc83ce 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/database/MediaTable.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/database/MediaTable.kt @@ -53,8 +53,6 @@ class MediaTable internal constructor(context: Context?, databaseHelper: SignalD ${AttachmentTable.TABLE_NAME}.${AttachmentTable.REMOTE_INCREMENTAL_DIGEST_CHUNK_SIZE}, ${AttachmentTable.TABLE_NAME}.${AttachmentTable.DATA_HASH_END}, ${AttachmentTable.TABLE_NAME}.${AttachmentTable.ARCHIVE_CDN}, - ${AttachmentTable.TABLE_NAME}.${AttachmentTable.ARCHIVE_MEDIA_NAME}, - ${AttachmentTable.TABLE_NAME}.${AttachmentTable.ARCHIVE_MEDIA_ID}, ${AttachmentTable.TABLE_NAME}.${AttachmentTable.THUMBNAIL_RESTORE_STATE}, ${AttachmentTable.TABLE_NAME}.${AttachmentTable.ARCHIVE_TRANSFER_STATE}, ${AttachmentTable.TABLE_NAME}.${AttachmentTable.ATTACHMENT_UUID}, diff --git a/app/src/main/java/org/thoughtcrime/securesms/database/MessageTable.kt b/app/src/main/java/org/thoughtcrime/securesms/database/MessageTable.kt index 41efeab471..5fdac0f729 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/database/MessageTable.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/database/MessageTable.kt @@ -389,8 +389,6 @@ open class MessageTable(context: Context?, databaseHelper: SignalDatabase) : Dat '${AttachmentTable.UPLOAD_TIMESTAMP}', ${AttachmentTable.TABLE_NAME}.${AttachmentTable.UPLOAD_TIMESTAMP}, '${AttachmentTable.DATA_HASH_END}', ${AttachmentTable.TABLE_NAME}.${AttachmentTable.DATA_HASH_END}, '${AttachmentTable.ARCHIVE_CDN}', ${AttachmentTable.TABLE_NAME}.${AttachmentTable.ARCHIVE_CDN}, - '${AttachmentTable.ARCHIVE_MEDIA_NAME}', ${AttachmentTable.TABLE_NAME}.${AttachmentTable.ARCHIVE_MEDIA_NAME}, - '${AttachmentTable.ARCHIVE_MEDIA_ID}', ${AttachmentTable.TABLE_NAME}.${AttachmentTable.ARCHIVE_MEDIA_ID}, '${AttachmentTable.THUMBNAIL_RESTORE_STATE}', ${AttachmentTable.TABLE_NAME}.${AttachmentTable.THUMBNAIL_RESTORE_STATE}, '${AttachmentTable.ARCHIVE_TRANSFER_STATE}', ${AttachmentTable.TABLE_NAME}.${AttachmentTable.ARCHIVE_TRANSFER_STATE}, '${AttachmentTable.ATTACHMENT_UUID}', ${AttachmentTable.TABLE_NAME}.${AttachmentTable.ATTACHMENT_UUID} diff --git a/app/src/main/java/org/thoughtcrime/securesms/database/helpers/SignalDatabaseMigrations.kt b/app/src/main/java/org/thoughtcrime/securesms/database/helpers/SignalDatabaseMigrations.kt index 5300a8d8b2..071abf0cec 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/database/helpers/SignalDatabaseMigrations.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/database/helpers/SignalDatabaseMigrations.kt @@ -123,6 +123,7 @@ import org.thoughtcrime.securesms.database.helpers.migration.V265_FixFtsTriggers import org.thoughtcrime.securesms.database.helpers.migration.V266_UniqueThreadPinOrder import org.thoughtcrime.securesms.database.helpers.migration.V267_FixGroupInvitationDeclinedUpdate import org.thoughtcrime.securesms.database.helpers.migration.V268_FixInAppPaymentsErrorStateConsistency +import org.thoughtcrime.securesms.database.helpers.migration.V269_BackupMediaSnapshotChanges import org.thoughtcrime.securesms.database.SQLiteDatabase as SignalSqliteDatabase /** @@ -251,10 +252,11 @@ object SignalDatabaseMigrations { 265 to V265_FixFtsTriggers, 266 to V266_UniqueThreadPinOrder, 267 to V267_FixGroupInvitationDeclinedUpdate, - 268 to V268_FixInAppPaymentsErrorStateConsistency + 268 to V268_FixInAppPaymentsErrorStateConsistency, + 269 to V269_BackupMediaSnapshotChanges ) - const val DATABASE_VERSION = 268 + const val DATABASE_VERSION = 269 @JvmStatic fun migrate(context: Application, db: SignalSqliteDatabase, oldVersion: Int, newVersion: Int) { diff --git a/app/src/main/java/org/thoughtcrime/securesms/database/helpers/migration/V269_BackupMediaSnapshotChanges.kt b/app/src/main/java/org/thoughtcrime/securesms/database/helpers/migration/V269_BackupMediaSnapshotChanges.kt new file mode 100644 index 0000000000..ca7b02fa0d --- /dev/null +++ b/app/src/main/java/org/thoughtcrime/securesms/database/helpers/migration/V269_BackupMediaSnapshotChanges.kt @@ -0,0 +1,26 @@ +/* + * Copyright 2025 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ + +package org.thoughtcrime.securesms.database.helpers.migration + +import android.app.Application +import org.thoughtcrime.securesms.database.SQLiteDatabase + +/** + * We made a change to stop storing mediaId/names in favor of computing them on-the-fly. + * So, this change removes those columns and adds some plumbing elsewhere that we need to keep things glued together correctly. + */ +object V269_BackupMediaSnapshotChanges : SignalDatabaseMigration { + override fun migrate(context: Application, db: SQLiteDatabase, oldVersion: Int, newVersion: Int) { + db.execSQL("DROP INDEX attachment_archive_media_id_index") + db.execSQL("ALTER TABLE attachment DROP COLUMN archive_media_id") + db.execSQL("ALTER TABLE attachment DROP COLUMN archive_media_name") + db.execSQL("ALTER TABLE attachment DROP COLUMN archive_thumbnail_media_id") + db.execSQL("CREATE INDEX IF NOT EXISTS attachment_remote_digest_index ON attachment (remote_digest);") + + db.execSQL("ALTER TABLE backup_media_snapshot ADD COLUMN is_thumbnail INTEGER DEFAULT 0") + db.execSQL("ALTER TABLE backup_media_snapshot ADD COLUMN remote_digest BLOB NOT NULL") + } +} diff --git a/app/src/main/java/org/thoughtcrime/securesms/jobs/ArchiveThumbnailUploadJob.kt b/app/src/main/java/org/thoughtcrime/securesms/jobs/ArchiveThumbnailUploadJob.kt index 3ffc7f21bc..e0d878ec5d 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/jobs/ArchiveThumbnailUploadJob.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/jobs/ArchiveThumbnailUploadJob.kt @@ -13,8 +13,7 @@ import org.thoughtcrime.securesms.attachments.AttachmentId import org.thoughtcrime.securesms.attachments.DatabaseAttachment import org.thoughtcrime.securesms.attachments.PointerAttachment import org.thoughtcrime.securesms.backup.v2.BackupRepository -import org.thoughtcrime.securesms.backup.v2.BackupRepository.getMediaName -import org.thoughtcrime.securesms.backup.v2.BackupRepository.getThumbnailMediaName +import org.thoughtcrime.securesms.backup.v2.requireThumbnailMediaName import org.thoughtcrime.securesms.database.SignalDatabase import org.thoughtcrime.securesms.dependencies.AppDependencies import org.thoughtcrime.securesms.jobmanager.Job @@ -98,7 +97,7 @@ class ArchiveThumbnailUploadJob private constructor( .getAttachmentUploadForm() .then { form -> SignalNetwork.attachments.getResumableUploadSpec( - key = mediaRootBackupKey.deriveThumbnailTransitKey(attachment.getThumbnailMediaName()), + key = mediaRootBackupKey.deriveThumbnailTransitKey(attachment.requireThumbnailMediaName()), iv = attachment.remoteIv!!, uploadForm = form ) @@ -133,15 +132,18 @@ class ArchiveThumbnailUploadJob private constructor( return Result.retry(defaultBackoff()) } - val mediaSecrets = mediaRootBackupKey.deriveMediaSecrets(attachment.getThumbnailMediaName()) + val mediaSecrets = mediaRootBackupKey.deriveMediaSecrets(attachment.requireThumbnailMediaName()) return when (val result = BackupRepository.copyThumbnailToArchive(attachmentPointer, attachment)) { is NetworkResult.Success -> { // save attachment thumbnail - val archiveMediaId = attachment.archiveMediaId ?: mediaRootBackupKey.deriveMediaId(attachment.getMediaName()).encode() - SignalDatabase.attachments.finalizeAttachmentThumbnailAfterUpload(attachmentId, archiveMediaId, mediaSecrets.id, thumbnailResult.data) + SignalDatabase.attachments.finalizeAttachmentThumbnailAfterUpload( + attachmentId = attachmentId, + attachmentDigest = attachment.remoteDigest!!, + data = thumbnailResult.data + ) - Log.d(TAG, "Successfully archived thumbnail for $attachmentId mediaName=${attachment.getThumbnailMediaName()}") + Log.d(TAG, "Successfully archived thumbnail for $attachmentId mediaName=${attachment.requireThumbnailMediaName()}") Result.success() } is NetworkResult.NetworkError -> { diff --git a/app/src/main/java/org/thoughtcrime/securesms/jobs/AttachmentDownloadJob.kt b/app/src/main/java/org/thoughtcrime/securesms/jobs/AttachmentDownloadJob.kt index 8da3facec6..a31a072324 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/jobs/AttachmentDownloadJob.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/jobs/AttachmentDownloadJob.kt @@ -94,12 +94,12 @@ class AttachmentDownloadJob private constructor( AttachmentTable.TRANSFER_PROGRESS_PENDING, AttachmentTable.TRANSFER_PROGRESS_FAILED -> { if (SignalStore.backup.backsUpMedia && databaseAttachment.remoteLocation == null) { - if (databaseAttachment.archiveMediaName.isNullOrEmpty()) { - Log.w(TAG, "No remote location or archive media name, can't download") - null - } else { + if (databaseAttachment.archiveTransferState == AttachmentTable.ArchiveTransferState.FINISHED) { Log.i(TAG, "Trying to restore attachment from archive cdn") RestoreAttachmentJob.restoreAttachment(databaseAttachment) + } else { + Log.w(TAG, "No remote location, and the archive transfer state is unfinished. Can't download.") + null } } else { val downloadJob = AttachmentDownloadJob( @@ -200,8 +200,8 @@ class AttachmentDownloadJob private constructor( } if (SignalStore.backup.backsUpMedia && attachment.remoteLocation == null) { - if (attachment.archiveMediaName.isNullOrEmpty()) { - throw InvalidAttachmentException("No remote location or archive media name") + if (attachment.archiveTransferState != AttachmentTable.ArchiveTransferState.FINISHED) { + throw InvalidAttachmentException("No remote location, and the archive transfer state is unfinished. Can't download.") } Log.i(TAG, "Trying to restore attachment from archive cdn instead") @@ -310,7 +310,7 @@ class AttachmentDownloadJob private constructor( Log.w(TAG, "Experienced exception while trying to download an attachment.", e) markFailed(messageId, attachmentId) } catch (e: NonSuccessfulResponseCodeException) { - if (SignalStore.backup.backsUpMedia && e.code == 404 && attachment.archiveMediaName?.isNotEmpty() == true) { + if (SignalStore.backup.backsUpMedia && e.code == 404 && attachment.archiveTransferState === AttachmentTable.ArchiveTransferState.FINISHED) { Log.i(TAG, "Retrying download from archive CDN") RestoreAttachmentJob.restoreAttachment(attachment) return false diff --git a/app/src/main/java/org/thoughtcrime/securesms/jobs/BackupMediaSnapshotSyncJob.kt b/app/src/main/java/org/thoughtcrime/securesms/jobs/BackupMediaSnapshotSyncJob.kt index f0a675e6f2..98a3684bce 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/jobs/BackupMediaSnapshotSyncJob.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/jobs/BackupMediaSnapshotSyncJob.kt @@ -6,19 +6,33 @@ package org.thoughtcrime.securesms.jobs import org.signal.core.util.logging.Log +import org.signal.core.util.nullIfBlank +import org.thoughtcrime.securesms.backup.v2.ArchivedMediaObject import org.thoughtcrime.securesms.backup.v2.BackupRepository import org.thoughtcrime.securesms.database.SignalDatabase import org.thoughtcrime.securesms.dependencies.AppDependencies import org.thoughtcrime.securesms.jobmanager.Job import org.thoughtcrime.securesms.jobmanager.impl.NetworkConstraint import org.thoughtcrime.securesms.jobs.protos.BackupMediaSnapshotSyncJobData +import org.thoughtcrime.securesms.keyvalue.SignalStore import org.whispersystems.signalservice.api.NetworkResult +import org.whispersystems.signalservice.api.archive.ArchiveGetMediaItemsResponse +import java.lang.RuntimeException +import kotlin.time.Duration.Companion.days +import kotlin.time.Duration.Companion.hours /** - * Synchronizes the server media via bulk deletions of old attachments not present - * in the user's current backup. + * When we delete attachments locally, we can't immediately delete them from the archive CDN. This is because there is still a backup that exists that + * references that attachment -- at least until a new backup is made. + * + * This job uses data we store locally in [org.thoughtcrime.securesms.database.BackupMediaSnapshotTable] to determine which media objects can be safely + * deleted from the archive CDN, and then deletes them. */ -class BackupMediaSnapshotSyncJob private constructor(private val syncTime: Long, parameters: Parameters) : Job(parameters) { +class BackupMediaSnapshotSyncJob private constructor( + private val syncTime: Long, + private var serverCursor: String?, + parameters: Parameters +) : Job(parameters) { companion object { @@ -26,51 +40,186 @@ class BackupMediaSnapshotSyncJob private constructor(private val syncTime: Long, const val KEY = "BackupMediaSnapshotSyncJob" - private const val PAGE_SIZE = 500 + private const val REMOTE_DELETE_BATCH_SIZE = 500 + private val BACKUP_MEDIA_SYNC_INTERVAL = 7.days.inWholeMilliseconds - fun enqueue(backupSnapshotId: Long) { + fun enqueue(syncTime: Long) { AppDependencies.jobManager.add( BackupMediaSnapshotSyncJob( - backupSnapshotId, - Parameters.Builder() + syncTime = syncTime, + serverCursor = null, + parameters = Parameters.Builder() .addConstraint(NetworkConstraint.KEY) - .setMaxInstancesForFactory(1) + .setQueue("BackupMediaSnapshotSyncJob") + .setMaxAttempts(Parameters.UNLIMITED) + .setLifespan(12.hours.inWholeMilliseconds) .build() ) ) } } - override fun serialize(): ByteArray = BackupMediaSnapshotSyncJobData(syncTime).encode() + override fun serialize(): ByteArray = BackupMediaSnapshotSyncJobData(syncTime, serverCursor ?: "").encode() override fun getFactoryKey(): String = KEY override fun run(): Result { - while (SignalDatabase.backupMediaSnapshots.hasOldMediaObjects(syncTime)) { - val mediaObjects = SignalDatabase.backupMediaSnapshots.getPageOfOldMediaObjects(syncTime, PAGE_SIZE) - - when (val networkResult = BackupRepository.deleteAbandonedMediaObjects(mediaObjects)) { - is NetworkResult.Success -> { - SignalDatabase.backupMediaSnapshots.deleteMediaObjects(mediaObjects) - } - - else -> { - Log.w(TAG, "Failed to delete media objects.", networkResult.getCause()) - return Result.failure() - } - } + if (serverCursor == null) { + removeLocallyDeletedAttachmentsFromCdn()?.let { result -> return result } + } else { + Log.d(TAG, "Already deleted old attachments from CDN. Skipping to syncing with remote.") } - return Result.success() + val timeSinceLastRemoteSync = System.currentTimeMillis() - SignalStore.backup.lastMediaSyncTime + if (serverCursor == null && timeSinceLastRemoteSync > 0 && timeSinceLastRemoteSync < BACKUP_MEDIA_SYNC_INTERVAL) { + Log.d(TAG, "No need to do a remote sync yet. Time since last sync: $timeSinceLastRemoteSync ms") + return Result.success() + } + + return syncDataFromCdn() ?: Result.success() } override fun onFailure() = Unit + /** + * Looks through our local snapshot of what attachments we put in the last backup file, and uses that to delete any old attachments from the archive CDN + * that we no longer need. + */ + private fun removeLocallyDeletedAttachmentsFromCdn(): Result? { + var mediaObjects = SignalDatabase.backupMediaSnapshots.getPageOfOldMediaObjects(syncTime, REMOTE_DELETE_BATCH_SIZE) + + while (mediaObjects.isNotEmpty()) { + deleteMediaObjectsFromCdn(mediaObjects)?.let { result -> return result } + SignalDatabase.backupMediaSnapshots.deleteMediaObjects(mediaObjects) + + mediaObjects = SignalDatabase.backupMediaSnapshots.getPageOfOldMediaObjects(syncTime, REMOTE_DELETE_BATCH_SIZE) + } + + return null + } + + /** + * Fetches all attachment metadata from the archive CDN and ensures that our local store is in sync with it. + * + * Specifically, we make sure that: + * (1) We delete any attachments from the CDN that we have no knowledge of in any backup. + * (2) We ensure that our local store has the correct CDN for any attachments on the CDN (they should only really fall out of sync when you restore a backup + * that was made before all of the attachments had been uploaded). + */ + private fun syncDataFromCdn(): Result? { + val attachmentsToDelete = HashSet() + var cursor: String? = serverCursor + + do { + val (archivedItemPage, jobResult) = getRemoteArchiveItemPage(cursor) + if (jobResult != null) { + return jobResult + } + check(archivedItemPage != null) + + cursor = archivedItemPage.cursor + attachmentsToDelete += syncCdnPage(archivedItemPage) + + if (attachmentsToDelete.size >= REMOTE_DELETE_BATCH_SIZE) { + deleteMediaObjectsFromCdn(attachmentsToDelete)?.let { result -> return result } + attachmentsToDelete.clear() + } + + // We don't persist attachmentsToDelete, so we can only update the persisted serverCursor if there's no pending deletes + if (attachmentsToDelete.isEmpty()) { + serverCursor = archivedItemPage.cursor + } + } while (cursor != null) + + if (attachmentsToDelete.isNotEmpty()) { + deleteMediaObjectsFromCdn(attachmentsToDelete)?.let { result -> return result } + } + + SignalStore.backup.lastMediaSyncTime = System.currentTimeMillis() + + return null + } + + /** + * Update CDNs of archived media items. Returns set of objects that don't match + * to a local attachment DB row. + */ + private fun syncCdnPage(archivedItemPage: ArchiveGetMediaItemsResponse): Set { + val mediaObjects = archivedItemPage.storedMediaObjects.map { + ArchivedMediaObject( + mediaId = it.mediaId, + cdn = it.cdn + ) + } + + val notFoundMediaObjects = SignalDatabase.backupMediaSnapshots.getMediaObjectsThatCantBeFound(mediaObjects) + val remainingObjects = mediaObjects - notFoundMediaObjects + + val cdnMismatches = SignalDatabase.backupMediaSnapshots.getMediaObjectsWithNonMatchingCdn(remainingObjects) + if (cdnMismatches.isNotEmpty()) { + Log.w(TAG, "Found ${cdnMismatches.size} items with CDNs that differ from what we have locally. Updating our local store.") + for (mismatch in cdnMismatches) { + SignalDatabase.attachments.setArchiveCdnByDigest(mismatch.digest, mismatch.cdn) + } + } + + return notFoundMediaObjects + } + + private fun getRemoteArchiveItemPage(cursor: String?): Pair { + return when (val result = BackupRepository.listRemoteMediaObjects(100, cursor)) { + is NetworkResult.Success -> result.result to null + is NetworkResult.NetworkError -> return null to Result.retry(defaultBackoff()) + is NetworkResult.StatusCodeError -> { + if (result.code == 429) { + Log.w(TAG, "Rate limited while attempting to list media objects. Retrying later.") + return null to Result.retry(result.retryAfter()?.inWholeMilliseconds ?: defaultBackoff()) + } else { + Log.w(TAG, "Failed to list remote media objects with code: ${result.code}. Unable to proceed.", result.getCause()) + return null to Result.failure() + } + } + is NetworkResult.ApplicationError -> { + Log.w(TAG, "Failed to list remote media objects due to a crash.", result.getCause()) + return null to Result.fatalFailure(RuntimeException(result.getCause())) + } + } + } + + private fun deleteMediaObjectsFromCdn(attachmentsToDelete: Set): Result? { + when (val result = BackupRepository.deleteAbandonedMediaObjects(attachmentsToDelete)) { + is NetworkResult.Success -> { + Log.i(TAG, "Successfully deleted ${attachmentsToDelete.size} attachments off of the CDN.") + } + is NetworkResult.NetworkError -> { + return Result.retry(defaultBackoff()) + } + is NetworkResult.StatusCodeError -> { + if (result.code == 429) { + Log.w(TAG, "Rate limited while attempting to delete media objects. Retrying later.") + return Result.retry(result.retryAfter()?.inWholeMilliseconds ?: defaultBackoff()) + } else { + Log.w(TAG, "Failed to delete attachments from CDN with code: ${result.code}. Not failing job, just skipping and trying next page.", result.getCause()) + } + } + else -> { + Log.w(TAG, "Crash when trying to delete attachments from the CDN", result.getCause()) + return Result.fatalFailure(RuntimeException(result.getCause())) + } + } + + return null + } + class Factory : Job.Factory { override fun create(parameters: Parameters, serializedData: ByteArray?): BackupMediaSnapshotSyncJob { - val syncTime: Long = BackupMediaSnapshotSyncJobData.ADAPTER.decode(serializedData!!).syncTime + val data = BackupMediaSnapshotSyncJobData.ADAPTER.decode(serializedData!!) - return BackupMediaSnapshotSyncJob(syncTime, parameters) + return BackupMediaSnapshotSyncJob( + syncTime = data.syncTime, + serverCursor = data.serverCursor.nullIfBlank(), + parameters = parameters + ) } } } diff --git a/app/src/main/java/org/thoughtcrime/securesms/jobs/BackupMessagesJob.kt b/app/src/main/java/org/thoughtcrime/securesms/jobs/BackupMessagesJob.kt index bea37bec3c..a37385a440 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/jobs/BackupMessagesJob.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/jobs/BackupMessagesJob.kt @@ -10,8 +10,8 @@ import org.signal.core.util.isNotNullOrBlank import org.signal.core.util.logging.Log import org.signal.protos.resumableuploads.ResumableUpload import org.thoughtcrime.securesms.backup.ArchiveUploadProgress +import org.thoughtcrime.securesms.backup.v2.ArchiveMediaItemIterator import org.thoughtcrime.securesms.backup.v2.ArchiveValidator -import org.thoughtcrime.securesms.backup.v2.ArchivedMediaObjectIterator import org.thoughtcrime.securesms.backup.v2.BackupRepository import org.thoughtcrime.securesms.backup.v2.ResumableMessagesBackupUploadSpec import org.thoughtcrime.securesms.database.SignalDatabase @@ -52,15 +52,11 @@ class BackupMessagesJob private constructor( * Pruning abandoned remote media is relatively expensive, so we should * not do this every time we backup. */ - fun enqueue(pruneAbandonedRemoteMedia: Boolean = false) { + fun enqueue() { val jobManager = AppDependencies.jobManager val chain = jobManager.startChain(BackupMessagesJob()) - if (pruneAbandonedRemoteMedia) { - chain.then(SyncArchivedMediaJob()) - } - if (SignalStore.backup.optimizeStorage && SignalStore.backup.backsUpMedia) { chain.then(OptimizeMediaJob()) } @@ -272,9 +268,9 @@ class BackupMessagesJob private constructor( private fun writeMediaCursorToTemporaryTable(db: SignalDatabase, mediaBackupEnabled: Boolean, currentTime: Long) { if (mediaBackupEnabled) { - db.attachmentTable.getMediaIdCursor().use { + db.attachmentTable.getAttachmentsEligibleForArchiveUpload().use { SignalDatabase.backupMediaSnapshots.writePendingMediaObjects( - mediaObjects = ArchivedMediaObjectIterator(it).asSequence(), + mediaObjects = ArchiveMediaItemIterator(it).asSequence(), pendingSyncTime = currentTime ) } diff --git a/app/src/main/java/org/thoughtcrime/securesms/jobs/JobManagerFactories.java b/app/src/main/java/org/thoughtcrime/securesms/jobs/JobManagerFactories.java index 41c99acf3f..85e6d85671 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/jobs/JobManagerFactories.java +++ b/app/src/main/java/org/thoughtcrime/securesms/jobs/JobManagerFactories.java @@ -263,7 +263,6 @@ public final class JobManagerFactories { put(SubmitRateLimitPushChallengeJob.KEY, new SubmitRateLimitPushChallengeJob.Factory()); put(Svr2MirrorJob.KEY, new Svr2MirrorJob.Factory()); put(Svr3MirrorJob.KEY, new Svr3MirrorJob.Factory()); - put(SyncArchivedMediaJob.KEY, new SyncArchivedMediaJob.Factory()); put(ThreadUpdateJob.KEY, new ThreadUpdateJob.Factory()); put(TrimThreadJob.KEY, new TrimThreadJob.Factory()); put(TypingSendJob.KEY, new TypingSendJob.Factory()); diff --git a/app/src/main/java/org/thoughtcrime/securesms/jobs/RestoreAttachmentJob.kt b/app/src/main/java/org/thoughtcrime/securesms/jobs/RestoreAttachmentJob.kt index 541c985f58..cdef890a26 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/jobs/RestoreAttachmentJob.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/jobs/RestoreAttachmentJob.kt @@ -19,7 +19,8 @@ import org.thoughtcrime.securesms.attachments.AttachmentId import org.thoughtcrime.securesms.attachments.DatabaseAttachment import org.thoughtcrime.securesms.attachments.InvalidAttachmentException import org.thoughtcrime.securesms.backup.v2.BackupRepository -import org.thoughtcrime.securesms.backup.v2.database.createArchiveAttachmentPointer +import org.thoughtcrime.securesms.backup.v2.createArchiveAttachmentPointer +import org.thoughtcrime.securesms.backup.v2.requireMediaName import org.thoughtcrime.securesms.database.AttachmentTable import org.thoughtcrime.securesms.database.SignalDatabase import org.thoughtcrime.securesms.dependencies.AppDependencies @@ -36,7 +37,6 @@ import org.thoughtcrime.securesms.notifications.NotificationChannels import org.thoughtcrime.securesms.notifications.NotificationIds import org.thoughtcrime.securesms.transport.RetryLaterException import org.thoughtcrime.securesms.util.RemoteConfig -import org.whispersystems.signalservice.api.backup.MediaName import org.whispersystems.signalservice.api.messages.SignalServiceAttachment import org.whispersystems.signalservice.api.push.exceptions.MissingConfigurationException import org.whispersystems.signalservice.api.push.exceptions.NonSuccessfulResponseCodeException @@ -211,8 +211,8 @@ class RestoreAttachmentJob private constructor( } useArchiveCdn = if (SignalStore.backup.backsUpMedia && !forceTransitTier) { - if (attachment.archiveMediaName.isNullOrEmpty()) { - throw InvalidAttachmentException("Invalid attachment configuration") + if (attachment.archiveTransferState != AttachmentTable.ArchiveTransferState.FINISHED) { + throw InvalidAttachmentException("Invalid attachment configuration! backsUpMedia: ${SignalStore.backup.backsUpMedia}, forceTransitTier: $forceTransitTier, archiveTransferState: ${attachment.archiveTransferState}") } true } else { @@ -238,7 +238,7 @@ class RestoreAttachmentJob private constructor( messageReceiver .retrieveArchivedAttachment( - SignalStore.backup.mediaRootBackupKey.deriveMediaSecrets(MediaName(attachment.archiveMediaName!!)), + SignalStore.backup.mediaRootBackupKey.deriveMediaSecrets(attachment.requireMediaName()), cdnCredentials, archiveFile, pointer, diff --git a/app/src/main/java/org/thoughtcrime/securesms/jobs/RestoreAttachmentThumbnailJob.kt b/app/src/main/java/org/thoughtcrime/securesms/jobs/RestoreAttachmentThumbnailJob.kt index 92b6157b62..433fa8fce9 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/jobs/RestoreAttachmentThumbnailJob.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/jobs/RestoreAttachmentThumbnailJob.kt @@ -9,8 +9,8 @@ import org.signal.libsignal.protocol.InvalidMessageException import org.thoughtcrime.securesms.attachments.AttachmentId import org.thoughtcrime.securesms.attachments.InvalidAttachmentException import org.thoughtcrime.securesms.backup.v2.BackupRepository -import org.thoughtcrime.securesms.backup.v2.BackupRepository.getThumbnailMediaName -import org.thoughtcrime.securesms.backup.v2.database.createArchiveThumbnailPointer +import org.thoughtcrime.securesms.backup.v2.createArchiveThumbnailPointer +import org.thoughtcrime.securesms.backup.v2.requireThumbnailMediaName import org.thoughtcrime.securesms.database.AttachmentTable import org.thoughtcrime.securesms.database.SignalDatabase import org.thoughtcrime.securesms.dependencies.AppDependencies @@ -112,8 +112,8 @@ class RestoreAttachmentThumbnailJob private constructor( return } - if (attachment.archiveMediaName == null) { - Log.w(TAG, "$attachmentId was never archived! Cannot proceed.") + if (attachment.remoteDigest == null) { + Log.w(TAG, "$attachmentId has no digest! Cannot proceed.") return } @@ -132,7 +132,7 @@ class RestoreAttachmentThumbnailJob private constructor( Log.i(TAG, "Downloading thumbnail for $attachmentId") val downloadResult = AppDependencies.signalServiceMessageReceiver .retrieveArchivedAttachment( - SignalStore.backup.mediaRootBackupKey.deriveMediaSecrets(attachment.getThumbnailMediaName()), + SignalStore.backup.mediaRootBackupKey.deriveMediaSecrets(attachment.requireThumbnailMediaName()), cdnCredentials, thumbnailTransferFile, pointer, @@ -142,7 +142,7 @@ class RestoreAttachmentThumbnailJob private constructor( progressListener ) - SignalDatabase.attachments.finalizeAttachmentThumbnailAfterDownload(attachmentId, attachment.archiveMediaId!!, downloadResult.dataStream, thumbnailTransferFile) + SignalDatabase.attachments.finalizeAttachmentThumbnailAfterDownload(attachmentId, attachment.remoteDigest!!, downloadResult.dataStream, thumbnailTransferFile) if (!SignalDatabase.messages.isStory(messageId)) { AppDependencies.messageNotifier.updateNotification(context) diff --git a/app/src/main/java/org/thoughtcrime/securesms/jobs/SyncArchivedMediaJob.kt b/app/src/main/java/org/thoughtcrime/securesms/jobs/SyncArchivedMediaJob.kt deleted file mode 100644 index fd4bf13e46..0000000000 --- a/app/src/main/java/org/thoughtcrime/securesms/jobs/SyncArchivedMediaJob.kt +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Copyright 2024 Signal Messenger, LLC - * SPDX-License-Identifier: AGPL-3.0-only - */ - -package org.thoughtcrime.securesms.jobs - -import org.signal.core.util.logging.Log -import org.signal.core.util.withinTransaction -import org.thoughtcrime.securesms.backup.v2.ArchivedMediaObject -import org.thoughtcrime.securesms.backup.v2.BackupRepository -import org.thoughtcrime.securesms.database.SignalDatabase -import org.thoughtcrime.securesms.jobmanager.Job -import org.thoughtcrime.securesms.jobmanager.JsonJobData -import org.thoughtcrime.securesms.keyvalue.SignalStore -import org.whispersystems.signalservice.api.NetworkResult -import org.whispersystems.signalservice.api.archive.ArchiveGetMediaItemsResponse -import org.whispersystems.signalservice.api.push.exceptions.NonSuccessfulResponseCodeException -import java.io.IOException -import java.lang.Exception - -/** - * Job responsible for keeping remote archive media objects in sync. That is - * we make sure our CDN number aligns on all media ids, as well as deleting any - * extra media ids that we don't know about. - */ -class SyncArchivedMediaJob private constructor( - parameters: Parameters, - private var jobCursor: String? -) : BaseJob(parameters) { - - companion object { - private val TAG = Log.tag(BackupRestoreMediaJob::class.java) - - private const val KEY_CURSOR = "cursor" - - const val KEY = "SyncArchivedMediaJob" - } - - constructor(cursor: String? = null) : this( - Parameters.Builder() - .setQueue("SyncArchivedMedia") - .setMaxAttempts(Parameters.UNLIMITED) - .setMaxInstancesForQueue(2) - .build(), - cursor - ) - - override fun serialize(): ByteArray? { - return JsonJobData.Builder() - .putString(KEY_CURSOR, jobCursor) - .serialize() - } - - override fun getFactoryKey(): String = KEY - - override fun onFailure() = Unit - - override fun onRun() { - val batchSize = 100 - val attachmentsToDelete = HashSet() - var cursor: String? = jobCursor - do { - val archivedItemPage = BackupRepository.listRemoteMediaObjects(batchSize, cursor).successOrThrow() - attachmentsToDelete += syncPage(archivedItemPage) - cursor = archivedItemPage.cursor - if (attachmentsToDelete.size >= batchSize) { - when (val result = BackupRepository.deleteAbandonedMediaObjects(attachmentsToDelete)) { - is NetworkResult.Success -> Log.i(TAG, "Deleted ${attachmentsToDelete.size} attachments off CDN") - else -> Log.w(TAG, "Failed to delete attachments from CDN", result.getCause()) - } - attachmentsToDelete.clear() - } - if (attachmentsToDelete.isEmpty()) { - jobCursor = archivedItemPage.cursor - } - } while (cursor != null) - - if (attachmentsToDelete.isNotEmpty()) { - BackupRepository.deleteAbandonedMediaObjects(attachmentsToDelete) - Log.i(TAG, "Deleted ${attachmentsToDelete.size} attachments off CDN") - } - SignalStore.backup.lastMediaSyncTime = System.currentTimeMillis() - } - - /** - * Update CDNs of archived media items. Returns set of objects that don't match - * to a local attachment DB row. - */ - private fun syncPage(archivedItemPage: ArchiveGetMediaItemsResponse): Set { - val abandonedObjects = HashSet() - SignalDatabase.rawDatabase.withinTransaction { - archivedItemPage.storedMediaObjects.forEach { storedMediaObject -> - val rows = SignalDatabase.attachments.updateArchiveCdnByMediaId(archiveMediaId = storedMediaObject.mediaId, archiveCdn = storedMediaObject.cdn) - if (rows == 0) { - abandonedObjects.add(ArchivedMediaObject(storedMediaObject.mediaId, storedMediaObject.cdn)) - } - } - } - return abandonedObjects - } - - override fun onShouldRetry(e: Exception): Boolean { - return e is IOException && e !is NonSuccessfulResponseCodeException - } - - class Factory : Job.Factory { - override fun create(parameters: Parameters, serializedData: ByteArray?): SyncArchivedMediaJob { - val data = JsonJobData.deserialize(serializedData) - return SyncArchivedMediaJob(parameters, if (data.hasString(KEY_CURSOR)) data.getString(KEY_CURSOR) else null) - } - } -} diff --git a/app/src/main/java/org/thoughtcrime/securesms/registrationv3/ui/restore/RemoteRestoreViewModel.kt b/app/src/main/java/org/thoughtcrime/securesms/registrationv3/ui/restore/RemoteRestoreViewModel.kt index 51e8cdbbd2..8af49dea6c 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/registrationv3/ui/restore/RemoteRestoreViewModel.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/registrationv3/ui/restore/RemoteRestoreViewModel.kt @@ -27,7 +27,6 @@ import org.thoughtcrime.securesms.dependencies.AppDependencies import org.thoughtcrime.securesms.jobmanager.JobTracker import org.thoughtcrime.securesms.jobs.BackupRestoreJob import org.thoughtcrime.securesms.jobs.BackupRestoreMediaJob -import org.thoughtcrime.securesms.jobs.SyncArchivedMediaJob import org.thoughtcrime.securesms.keyvalue.Completed import org.thoughtcrime.securesms.keyvalue.SignalStore import org.thoughtcrime.securesms.keyvalue.Skipped @@ -90,7 +89,6 @@ class RemoteRestoreViewModel(isOnlyRestoreOption: Boolean) : ViewModel() { AppDependencies .jobManager .startChain(BackupRestoreJob()) - .then(SyncArchivedMediaJob()) .then(BackupRestoreMediaJob()) .enqueue(listener) diff --git a/app/src/main/java/org/thoughtcrime/securesms/service/MessageBackupListener.kt b/app/src/main/java/org/thoughtcrime/securesms/service/MessageBackupListener.kt index 534357a0fa..fd0a28c0e0 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/service/MessageBackupListener.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/service/MessageBackupListener.kt @@ -13,7 +13,6 @@ import org.thoughtcrime.securesms.util.RemoteConfig import org.thoughtcrime.securesms.util.toMillis import java.time.LocalDateTime import java.util.Random -import kotlin.time.Duration.Companion.days import kotlin.time.Duration.Companion.minutes class MessageBackupListener : PersistentAlarmManagerListener() { @@ -27,15 +26,13 @@ class MessageBackupListener : PersistentAlarmManagerListener() { override fun onAlarm(context: Context, scheduledTime: Long): Long { if (SignalStore.backup.areBackupsEnabled) { - val timeSinceLastSync = System.currentTimeMillis() - SignalStore.backup.lastMediaSyncTime - BackupMessagesJob.enqueue(pruneAbandonedRemoteMedia = timeSinceLastSync >= BACKUP_MEDIA_SYNC_INTERVAL || timeSinceLastSync < 0) + BackupMessagesJob.enqueue() } return setNextBackupTimeToIntervalFromNow() } companion object { private val BACKUP_JITTER_WINDOW_SECONDS = 10.minutes.inWholeSeconds.toInt() - private val BACKUP_MEDIA_SYNC_INTERVAL = 7.days.inWholeMilliseconds @JvmStatic fun schedule(context: Context?) { diff --git a/app/src/main/java/org/thoughtcrime/securesms/video/exo/PartDataSource.java b/app/src/main/java/org/thoughtcrime/securesms/video/exo/PartDataSource.java index 1b4046e93b..2dd1ac5c95 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/video/exo/PartDataSource.java +++ b/app/src/main/java/org/thoughtcrime/securesms/video/exo/PartDataSource.java @@ -14,11 +14,15 @@ import androidx.media3.datasource.TransferListener; import org.signal.core.util.logging.Log; import org.signal.libsignal.protocol.InvalidMessageException; import org.thoughtcrime.securesms.attachments.DatabaseAttachment; +import org.thoughtcrime.securesms.backup.v2.BackupRepository; +import org.thoughtcrime.securesms.backup.v2.DatabaseAttachmentArchiveUtil; import org.thoughtcrime.securesms.database.AttachmentTable; import org.thoughtcrime.securesms.database.SignalDatabase; import org.thoughtcrime.securesms.keyvalue.SignalStore; import org.thoughtcrime.securesms.mms.PartUriParser; import org.signal.core.util.Base64; +import org.whispersystems.signalservice.api.backup.MediaId; +import org.whispersystems.signalservice.api.backup.MediaName; import org.whispersystems.signalservice.api.backup.MediaRootBackupKey; import org.whispersystems.signalservice.api.crypto.AttachmentCipherInputStream; import org.whispersystems.signalservice.api.crypto.AttachmentCipherStreamUtil; @@ -68,10 +72,13 @@ class PartDataSource implements DataSource { if (inProgress && !hasData && hasIncrementalDigest && attachmentKey != null) { final byte[] decode = Base64.decode(attachmentKey); - if (attachment.transferState == AttachmentTable.TRANSFER_RESTORE_IN_PROGRESS && attachment.archiveMediaId != null) { + if (attachment.transferState == AttachmentTable.TRANSFER_RESTORE_IN_PROGRESS && attachment.archiveTransferState == AttachmentTable.ArchiveTransferState.FINISHED) { final File archiveFile = attachmentDatabase.getOrCreateArchiveTransferFile(attachment.attachmentId); try { - MediaRootBackupKey.MediaKeyMaterial mediaKeyMaterial = SignalStore.backup().getMediaRootBackupKey().deriveMediaSecretsFromMediaId(attachment.archiveMediaId); + String mediaName = DatabaseAttachmentArchiveUtil.requireMediaNameAsString(attachment); + String mediaId = MediaName.toMediaIdString(mediaName, SignalStore.backup().getMediaRootBackupKey()); + + MediaRootBackupKey.MediaKeyMaterial mediaKeyMaterial = SignalStore.backup().getMediaRootBackupKey().deriveMediaSecretsFromMediaId(mediaId); long originalCipherLength = AttachmentCipherStreamUtil.getCiphertextLength(PaddingInputStream.getPaddedSize(attachment.size)); this.inputStream = AttachmentCipherInputStream.createStreamingForArchivedAttachment(mediaKeyMaterial, archiveFile, originalCipherLength, attachment.size, attachment.remoteDigest, decode, attachment.getIncrementalDigest(), attachment.incrementalMacChunkSize); diff --git a/app/src/main/protowire/JobData.proto b/app/src/main/protowire/JobData.proto index 3a8b4ffd9d..50c736fc52 100644 --- a/app/src/main/protowire/JobData.proto +++ b/app/src/main/protowire/JobData.proto @@ -142,6 +142,7 @@ message UploadAttachmentToArchiveJobData { message BackupMediaSnapshotSyncJobData { uint64 syncTime = 1; + string serverCursor = 2; } message DeviceNameChangeJobData { diff --git a/app/src/test/java/org/thoughtcrime/securesms/backup/v2/ArchivedMediaObjectIteratorTest.kt b/app/src/test/java/org/thoughtcrime/securesms/backup/v2/ArchivedMediaObjectIteratorTest.kt index 5623fd204f..17c0646553 100644 --- a/app/src/test/java/org/thoughtcrime/securesms/backup/v2/ArchivedMediaObjectIteratorTest.kt +++ b/app/src/test/java/org/thoughtcrime/securesms/backup/v2/ArchivedMediaObjectIteratorTest.kt @@ -4,8 +4,13 @@ import assertk.assertThat import assertk.assertions.hasSize import io.mockk.every import io.mockk.mockk +import io.mockk.mockkObject +import org.junit.Before import org.junit.Test import org.thoughtcrime.securesms.MockCursor +import org.thoughtcrime.securesms.keyvalue.BackupValues +import org.thoughtcrime.securesms.keyvalue.SignalStore +import org.whispersystems.signalservice.api.backup.MediaRootBackupKey class ArchivedMediaObjectIteratorTest { private val cursor = mockk(relaxed = true) { @@ -17,6 +22,15 @@ class ArchivedMediaObjectIteratorTest { every { isAfterLast } answers { callOriginal() } } + @Before + fun setup() { + val mockBackupValues = mockk() + every { mockBackupValues.mediaRootBackupKey } returns MediaRootBackupKey(ByteArray(32)) + + mockkObject(SignalStore) + every { SignalStore.backup } returns mockBackupValues + } + @Test fun `Given a cursor with 0 items, when I convert to a list, then I expect a size of 0`() { runTest(0) @@ -29,7 +43,7 @@ class ArchivedMediaObjectIteratorTest { private fun runTest(size: Int) { every { cursor.count } returns size - val iterator = ArchivedMediaObjectIterator(cursor) + val iterator = ArchiveMediaItemIterator(cursor) val list = iterator.asSequence().toList() diff --git a/app/src/test/java/org/thoughtcrime/securesms/sms/UploadDependencyGraphTest.kt b/app/src/test/java/org/thoughtcrime/securesms/sms/UploadDependencyGraphTest.kt index 1ee01f035e..0433f10341 100644 --- a/app/src/test/java/org/thoughtcrime/securesms/sms/UploadDependencyGraphTest.kt +++ b/app/src/test/java/org/thoughtcrime/securesms/sms/UploadDependencyGraphTest.kt @@ -227,7 +227,6 @@ class UploadDependencyGraphTest { mmsId = AttachmentTable.PREUPLOAD_MESSAGE_ID, hasData = false, hasThumbnail = false, - hasArchiveThumbnail = false, contentType = attachment.contentType, transferProgress = AttachmentTable.TRANSFER_PROGRESS_PENDING, size = attachment.size, @@ -254,8 +253,6 @@ class UploadDependencyGraphTest { displayOrder = 0, uploadTimestamp = attachment.uploadTimestamp, dataHash = null, - archiveMediaId = null, - archiveMediaName = null, archiveCdn = 0, thumbnailRestoreState = AttachmentTable.ThumbnailRestoreState.NONE, archiveTransferState = AttachmentTable.ArchiveTransferState.NONE, diff --git a/app/src/testShared/org/thoughtcrime/securesms/database/FakeMessageRecords.kt b/app/src/testShared/org/thoughtcrime/securesms/database/FakeMessageRecords.kt index e4ac088a37..d73cde1a36 100644 --- a/app/src/testShared/org/thoughtcrime/securesms/database/FakeMessageRecords.kt +++ b/app/src/testShared/org/thoughtcrime/securesms/database/FakeMessageRecords.kt @@ -72,7 +72,6 @@ object FakeMessageRecords { mmsId = mmsId, hasData = hasData, hasThumbnail = hasThumbnail, - hasArchiveThumbnail = hasArchiveThumbnail, contentType = contentType, transferProgress = transferProgress, size = size, @@ -100,8 +99,6 @@ object FakeMessageRecords { uploadTimestamp = uploadTimestamp, dataHash = dataHash, archiveCdn = archiveCdn, - archiveMediaName = archiveMediaId, - archiveMediaId = archiveMediaName, thumbnailRestoreState = thumbnailRestoreState, archiveTransferState = archiveTransferState, uuid = null diff --git a/core-util/src/main/java/org/signal/core/util/SqlUtil.kt b/core-util/src/main/java/org/signal/core/util/SqlUtil.kt index 626bd14c02..a73d119ba6 100644 --- a/core-util/src/main/java/org/signal/core/util/SqlUtil.kt +++ b/core-util/src/main/java/org/signal/core/util/SqlUtil.kt @@ -14,7 +14,7 @@ object SqlUtil { private val TAG = Log.tag(SqlUtil::class.java) /** The maximum number of arguments (i.e. question marks) allowed in a SQL statement. */ - private const val MAX_QUERY_ARGS = 999 + const val MAX_QUERY_ARGS = 999 @JvmField val COUNT = arrayOf("COUNT(*)") diff --git a/libsignal-service/src/main/java/org/whispersystems/signalservice/api/NetworkResult.kt b/libsignal-service/src/main/java/org/whispersystems/signalservice/api/NetworkResult.kt index 5a59840068..a5cc062de8 100644 --- a/libsignal-service/src/main/java/org/whispersystems/signalservice/api/NetworkResult.kt +++ b/libsignal-service/src/main/java/org/whispersystems/signalservice/api/NetworkResult.kt @@ -21,6 +21,7 @@ import java.util.concurrent.TimeoutException import kotlin.reflect.KClass import kotlin.reflect.cast import kotlin.time.Duration +import kotlin.time.Duration.Companion.seconds typealias StatusCodeErrorAction = (StatusCodeError<*>) -> Unit @@ -215,6 +216,10 @@ sealed class NetworkResult( fun header(key: String): String? { return headers[key.lowercase()] } + + fun retryAfter(): Duration? { + return header("retry-after")?.toLongOrNull()?.seconds + } } /** Indicates that the application somehow failed in a way unrelated to network activity. Usually a runtime crash. */ diff --git a/libsignal-service/src/main/java/org/whispersystems/signalservice/api/archive/ArchiveApi.kt b/libsignal-service/src/main/java/org/whispersystems/signalservice/api/archive/ArchiveApi.kt index ab42396c34..6c55589d90 100644 --- a/libsignal-service/src/main/java/org/whispersystems/signalservice/api/archive/ArchiveApi.kt +++ b/libsignal-service/src/main/java/org/whispersystems/signalservice/api/archive/ArchiveApi.kt @@ -333,7 +333,7 @@ class ArchiveApi( * POST /v1/archives/media/delete * * - 400: Bad args or made on an authenticated channel - * - 401: Bad presentation, invalid public key signature, no matching backupId on teh server, or the credential was of the wrong type (messages/media) + * - 401: Bad presentation, invalid public key signature, no matching backupId on the server, or the credential was of the wrong type (messages/media) * - 403: Forbidden * - 429: Rate-limited */ diff --git a/libsignal-service/src/main/java/org/whispersystems/signalservice/api/backup/MediaName.kt b/libsignal-service/src/main/java/org/whispersystems/signalservice/api/backup/MediaName.kt index 3c79960613..479225db4f 100644 --- a/libsignal-service/src/main/java/org/whispersystems/signalservice/api/backup/MediaName.kt +++ b/libsignal-service/src/main/java/org/whispersystems/signalservice/api/backup/MediaName.kt @@ -17,6 +17,18 @@ value class MediaName(val name: String) { fun fromDigest(digest: ByteArray) = MediaName(Hex.toStringCondensed(digest)) fun fromDigestForThumbnail(digest: ByteArray) = MediaName("${Hex.toStringCondensed(digest)}_thumbnail") fun forThumbnailFromMediaName(mediaName: String) = MediaName("${mediaName}_thumbnail") + + /** + * For java, since it struggles with value classes. + */ + @JvmStatic + fun toMediaIdString(mediaName: String, mediaRootBackupKey: MediaRootBackupKey): String { + return MediaName(mediaName).toMediaId(mediaRootBackupKey).encode() + } + } + + fun toMediaId(mediaRootBackupKey: MediaRootBackupKey): MediaId { + return mediaRootBackupKey.deriveMediaId(this) } fun toByteArray(): ByteArray {