diff --git a/app/src/androidTest/java/org/thoughtcrime/securesms/database/AttachmentTableTest.kt b/app/src/androidTest/java/org/thoughtcrime/securesms/database/AttachmentTableTest.kt index 35272963db..63115ac0cf 100644 --- a/app/src/androidTest/java/org/thoughtcrime/securesms/database/AttachmentTableTest.kt +++ b/app/src/androidTest/java/org/thoughtcrime/securesms/database/AttachmentTableTest.kt @@ -9,6 +9,7 @@ import assertk.assertThat import assertk.assertions.hasSize import assertk.assertions.isEmpty import assertk.assertions.isEqualTo +import assertk.assertions.isFalse import assertk.assertions.isNotEmpty import assertk.assertions.isNotEqualTo import assertk.assertions.isNull @@ -27,6 +28,7 @@ import org.signal.core.util.Base64 import org.signal.core.util.Base64.decodeBase64OrThrow import org.signal.core.util.copyTo import org.signal.core.util.stream.NullOutputStream +import org.signal.core.util.update import org.signal.mediasend.SentMediaQuality import org.thoughtcrime.securesms.attachments.ArchivedAttachment import org.thoughtcrime.securesms.attachments.Attachment @@ -58,6 +60,10 @@ import kotlin.time.Duration.Companion.seconds @RunWith(AndroidJUnit4::class) class AttachmentTableTest { + companion object { + private const val TEST_PAGE_SIZE = 3 + } + @get:Rule val harness = SignalActivityRule(othersCount = 10) @@ -594,6 +600,210 @@ class AttachmentTableTest { assertThat(updatedCount).isEqualTo(0) } + /** + * Offloading deletes the only local copy, so it must be gated on evidence that came from the server. Being in the snapshot is our own bookkeeping. Only + * last_seen_on_remote_snapshot_version is the part the CDN told us, so only it can authorize the delete. + */ + @Test + fun givenAnOffloadCandidateConfirmedOnTheCdn_whenIOptimize_thenIExpectItOffloaded() { + val attachmentId = seedOffloadCandidate() + val snapshotVersion = commitSnapshotFor(attachmentId, markSeenOnRemote = true) + + SignalDatabase.attachments.markEligibleAttachmentsAsOptimized(lastCompletedCrawlVersion = snapshotVersion, minimumAge = 30.days, now = pastTheOffloadWindow()) + + assertThat(SignalDatabase.attachments.getAttachment(attachmentId)!!.transferState).isEqualTo(AttachmentTable.TRANSFER_RESTORE_OFFLOADED) + } + + @Test + fun givenAnOffloadCandidateNeverSeenOnTheCdn_whenIOptimize_thenIExpectItLeftAlone() { + val attachmentId = seedOffloadCandidate() + val snapshotVersion = commitSnapshotFor(attachmentId, markSeenOnRemote = false) + + SignalDatabase.attachments.markEligibleAttachmentsAsOptimized(lastCompletedCrawlVersion = snapshotVersion, minimumAge = 30.days, now = pastTheOffloadWindow()) + + assertThat(SignalDatabase.attachments.getAttachment(attachmentId)!!.transferState).isEqualTo(AttachmentTable.TRANSFER_PROGRESS_DONE) + } + + @Test + fun givenNoCompletedReconciliation_whenIOptimize_thenIExpectNothingOffloaded() { + val attachmentId = seedOffloadCandidate() + commitSnapshotFor(attachmentId, markSeenOnRemote = true) + + SignalDatabase.attachments.markEligibleAttachmentsAsOptimized(lastCompletedCrawlVersion = -1, minimumAge = 30.days, now = pastTheOffloadWindow()) + + assertThat(SignalDatabase.attachments.getAttachment(attachmentId)!!.transferState).isEqualTo(AttachmentTable.TRANSFER_PROGRESS_DONE) + } + + /** + * Coverage across pages: if only the first page were processed, the overflow would silently never be reclaimed. + */ + @Test + fun givenMoreConfirmedCandidatesThanOnePage_whenIOptimize_thenIExpectEveryPageOffloaded() { + val attachmentIds = List(TEST_PAGE_SIZE * 3) { seedOffloadCandidate() } + val snapshotVersion = commitSnapshotForAll(attachmentIds, markSeenOnRemote = true) + + SignalDatabase.attachments.markEligibleAttachmentsAsOptimized(lastCompletedCrawlVersion = snapshotVersion, minimumAge = 30.days, now = pastTheOffloadWindow(), pageSize = TEST_PAGE_SIZE) + + assertThat(attachmentIds.count { isOffloaded(it) }).isEqualTo(attachmentIds.size) + } + + /** + * Termination when nothing is offloadable. Unconfirmed candidates stay eligible forever, so a loop that ran until the candidate query drained would spin here + * and this test would hang rather than fail. Only the advancing id cursor ends it. + */ + @Test + fun givenMoreUnconfirmedCandidatesThanOnePage_whenIOptimize_thenIExpectItToTerminateAndOffloadNothing() { + val attachmentIds = List(TEST_PAGE_SIZE * 3) { seedOffloadCandidate() } + val snapshotVersion = commitSnapshotForAll(attachmentIds, markSeenOnRemote = false) + + SignalDatabase.attachments.markEligibleAttachmentsAsOptimized(lastCompletedCrawlVersion = snapshotVersion, minimumAge = 30.days, now = pastTheOffloadWindow(), pageSize = TEST_PAGE_SIZE) + + assertThat(attachmentIds.count { isOffloaded(it) }).isEqualTo(0) + } + + /** + * Stickers come back from their pack rather than the archive, so they are never offloaded no matter what the CDN confirms. + */ + @Test + fun givenAStickerOffloadCandidate_whenIOptimize_thenIExpectItLeftAlone() { + val attachmentId = seedOffloadCandidate() + val snapshotVersion = commitSnapshotFor(attachmentId, markSeenOnRemote = true) + markAsSticker(attachmentId) + + SignalDatabase.attachments.markEligibleAttachmentsAsOptimized(lastCompletedCrawlVersion = snapshotVersion, minimumAge = 30.days, now = pastTheOffloadWindow()) + + assertThat(isOffloaded(attachmentId)).isFalse() + } + + /** + * Offloading an image with no thumbnail would leave nothing at all to render in the conversation, so visual media has to keep a local thumbnail to qualify. + */ + @Test + fun givenAnImageOffloadCandidateWithNoThumbnail_whenIOptimize_thenIExpectItLeftAlone() { + val attachmentId = seedOffloadCandidate(contentType = MediaUtil.IMAGE_JPEG) + val snapshotVersion = commitSnapshotFor(attachmentId, markSeenOnRemote = true) + + SignalDatabase.attachments.markEligibleAttachmentsAsOptimized(lastCompletedCrawlVersion = snapshotVersion, minimumAge = 30.days, now = pastTheOffloadWindow()) + + assertThat(isOffloaded(attachmentId)).isFalse() + } + + /** + * Media the user just pulled back down would otherwise be offloaded again immediately, undoing the restore they asked for. + */ + @Test + fun givenARecentlyRestoredOffloadCandidate_whenIOptimize_thenIExpectItLeftAlone() { + val attachmentId = seedOffloadCandidate() + val snapshotVersion = commitSnapshotFor(attachmentId, markSeenOnRemote = true) + val evaluatedAt = pastTheOffloadWindow() + markRestoredAt(attachmentId, evaluatedAt) + + SignalDatabase.attachments.markEligibleAttachmentsAsOptimized(lastCompletedCrawlVersion = snapshotVersion, minimumAge = 30.days, now = evaluatedAt) + + assertThat(isOffloaded(attachmentId)).isFalse() + } + + @Test + fun givenAnOffloadCandidateOnATooRecentMessage_whenIOptimize_thenIExpectItLeftAlone() { + val attachmentId = seedOffloadCandidate() + val snapshotVersion = commitSnapshotFor(attachmentId, markSeenOnRemote = true) + + SignalDatabase.attachments.markEligibleAttachmentsAsOptimized(lastCompletedCrawlVersion = snapshotVersion, minimumAge = 30.days, now = System.currentTimeMillis()) + + assertThat(isOffloaded(attachmentId)).isFalse() + } + + /** Far enough ahead of the seeded message that both the offload age and the 7-day offload-restore window are satisfied. */ + private fun pastTheOffloadWindow(): Long = System.currentTimeMillis() + 60.days.inWholeMilliseconds + + /** A full locator, because a partial one makes reading the attachment back blow up on the sticker fields. */ + private fun markAsSticker(attachmentId: AttachmentId) { + SignalDatabase.rawDatabase + .update(AttachmentTable.TABLE_NAME) + .values( + AttachmentTable.STICKER_ID to 7, + AttachmentTable.STICKER_PACK_ID to "aa1111bbcc2222ddee3333ff44445555", + AttachmentTable.STICKER_PACK_KEY to "YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXoxMjM0NTY3ODkwYWI=", + AttachmentTable.STICKER_EMOJI to ":)" + ) + .where("${AttachmentTable.ID} = ?", attachmentId.id) + .run() + } + + private fun markRestoredAt(attachmentId: AttachmentId, restoredAt: Long) { + SignalDatabase.rawDatabase + .update(AttachmentTable.TABLE_NAME) + .values(AttachmentTable.OFFLOAD_RESTORED_AT to restoredAt) + .where("${AttachmentTable.ID} = ?", attachmentId.id) + .run() + } + + /** Defaults to a non-media content type, which keeps the candidate eligible without also having to generate a thumbnail file. */ + private fun seedOffloadCandidate(contentType: String = "application/pdf"): AttachmentId { + val data = byteArrayOf(1, 2, 3, 4, 5) + val attachment = createAttachmentPointer(Random.nextBytes(32), data.size, contentType = contentType) + val messageResult = SignalDatabase.messages.insertMessageInbox(createIncomingMessage(serverTime = System.currentTimeMillis().milliseconds, attachment = attachment)).get() + val attachmentId = messageResult.insertedAttachments!![attachment]!! + + SignalDatabase.attachments.setTransferState(messageResult.messageId, attachmentId, AttachmentTable.TRANSFER_PROGRESS_STARTED) + SignalDatabase.attachments.finalizeAttachmentAfterDownload(messageResult.messageId, attachmentId, ByteArrayInputStream(data)) + SignalDatabase.attachments.setArchiveTransferState(attachmentId, AttachmentTable.ArchiveTransferState.FINISHED) + + return attachmentId + } + + /** + * Commits every entry in a single snapshot version. Calling [commitSnapshotFor] in a loop would not work: each commit bumps the version, leaving all but the + * last entry below MAX_VERSION and therefore unconfirmable. + */ + private fun commitSnapshotForAll(attachmentIds: List, markSeenOnRemote: Boolean): Long { + val mediaIds: MutableList = mutableListOf() + val entries: MutableList = mutableListOf() + + for (attachmentId in attachmentIds) { + val attachment = SignalDatabase.attachments.getAttachment(attachmentId)!! + val plaintextHash = attachment.dataHash!!.decodeBase64OrThrow() + val remoteKey = attachment.remoteKey!!.decodeBase64OrThrow() + val mediaId = MediaName.fromPlaintextHashAndRemoteKey(plaintextHash, remoteKey).toMediaId(SignalStore.backup.mediaRootBackupKey).encode() + + mediaIds += mediaId + entries += BackupMediaSnapshotTable.MediaEntry(mediaId = mediaId, cdn = 3, plaintextHash = plaintextHash, remoteKey = remoteKey, isThumbnail = false) + } + + SignalDatabase.backupMediaSnapshots.writePendingMediaEntries(entries) + SignalDatabase.backupMediaSnapshots.commitPendingRows() + + val snapshotVersion = SignalDatabase.backupMediaSnapshots.getCurrentSnapshotVersion() + if (markSeenOnRemote) { + SignalDatabase.backupMediaSnapshots.markSeenOnRemote(mediaIds, snapshotVersion) + } + + return snapshotVersion + } + + private fun isOffloaded(attachmentId: AttachmentId): Boolean { + return SignalDatabase.attachments.getAttachment(attachmentId)!!.transferState == AttachmentTable.TRANSFER_RESTORE_OFFLOADED + } + + private fun commitSnapshotFor(attachmentId: AttachmentId, markSeenOnRemote: Boolean): Long { + val attachment = SignalDatabase.attachments.getAttachment(attachmentId)!! + val plaintextHash = attachment.dataHash!!.decodeBase64OrThrow() + val remoteKey = attachment.remoteKey!!.decodeBase64OrThrow() + val mediaId = MediaName.fromPlaintextHashAndRemoteKey(plaintextHash, remoteKey).toMediaId(SignalStore.backup.mediaRootBackupKey).encode() + + SignalDatabase.backupMediaSnapshots.writePendingMediaEntries( + listOf(BackupMediaSnapshotTable.MediaEntry(mediaId = mediaId, cdn = 3, plaintextHash = plaintextHash, remoteKey = remoteKey, isThumbnail = false)) + ) + SignalDatabase.backupMediaSnapshots.commitPendingRows() + + val snapshotVersion = SignalDatabase.backupMediaSnapshots.getCurrentSnapshotVersion() + if (markSeenOnRemote) { + SignalDatabase.backupMediaSnapshots.markSeenOnRemote(listOf(mediaId), snapshotVersion) + } + + return snapshotVersion + } + private fun createIncomingMessage( serverTime: Duration, attachment: Attachment, @@ -611,13 +821,13 @@ class AttachmentTableTest { ) } - private fun createAttachmentPointer(key: ByteArray, size: Int): Attachment { + private fun createAttachmentPointer(key: ByteArray, size: Int, contentType: String = MediaUtil.IMAGE_JPEG): Attachment { return PointerAttachment.forPointer( pointer = Optional.of( SignalServiceAttachmentPointer( cdnNumber = 3, remoteId = SignalServiceAttachmentRemoteId.V4("asdf"), - contentType = MediaUtil.IMAGE_JPEG, + contentType = contentType, key = key, size = Optional.of(size), preview = Optional.empty(), diff --git a/app/src/androidTest/java/org/thoughtcrime/securesms/jobs/ArchiveAttachmentReconciliationJobTest.kt b/app/src/androidTest/java/org/thoughtcrime/securesms/jobs/ArchiveAttachmentReconciliationJobTest.kt index 8080f6b85f..e3e2207819 100644 --- a/app/src/androidTest/java/org/thoughtcrime/securesms/jobs/ArchiveAttachmentReconciliationJobTest.kt +++ b/app/src/androidTest/java/org/thoughtcrime/securesms/jobs/ArchiveAttachmentReconciliationJobTest.kt @@ -6,16 +6,22 @@ package org.thoughtcrime.securesms.jobs import androidx.test.ext.junit.runners.AndroidJUnit4 +import arrow.core.left import arrow.core.right import assertk.assertThat +import assertk.assertions.contains +import assertk.assertions.doesNotContain import assertk.assertions.isEqualTo import assertk.assertions.isFalse +import assertk.assertions.isGreaterThan import assertk.assertions.isNull import io.mockk.Runs import io.mockk.coEvery +import io.mockk.coVerify import io.mockk.every import io.mockk.just import io.mockk.mockkObject +import io.mockk.slot import io.mockk.unmockkAll import io.mockk.verify import org.junit.After @@ -27,10 +33,12 @@ import org.signal.core.models.backup.MediaName import org.signal.core.models.database.AttachmentId import org.signal.core.util.Base64.decodeBase64OrThrow import org.signal.network.api.ArchiveApiV2 +import org.signal.network.service.ArchiveError import org.signal.network.service.ArchiveService +import org.thoughtcrime.securesms.attachments.ArchivedAttachment import org.thoughtcrime.securesms.attachments.Attachment import org.thoughtcrime.securesms.attachments.PointerAttachment -import org.thoughtcrime.securesms.backup.v2.BackupRepository +import org.thoughtcrime.securesms.backup.v2.ArchivedMediaObject import org.thoughtcrime.securesms.backup.v2.MessageBackupTier import org.thoughtcrime.securesms.database.AttachmentTable import org.thoughtcrime.securesms.database.BackupMediaSnapshotTable.MediaEntry @@ -44,7 +52,10 @@ import org.thoughtcrime.securesms.util.MediaUtil import org.whispersystems.signalservice.api.messages.SignalServiceAttachmentPointer import org.whispersystems.signalservice.api.messages.SignalServiceAttachmentRemoteId import java.io.ByteArrayInputStream +import java.io.IOException import java.util.Optional +import java.util.UUID +import kotlin.random.Random import kotlin.time.Duration import kotlin.time.Duration.Companion.days @@ -56,16 +67,18 @@ class ArchiveAttachmentReconciliationJobTest { private val archiveService: ArchiveService = AppDependencies.archiveService + private val deletedFromCdn = slot>() + @Before fun setUp() { SignalStore.backup.backupTier = MessageBackupTier.PAID SignalStore.backup.hasBackupBeenUploaded = true SignalStore.backup.lastAttachmentReconciliationTime = System.currentTimeMillis() SignalStore.backup.localRestoreReconcilePending = false + SignalStore.backup.lastUsedMessageCutoffTime = 0 - mockkObject(BackupRepository) mockkObject(ArchiveCommitAttachmentDeletesJob) - coEvery { ArchiveCommitAttachmentDeletesJob.deleteMediaObjectsFromCdn(any(), any(), any(), any()) } returns null + coEvery { ArchiveCommitAttachmentDeletesJob.deleteMediaObjectsFromCdn(any(), capture(deletedFromCdn), any(), any()) } returns null } @After @@ -174,6 +187,36 @@ class ArchiveAttachmentReconciliationJobTest { assertThat(SignalDatabase.attachments.getAttachment(attachmentId)!!.archiveTransferState).isEqualTo(AttachmentTable.ArchiveTransferState.FINISHED) } + /** + * The over-size-limit cutoff leaves old messages out of the export, so nothing in the backup references their media any more. Reconciliation applies the same + * rule as [ArchiveCommitAttachmentDeletesJob] so the two reclaim paths can't disagree about what still counts as referenced. + */ + @Test + fun givenCdnMediaWhoseMessagePredatesTheMessageCutoff_whenIReconcile_thenIExpectItDeletedFromTheCdn() { + val attachmentId = seedFinalizedAttachment("remote-key-cutoff".toByteArray(), byteArrayOf(21, 22, 23, 24, 25), receivedAt = 10.days) + SignalStore.backup.lastUsedMessageCutoffTime = 30.days.inWholeMilliseconds + fakeCdnContains(attachmentId, cdn = 3) + + ArchiveAttachmentReconciliationJob(forced = true).run() + + assertThat(deletedFromCdn.captured.map { it.mediaId }).contains(mediaIdFor(attachmentId)) + } + + /** + * A cutoff being set at all must not weaken the protection for media the backup still references, which is what keeps a lost snapshot table from wiping the + * archive. + */ + @Test + fun givenCdnMediaWhoseMessageIsWithinTheMessageCutoff_whenIReconcile_thenIExpectItProtected() { + val attachmentId = seedFinalizedAttachment("remote-key-recent".toByteArray(), byteArrayOf(26, 27, 28, 29, 30), receivedAt = 40.days) + SignalStore.backup.lastUsedMessageCutoffTime = 30.days.inWholeMilliseconds + fakeCdnContains(attachmentId, cdn = 3) + + ArchiveAttachmentReconciliationJob(forced = true).run() + + coVerify(exactly = 0) { ArchiveCommitAttachmentDeletesJob.deleteMediaObjectsFromCdn(any(), any(), any(), any()) } + } + @Test fun givenFirstEverReconciliation_whenIForceIt_thenItStillRunsAndRepairs() { SignalStore.backup.lastAttachmentReconciliationTime = -1 @@ -198,9 +241,254 @@ class ArchiveAttachmentReconciliationJobTest { assertThat(SignalStore.backup.localRestoreReconcilePending).isFalse() } - private fun seedFinalizedAttachment(remoteKey: ByteArray, data: ByteArray): AttachmentId { + /** + * Media that was offloaded (or restored from a backup) has no local data file, so resetting it to NONE cannot lead to a re-upload. It only makes the export + * emit a tombstone, which drops the locator and marks the media for deletion off of the CDN. It must stay FINISHED. + */ + @Test + fun givenFinishedMediaMissingFromCdnWithNoLocalDataFile_whenIReconcile_thenItStaysFinishedAndKeepsItsCdn() { + val attachmentId = seedArchivedAttachment() + commitSnapshotFor(attachmentId, cdn = 3) + fakeCdnEmpty() + + ArchiveAttachmentReconciliationJob(forced = true).run() + + val after = SignalDatabase.attachments.getAttachment(attachmentId)!! + assertThat(after.archiveTransferState).isEqualTo(AttachmentTable.ArchiveTransferState.FINISHED) + assertThat(after.archiveCdn).isEqualTo(3) + } + + @Test + fun givenACrawlThatCompletes_whenIReconcile_thenIExpectTheCompletedSnapshotVersionRecorded() { + SignalStore.backup.lastCompletedReconciliationSnapshotVersion = -1 + + val attachmentId = seedFinalizedAttachment("remote-key-recorded".toByteArray(), byteArrayOf(1, 2, 3, 4, 5)) + commitSnapshotFor(attachmentId, cdn = 3) + fakeCdnContains(attachmentId, cdn = 3) + + ArchiveAttachmentReconciliationJob(forced = true).run() + + assertThat(SignalStore.backup.lastCompletedReconciliationSnapshotVersion).isEqualTo(SignalDatabase.backupMediaSnapshots.getCurrentSnapshotVersion()) + } + + /** + * Offloading trusts the recorded version as proof the server confirmed our media, so a crawl that dies part way through must leave it untouched rather than + * recording a version it never finished verifying. + */ + @Test + fun givenACrawlThatFailsPartWayThrough_whenIReconcile_thenIExpectNoCompletedSnapshotVersionRecorded() { + SignalStore.backup.lastCompletedReconciliationSnapshotVersion = -1 + + val attachmentId = seedFinalizedAttachment("remote-key-failed".toByteArray(), byteArrayOf(2, 3, 4, 5, 6)) + commitSnapshotFor(attachmentId, cdn = 3) + coEvery { archiveService.listRemoteMediaObjects(any(), any()) } returns ArchiveError.NetworkError(IOException("boom")).left() + + val result = ArchiveAttachmentReconciliationJob(forced = true).run() + + assertThat(result.isSuccess).isFalse() + assertThat(SignalStore.backup.lastCompletedReconciliationSnapshotVersion).isEqualTo(-1L) + } + + /** + * A completed crawl proves the object isn't on the CDN, so the row is bookkeeping for something that no longer exists and there is nothing left to orphan. + */ + @Test + fun givenOldSnapshotMediaAbsentFromCdn_whenIReconcile_thenIExpectTheRowPruned() { + val absentMediaId = commitRandomSnapshotEntry() + commitRandomSnapshotEntry() + fakeCdnEmpty() + + ArchiveAttachmentReconciliationJob(forced = true).run() + + assertThat(oldSnapshotMediaIds()).doesNotContain(absentMediaId) + } + + /** + * A crawl confirmed this object once, so a later crawl failing to list it is not enough to conclude it's gone. An upload landing mid-crawl produces the same + * evidence. Pruning here would contradict our own earlier proof and orphan the object. + */ + @Test + fun givenOldSnapshotMediaConfirmedByAnEarlierCrawl_whenIReconcile_thenIExpectTheRowKept() { + val previouslySeenMediaId = commitRandomSnapshotEntry() + SignalDatabase.backupMediaSnapshots.markSeenOnRemote(listOf(previouslySeenMediaId), 1) + commitRandomSnapshotEntry() + fakeCdnEmpty() + + ArchiveAttachmentReconciliationJob(forced = true).run() + + assertThat(oldSnapshotMediaIds()).contains(previouslySeenMediaId) + } + + /** The crawl found the object, so this row is the only thing tracking it and must survive even though it left the latest snapshot. */ + @Test + fun givenOldSnapshotMediaStillOnCdn_whenIReconcile_thenIExpectTheRowKept() { + val attachmentId = seedFinalizedAttachment("remote-key-kept".toByteArray(), byteArrayOf(3, 1, 4, 1, 5)) + commitSnapshotFor(attachmentId, cdn = 3) + commitRandomSnapshotEntry() + fakeCdnContains(attachmentId, cdn = 3) + + ArchiveAttachmentReconciliationJob(forced = true).run() + + assertThat(oldSnapshotMediaIds()).contains(mediaIdFor(attachmentId)) + } + + /** + * The forced-crawl rate limit must only be consumed by a crawl that actually starts, otherwise a job that returns early leaves offloading blocked for a full + * interval without having verified anything. + */ + @Test + fun givenNoBackupUploaded_whenIForceAReconciliation_thenIExpectTheAttemptTimeUnchanged() { + SignalStore.backup.hasBackupBeenUploaded = false + SignalStore.backup.lastForcedReconciliationAttemptTime = 0 + fakeCdnEmpty() + + ArchiveAttachmentReconciliationJob(forced = true).run() + + assertThat(SignalStore.backup.lastForcedReconciliationAttemptTime).isEqualTo(0L) + } + + @Test + fun givenACrawlThatStarts_whenIForceAReconciliation_thenIExpectTheAttemptTimeAdvanced() { + SignalStore.backup.lastForcedReconciliationAttemptTime = 0 + commitRandomSnapshotEntry() + fakeCdnEmpty() + + ArchiveAttachmentReconciliationJob(forced = true).run() + + assertThat(SignalStore.backup.lastForcedReconciliationAttemptTime).isGreaterThan(0L) + } + + /** + * A crawl with no snapshot to compare against can't confirm anything, so it must not consume the forced-attempt budget. Spending it here would rate-limit the + * retry that becomes useful once a backup has built a snapshot. + */ + @Test + fun givenNoSnapshotYet_whenIForceAReconciliation_thenIExpectTheAttemptTimeUnchanged() { + SignalStore.backup.lastForcedReconciliationAttemptTime = 0 + fakeCdnEmpty() + + ArchiveAttachmentReconciliationJob(forced = true).run() + + assertThat(SignalStore.backup.lastForcedReconciliationAttemptTime).isEqualTo(0L) + } + + /** + * Same reasoning for the offload gate: a snapshot-less crawl verified nothing, so it must not record a completed version that would let us start deleting + * local copies of media. + */ + @Test + fun givenNoSnapshotYet_whenIReconcile_thenIExpectNoCompletedSnapshotVersionRecorded() { + SignalStore.backup.lastCompletedReconciliationSnapshotVersion = -1 + fakeCdnEmpty() + + ArchiveAttachmentReconciliationJob(forced = true).run() + + assertThat(SignalStore.backup.lastCompletedReconciliationSnapshotVersion).isEqualTo(-1L) + } + + // TODO [cody] return after fixing perf problem of restoring thumbnail state + // Commented out alongside the skipped thumbnail promotion. See the TODO in ArchiveAttachmentReconciliationJob.syncCdnPage. + // + // /** + // * A restore sets the full-size state from the backup's CDN claim but leaves the thumbnail state NONE, so the listing is the only thing that can promote it. + // * Until it does, the thumbnail is dropped from the snapshot on every backup and gets needlessly re-uploaded once the media is downloaded. + // */ + // @Test + // fun givenRestoredMediaWhoseThumbnailIsOnTheCdn_whenIReconcile_thenIExpectTheThumbnailMarkedFinished() { + // val attachmentId = seedArchivedAttachment() + // assertThat(SignalDatabase.attachments.getArchiveThumbnailTransferState(attachmentId)).isEqualTo(AttachmentTable.ArchiveTransferState.NONE) + // + // fakeCdnContainsThumbnail(attachmentId, cdn = 3) + // + // ArchiveAttachmentReconciliationJob(forced = true).run() + // + // assertThat(SignalDatabase.attachments.getArchiveThumbnailTransferState(attachmentId)).isEqualTo(AttachmentTable.ArchiveTransferState.FINISHED) + // } + // + // @Test + // fun givenRestoredMediaWhoseThumbnailIsNotOnTheCdn_whenIReconcile_thenIExpectTheThumbnailLeftAlone() { + // val attachmentId = seedArchivedAttachment() + // fakeCdnEmpty() + // + // ArchiveAttachmentReconciliationJob(forced = true).run() + // + // assertThat(SignalDatabase.attachments.getArchiveThumbnailTransferState(attachmentId)).isEqualTo(AttachmentTable.ArchiveTransferState.NONE) + // } + + // private fun fakeCdnContainsThumbnail(attachmentId: AttachmentId, cdn: Int) { + // val attachment = SignalDatabase.attachments.getAttachment(attachmentId)!! + // val plaintextHash = attachment.dataHash!!.decodeBase64OrThrow() + // val remoteKey = attachment.remoteKey!!.decodeBase64OrThrow() + // val mediaId = MediaName.fromPlaintextHashAndRemoteKeyForThumbnail(plaintextHash, remoteKey).toMediaId(SignalStore.backup.mediaRootBackupKey).encode() + // + // coEvery { archiveService.listRemoteMediaObjects(any(), any()) } returns ArchiveApiV2.MediaItemsPage( + // storedMediaObjects = listOf(ArchiveApiV2.StoredMediaObject(cdn = cdn, mediaId = mediaId, objectLength = attachment.size)), + // cursor = null + // ).right() + // } + + private fun oldSnapshotMediaIds(): List { + return SignalDatabase.backupMediaSnapshots.getPageOfOldMediaEntries(pageSize = 1_000).map { it.mediaId } + } + + private fun mediaIdFor(attachmentId: AttachmentId): String { + val attachment = SignalDatabase.attachments.getAttachment(attachmentId)!! + val plaintextHash = attachment.dataHash!!.decodeBase64OrThrow() + val remoteKey = attachment.remoteKey!!.decodeBase64OrThrow() + + return MediaName.fromPlaintextHashAndRemoteKey(plaintextHash, remoteKey).toMediaId(SignalStore.backup.mediaRootBackupKey).encode() + } + + /** + * Committing a second entry is what pushes any previously committed row below MAX(snapshot_version), which is the "fell out of the latest snapshot" state. + */ + private fun commitRandomSnapshotEntry(): String { + val plaintextHash = Random.nextBytes(32) + val remoteKey = Random.nextBytes(32) + val mediaId = MediaName.fromPlaintextHashAndRemoteKey(plaintextHash, remoteKey).toMediaId(SignalStore.backup.mediaRootBackupKey).encode() + + SignalDatabase.backupMediaSnapshots.writePendingMediaEntries( + listOf(MediaEntry(mediaId = mediaId, cdn = 3, plaintextHash = plaintextHash, remoteKey = remoteKey, isThumbnail = false)) + ) + SignalDatabase.backupMediaSnapshots.commitPendingRows() + + return mediaId + } + + private fun seedArchivedAttachment(): AttachmentId { + val attachment = ArchivedAttachment( + contentType = MediaUtil.IMAGE_JPEG, + size = 1024, + cdn = 3, + uploadTimestamp = 0, + key = Random.nextBytes(8), + cdnKey = "password", + archiveCdn = 3, + plaintextHash = Random.nextBytes(8), + incrementalMac = Random.nextBytes(8), + incrementalMacChunkSize = 8, + width = 100, + height = 100, + caption = null, + blurHash = null, + voiceNote = false, + borderless = false, + stickerLocator = null, + gif = false, + quote = false, + quoteTargetContentType = null, + uuid = UUID.randomUUID(), + fileName = null, + localBackupKey = null + ) + + val messageId = SignalDatabase.messages.insertMessageInbox(createIncomingMessage(serverTime = 0.days, attachment = attachment)).get().messageId + return SignalDatabase.attachments.getAttachmentsForMessage(messageId).first().attachmentId + } + + private fun seedFinalizedAttachment(remoteKey: ByteArray, data: ByteArray, receivedAt: Duration = 0.days): AttachmentId { val attachment = createAttachmentPointer(remoteKey, data.size) - val messageResult = SignalDatabase.messages.insertMessageInbox(createIncomingMessage(serverTime = 0.days, attachment = attachment)).get() + val messageResult = SignalDatabase.messages.insertMessageInbox(createIncomingMessage(serverTime = receivedAt, attachment = attachment)).get() val attachmentId = messageResult.insertedAttachments!![attachment]!! SignalDatabase.attachments.setTransferState(messageResult.messageId, attachmentId, AttachmentTable.TRANSFER_PROGRESS_STARTED) SignalDatabase.attachments.finalizeAttachmentAfterDownload(messageResult.messageId, attachmentId, ByteArrayInputStream(data)) diff --git a/app/src/androidTest/java/org/thoughtcrime/securesms/jobs/ArchiveCommitAttachmentDeletesJobTest.kt b/app/src/androidTest/java/org/thoughtcrime/securesms/jobs/ArchiveCommitAttachmentDeletesJobTest.kt new file mode 100644 index 0000000000..91d6d31896 --- /dev/null +++ b/app/src/androidTest/java/org/thoughtcrime/securesms/jobs/ArchiveCommitAttachmentDeletesJobTest.kt @@ -0,0 +1,411 @@ +/* + * Copyright 2026 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ + +package org.thoughtcrime.securesms.jobs + +import androidx.test.ext.junit.runners.AndroidJUnit4 +import assertk.assertThat +import assertk.assertions.contains +import assertk.assertions.doesNotContain +import assertk.assertions.hasSize +import assertk.assertions.isEmpty +import assertk.assertions.isEqualTo +import io.mockk.coEvery +import io.mockk.coVerify +import io.mockk.mockkObject +import io.mockk.slot +import io.mockk.unmockkAll +import org.junit.After +import org.junit.Before +import org.junit.Rule +import org.junit.Test +import org.junit.runner.RunWith +import org.signal.core.models.backup.MediaName +import org.signal.core.models.database.AttachmentId +import org.signal.core.util.Base64.decodeBase64OrThrow +import org.thoughtcrime.securesms.attachments.Attachment +import org.thoughtcrime.securesms.attachments.PointerAttachment +import org.thoughtcrime.securesms.backup.v2.ArchivedMediaObject +import org.thoughtcrime.securesms.backup.v2.MessageBackupTier +import org.thoughtcrime.securesms.database.AttachmentTable +import org.thoughtcrime.securesms.database.BackupMediaSnapshotTable.MediaEntry +import org.thoughtcrime.securesms.database.MessageType +import org.thoughtcrime.securesms.database.SignalDatabase +import org.thoughtcrime.securesms.keyvalue.SignalStore +import org.thoughtcrime.securesms.mms.IncomingMessage +import org.thoughtcrime.securesms.testing.SignalActivityRule +import org.thoughtcrime.securesms.util.MediaUtil +import org.whispersystems.signalservice.api.messages.SignalServiceAttachmentPointer +import org.whispersystems.signalservice.api.messages.SignalServiceAttachmentRemoteId +import java.io.ByteArrayInputStream +import java.util.Optional +import kotlin.random.Random +import kotlin.time.Duration +import kotlin.time.Duration.Companion.days + +@RunWith(AndroidJUnit4::class) +class ArchiveCommitAttachmentDeletesJobTest { + + @get:Rule + val harness = SignalActivityRule() + + private val deletedFromCdn = slot>() + + private val jobPageSize = ArchiveCommitAttachmentDeletesJob.REMOTE_DELETE_BATCH_SIZE + + @Before + fun setUp() { + SignalStore.backup.backupTier = MessageBackupTier.PAID + SignalStore.backup.lastUsedMessageCutoffTime = 0 + + mockkObject(ArchiveCommitAttachmentDeletesJob) + coEvery { ArchiveCommitAttachmentDeletesJob.deleteMediaObjectsFromCdn(any(), capture(deletedFromCdn), any(), any()) } returns null + } + + @After + fun tearDown() { + unmockkAll() + } + + /** + * Media only leaves the snapshot because the backup stopped referencing it, which a bookkeeping bug can cause while the attachment is still very much present. + * Deleting on that basis alone is how the only remaining copy of media gets destroyed, so the pass has to discriminate within a page rather than trusting the + * snapshot. + */ + @Test + fun givenOldSnapshotMediaBothReferencedAndOrphaned_whenIRun_thenIExpectOnlyTheOrphanDeletedFromTheCdn() { + val attachmentId = seedFinalizedAttachment(byteArrayOf(1, 2, 3, 4, 5)) + val referenced = entryFor(attachmentId) + val orphan = randomEntry() + + commit(referenced, orphan) + commit(randomEntry()) + + ArchiveCommitAttachmentDeletesJob().run() + + val deletedMediaIds = deletedFromCdn.captured.map { it.mediaId } + assertThat(deletedMediaIds).contains(orphan.mediaId) + assertThat(deletedMediaIds).doesNotContain(referenced.mediaId) + } + + @Test + fun givenOnlyStillReferencedOldSnapshotMedia_whenIRun_thenIExpectNothingDeletedFromTheCdn() { + val attachmentId = seedFinalizedAttachment(byteArrayOf(9, 8, 7, 6, 5)) + + commit(entryFor(attachmentId)) + commit(randomEntry()) + + ArchiveCommitAttachmentDeletesJob().run() + + coVerify(exactly = 0) { ArchiveCommitAttachmentDeletesJob.deleteMediaObjectsFromCdn(any(), any(), any(), any()) } + } + + /** + * A retained row is the only record that its CDN object exists, so pruning it would orphan the object permanently. The run still has to terminate, which it + * does by paging past retained rows by id. + */ + @Test + fun givenOnlyStillReferencedOldSnapshotMedia_whenIRun_thenIExpectTheSnapshotRowsRetained() { + val attachmentId = seedFinalizedAttachment(byteArrayOf(4, 4, 4, 4, 4)) + val referenced = entryFor(attachmentId) + + commit(referenced) + commit(randomEntry()) + + ArchiveCommitAttachmentDeletesJob().run() + + assertThat(remainingOldMediaIds()).contains(referenced.mediaId) + } + + @Test + fun givenOldSnapshotMediaWithNoAttachment_whenIRun_thenIExpectTheSnapshotRowPruned() { + val orphan = randomEntry() + + commit(orphan) + commit(randomEntry()) + + ArchiveCommitAttachmentDeletesJob().run() + + assertThat(remainingOldMediaIds()).doesNotContain(orphan.mediaId) + } + + /** + * Coverage across pages: if the loop only ever handled the first page, the overflow rows would stay on the CDN forever with nothing left to reconsider them. + */ + @Test + fun givenMoreOrphansThanOnePage_whenIRun_thenIExpectEveryPageProcessed() { + val orphans = List(jobPageSize + 1) { randomEntry() } + + commit(orphans) + commit(randomEntry()) + + ArchiveCommitAttachmentDeletesJob().run() + + assertThat(remainingOldMediaIds()).isEmpty() + } + + /** + * Termination when a whole page is retained. Nothing is removed from the table here, so the loop can only end by advancing its cursor past rows it declined to + * delete. If it paged from the start each time it would spin forever and this test would hang rather than fail. + */ + @Test + fun givenMoreUndeletableRowsThanOnePage_whenIRun_thenIExpectItToTerminateAndRetainThem() { + val unknownCdn = List(jobPageSize + 1) { randomEntry(cdn = null) } + + commit(unknownCdn) + commit(randomEntry()) + + ArchiveCommitAttachmentDeletesJob().run() + + assertThat(remainingOldMediaIds()).hasSize(unknownCdn.size) + coVerify(exactly = 0) { ArchiveCommitAttachmentDeletesJob.deleteMediaObjectsFromCdn(any(), any(), any(), any()) } + } + + /** + * Users over the backup size limit get old messages left out of the export entirely, so nothing in the backup references their media any more and it should + * stop occupying paid archive quota. Before this, an attachment row existing was enough to retain it forever. + */ + @Test + fun givenMediaWhoseMessagePredatesTheMessageCutoff_whenIRun_thenIExpectItDeletedFromTheCdn() { + val attachmentId = seedFinalizedAttachment(byteArrayOf(1, 1, 2, 3, 5), receivedAt = 10.days) + val agedOut = entryFor(attachmentId) + SignalStore.backup.lastUsedMessageCutoffTime = 30.days.inWholeMilliseconds + + commit(agedOut) + commit(randomEntry()) + + ArchiveCommitAttachmentDeletesJob().run() + + assertThat(deletedFromCdn.captured.map { it.mediaId }).contains(agedOut.mediaId) + } + + /** + * The exporter includes messages on `date_received >= cutoff`, so one landing exactly on the threshold is still in the backup. Deleting it would take media + * the backup still references. + */ + @Test + fun givenMediaWhoseMessageIsExactlyAtTheMessageCutoff_whenIRun_thenIExpectItRetained() { + val attachmentId = seedFinalizedAttachment(byteArrayOf(2, 2, 2, 2, 2), receivedAt = 30.days) + val boundary = entryFor(attachmentId) + SignalStore.backup.lastUsedMessageCutoffTime = 30.days.inWholeMilliseconds + + commit(boundary) + commit(randomEntry()) + + ArchiveCommitAttachmentDeletesJob().run() + + assertThat(remainingOldMediaIds()).contains(boundary.mediaId) + coVerify(exactly = 0) { ArchiveCommitAttachmentDeletesJob.deleteMediaObjectsFromCdn(any(), any(), any(), any()) } + } + + /** + * A wallpaper has no message row at all. Phrasing the cutoff as "its message is recent enough" would drop wallpapers out of the referent set and delete media + * the user still has applied, so the check only disqualifies an attachment whose message exists and is too old. + */ + @Test + fun givenAWallpaperAndAMessageCutoff_whenIRun_thenIExpectItRetained() { + val attachmentId = SignalDatabase.attachments.insertWallpaper(ByteArrayInputStream(byteArrayOf(7, 7, 7, 7, 7))) + val wallpaper = entryFor(attachmentId) + SignalStore.backup.lastUsedMessageCutoffTime = 30.days.inWholeMilliseconds + + commit(wallpaper) + commit(randomEntry()) + + ArchiveCommitAttachmentDeletesJob().run() + + assertThat(remainingOldMediaIds()).contains(wallpaper.mediaId) + coVerify(exactly = 0) { ArchiveCommitAttachmentDeletesJob.deleteMediaObjectsFromCdn(any(), any(), any(), any()) } + } + + /** + * Cutting off messages only happens for backups over the size limit, which is vanishingly rare, so the ordinary case has to behave exactly as it did before. + */ + @Test + fun givenNoMessageCutoff_whenIRun_thenIExpectAncientMediaRetained() { + val attachmentId = seedFinalizedAttachment(byteArrayOf(3, 3, 3, 3, 3), receivedAt = 0.days) + val ancient = entryFor(attachmentId) + + commit(ancient) + commit(randomEntry()) + + ArchiveCommitAttachmentDeletesJob().run() + + assertThat(remainingOldMediaIds()).contains(ancient.mediaId) + coVerify(exactly = 0) { ArchiveCommitAttachmentDeletesJob.deleteMediaObjectsFromCdn(any(), any(), any(), any()) } + } + + /** + * Media is keyed by hash and remote key, so the same bytes can hang off several messages. One of them surviving the cutoff means the backup still references + * the media, and it has to be judged referenced rather than per-row. + */ + @Test + fun givenMediaReferencedByBothAnAgedOutAndACurrentMessage_whenIRun_thenIExpectItRetained() { + val data = byteArrayOf(4, 5, 6, 7, 8) + val agedOutId = seedFinalizedAttachment(data, receivedAt = 10.days) + val currentId = seedFinalizedAttachment(data, receivedAt = 40.days) + + val shared = entryFor(agedOutId) + assertThat(entryFor(currentId).mediaId).isEqualTo(shared.mediaId) + + SignalStore.backup.lastUsedMessageCutoffTime = 30.days.inWholeMilliseconds + + commit(shared) + commit(randomEntry()) + + ArchiveCommitAttachmentDeletesJob().run() + + assertThat(remainingOldMediaIds()).contains(shared.mediaId) + coVerify(exactly = 0) { ArchiveCommitAttachmentDeletesJob.deleteMediaObjectsFromCdn(any(), any(), any(), any()) } + } + + /** + * Full-size and thumbnail media share a hash and key but are judged by different rules, so a thumbnail can be reclaimable while its full-size copy is not. + */ + @Test + fun givenAThumbnailWhoseAttachmentStillWantsIt_whenIRun_thenIExpectItRetained() { + val attachmentId = seedFinalizedAttachment(byteArrayOf(5, 1, 5, 1, 5)) + val thumbnail = entryFor(attachmentId, isThumbnail = true) + + commit(thumbnail) + commit(randomEntry()) + + ArchiveCommitAttachmentDeletesJob().run() + + assertThat(remainingOldMediaIds()).contains(thumbnail.mediaId) + coVerify(exactly = 0) { ArchiveCommitAttachmentDeletesJob.deleteMediaObjectsFromCdn(any(), any(), any(), any()) } + } + + /** + * Wallpapers never get a thumbnail written into the snapshot, so one already on the CDN is unreachable bookkeeping. Its full-size object still has a referent + * and has to survive, which is what makes this a thumbnail-only reclaim rather than a blanket delete. + */ + @Test + fun givenAWallpaperThumbnail_whenIRun_thenIExpectOnlyTheThumbnailDeletedFromTheCdn() { + val attachmentId = SignalDatabase.attachments.insertWallpaper(ByteArrayInputStream(byteArrayOf(6, 2, 6, 2, 6))) + val thumbnail = entryFor(attachmentId, isThumbnail = true) + val fullSize = entryFor(attachmentId) + + commit(thumbnail, fullSize) + commit(randomEntry()) + + ArchiveCommitAttachmentDeletesJob().run() + + val deletedMediaIds = deletedFromCdn.captured.map { it.mediaId } + assertThat(deletedMediaIds).contains(thumbnail.mediaId) + assertThat(deletedMediaIds).doesNotContain(fullSize.mediaId) + } + + /** + * The message cutoff has to reach the thumbnail path too, otherwise media dropped from the backup keeps half of its CDN footprint. + */ + @Test + fun givenAThumbnailWhoseMessagePredatesTheMessageCutoff_whenIRun_thenIExpectItDeletedFromTheCdn() { + val attachmentId = seedFinalizedAttachment(byteArrayOf(7, 3, 7, 3, 7), receivedAt = 10.days) + val thumbnail = entryFor(attachmentId, isThumbnail = true) + SignalStore.backup.lastUsedMessageCutoffTime = 30.days.inWholeMilliseconds + + commit(thumbnail) + commit(randomEntry()) + + ArchiveCommitAttachmentDeletesJob().run() + + assertThat(deletedFromCdn.captured.map { it.mediaId }).contains(thumbnail.mediaId) + } + + private fun entryFor(attachmentId: AttachmentId, isThumbnail: Boolean = false): MediaEntry { + val attachment = SignalDatabase.attachments.getAttachment(attachmentId)!! + val plaintextHash = attachment.dataHash!!.decodeBase64OrThrow() + val remoteKey = attachment.remoteKey!!.decodeBase64OrThrow() + + val mediaName = if (isThumbnail) { + MediaName.fromPlaintextHashAndRemoteKeyForThumbnail(plaintextHash, remoteKey) + } else { + MediaName.fromPlaintextHashAndRemoteKey(plaintextHash, remoteKey) + } + + return MediaEntry( + mediaId = mediaName.toMediaId(SignalStore.backup.mediaRootBackupKey).encode(), + cdn = 3, + plaintextHash = plaintextHash, + remoteKey = remoteKey, + isThumbnail = isThumbnail + ) + } + + private fun randomEntry(cdn: Int? = 3): MediaEntry { + val plaintextHash = Random.nextBytes(32) + val remoteKey = Random.nextBytes(32) + + return MediaEntry( + mediaId = MediaName.fromPlaintextHashAndRemoteKey(plaintextHash, remoteKey).toMediaId(SignalStore.backup.mediaRootBackupKey).encode(), + cdn = cdn, + plaintextHash = plaintextHash, + remoteKey = remoteKey, + isThumbnail = false + ) + } + + private fun commit(vararg entries: MediaEntry) { + commit(entries.toList()) + } + + private fun commit(entries: List) { + SignalDatabase.backupMediaSnapshots.writePendingMediaEntries(entries) + SignalDatabase.backupMediaSnapshots.commitPendingRows() + } + + private fun remainingOldMediaIds(): List { + return SignalDatabase.backupMediaSnapshots.getPageOfOldMediaEntries(pageSize = (jobPageSize * 3)).map { it.mediaId } + } + + private fun seedFinalizedAttachment(data: ByteArray, receivedAt: Duration = 0.days): AttachmentId { + val attachment = createAttachmentPointer(Random.nextBytes(32), data.size) + val messageResult = SignalDatabase.messages.insertMessageInbox(createIncomingMessage(serverTime = receivedAt, attachment = attachment)).get() + val attachmentId = messageResult.insertedAttachments!![attachment]!! + SignalDatabase.attachments.setTransferState(messageResult.messageId, attachmentId, AttachmentTable.TRANSFER_PROGRESS_STARTED) + SignalDatabase.attachments.finalizeAttachmentAfterDownload(messageResult.messageId, attachmentId, ByteArrayInputStream(data)) + return attachmentId + } + + private fun createIncomingMessage(serverTime: Duration, attachment: Attachment): IncomingMessage { + return IncomingMessage( + type = MessageType.NORMAL, + from = harness.others[0], + body = null, + sentTimeMillis = serverTime.inWholeMilliseconds, + serverTimeMillis = serverTime.inWholeMilliseconds, + receivedTimeMillis = serverTime.inWholeMilliseconds, + attachments = listOf(attachment) + ) + } + + private fun createAttachmentPointer(key: ByteArray, size: Int): Attachment { + return PointerAttachment.forPointer( + pointer = Optional.of( + SignalServiceAttachmentPointer( + cdnNumber = 3, + remoteId = SignalServiceAttachmentRemoteId.V4("asdf"), + contentType = MediaUtil.IMAGE_JPEG, + key = key, + size = Optional.of(size), + preview = Optional.empty(), + width = 2, + height = 2, + digest = Optional.of(byteArrayOf()), + incrementalDigest = Optional.empty(), + incrementalMacChunkSize = 0, + fileName = Optional.of("file.jpg"), + voiceNote = false, + isBorderless = false, + isGif = false, + caption = Optional.empty(), + blurHash = Optional.empty(), + uploadTimestamp = 0, + uuid = null + ) + ) + ).get() + } +} diff --git a/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/backup/InternalBackupPlaygroundFragment.kt b/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/backup/InternalBackupPlaygroundFragment.kt index 80ab407c03..3edbfcef34 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/backup/InternalBackupPlaygroundFragment.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/backup/InternalBackupPlaygroundFragment.kt @@ -83,9 +83,11 @@ import org.thoughtcrime.securesms.components.settings.app.internal.backup.Intern import org.thoughtcrime.securesms.dependencies.AppDependencies import org.thoughtcrime.securesms.jobs.ArchiveAttachmentBackfillJob import org.thoughtcrime.securesms.jobs.ArchiveAttachmentReconciliationJob +import org.thoughtcrime.securesms.jobs.ArchiveCommitAttachmentDeletesJob import org.thoughtcrime.securesms.jobs.ArchiveThumbnailBackfillJob import org.thoughtcrime.securesms.jobs.BackupRestoreMediaJob import org.thoughtcrime.securesms.jobs.LocalBackupJob +import org.thoughtcrime.securesms.jobs.OptimizeMediaJob import org.thoughtcrime.securesms.keyvalue.BackupValues import org.thoughtcrime.securesms.keyvalue.SignalStore import org.thoughtcrime.securesms.registration.ui.restore.local.RestoreLocalBackupActivity @@ -161,6 +163,8 @@ class InternalBackupPlaygroundFragment : ComposeFragment() { onEnqueueAttachmentBackfillJob = { AppDependencies.jobManager.add(ArchiveAttachmentBackfillJob()) }, onEnqueueThumbnailBackfillJob = { AppDependencies.jobManager.add(ArchiveThumbnailBackfillJob()) }, onEnqueueMediaRestoreClicked = { AppDependencies.jobManager.add(BackupRestoreMediaJob()) }, + onEnqueueOptimizeMediaClicked = { AppDependencies.jobManager.add(OptimizeMediaJob()) }, + onEnqueueCommitDeletesClicked = { AppDependencies.jobManager.add(ArchiveCommitAttachmentDeletesJob()) }, onHaltAllBackupJobsClicked = { viewModel.haltAllJobs() }, onValidateBackupClicked = { viewModel.validateBackup() }, onSaveEncryptedBackupToDiskClicked = { @@ -357,6 +361,8 @@ fun Screen( onEnqueueMediaRestoreClicked: () -> Unit = {}, onEnqueueAttachmentBackfillJob: () -> Unit = {}, onEnqueueThumbnailBackfillJob: () -> Unit = {}, + onEnqueueOptimizeMediaClicked: () -> Unit = {}, + onEnqueueCommitDeletesClicked: () -> Unit = {}, onWipeDataAndRestoreFromRemoteClicked: () -> Unit = {}, onHaltAllBackupJobsClicked: () -> Unit = {}, onSavePlaintextCopyOfRemoteBackupClicked: () -> Unit = {}, @@ -456,6 +462,18 @@ fun Screen( onClick = onEnqueueMediaRestoreClicked ) + Rows.TextRow( + text = "Enqueue optimize media job", + label = "Schedules a job that will offload local copies of media the archive CDN has confirmed. Normally only runs after a backup.", + onClick = onEnqueueOptimizeMediaClicked + ) + + Rows.TextRow( + text = "Enqueue commit deletes job", + label = "Schedules a job that will delete unreferenced media from the archive CDN. Normally only runs after a backup.", + onClick = onEnqueueCommitDeletesClicked + ) + Rows.TextRow( text = "Halt all backup jobs", label = "Stops all backup-related jobs to the best of our ability.", diff --git a/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/backup/InternalBackupPlaygroundViewModel.kt b/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/backup/InternalBackupPlaygroundViewModel.kt index 7a48de131b..d46be1649c 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/backup/InternalBackupPlaygroundViewModel.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/backup/InternalBackupPlaygroundViewModel.kt @@ -290,6 +290,7 @@ class InternalBackupPlaygroundViewModel : ViewModel() { fun wipeAllDataAndRestoreFromRemote(afterDbRestoreCallback: () -> Unit) { SignalExecutors.BOUNDED_IO.execute { SignalStore.backup.restoreWithCellular = false + SignalStore.backup.clearArchiveVerificationState() restoreFromRemote(afterDbRestoreCallback) } } diff --git a/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/storage/ManageStorageSettingsViewModel.kt b/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/storage/ManageStorageSettingsViewModel.kt index 25996da2bd..a3ae6e4faa 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/storage/ManageStorageSettingsViewModel.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/storage/ManageStorageSettingsViewModel.kt @@ -24,7 +24,7 @@ import org.thoughtcrime.securesms.database.SignalDatabase import org.thoughtcrime.securesms.database.SignalDatabase.Companion.media import org.thoughtcrime.securesms.database.ThreadTable import org.thoughtcrime.securesms.dependencies.AppDependencies -import org.thoughtcrime.securesms.jobs.OptimizeMediaJob +import org.thoughtcrime.securesms.jobs.BackupMessagesJob import org.thoughtcrime.securesms.jobs.RestoreOptimizedMediaJob import org.thoughtcrime.securesms.keyvalue.KeepMessagesDuration import org.thoughtcrime.securesms.keyvalue.SignalStore @@ -37,7 +37,8 @@ class ManageStorageSettingsViewModel : ViewModel() { lengthLimit = if (SignalStore.settings.isTrimByLengthEnabled) SignalStore.settings.threadTrimLength else ManageStorageState.NO_LIMIT, syncTrimDeletes = SignalStore.settings.shouldSyncThreadTrimDeletes(), localBackupsEnabled = SignalStore.backup.newLocalBackupsEnabled, - isPrimary = SignalStore.account.isPrimaryDevice + isPrimary = SignalStore.account.isPrimaryDevice, + initialOptimizeStorage = SignalStore.backup.optimizeStorage ) ) val state = store.asStateFlow() @@ -121,7 +122,7 @@ class ManageStorageSettingsViewModel : ViewModel() { store.update { it.copy( onDeviceStorageOptimizationState = if (enabled) OnDeviceStorageOptimizationState.ENABLED else OnDeviceStorageOptimizationState.DISABLED, - storageOptimizationStateChanged = true + storageOptimizationStateChanged = enabled != it.initialOptimizeStorage ) } } @@ -146,7 +147,10 @@ class ManageStorageSettingsViewModel : ViewModel() { if (state.value.storageOptimizationStateChanged) { when (state.value.onDeviceStorageOptimizationState) { OnDeviceStorageOptimizationState.DISABLED -> RestoreOptimizedMediaJob.enqueue() - OnDeviceStorageOptimizationState.ENABLED -> OptimizeMediaJob.enqueue() + + // Backing up first guarantees a media snapshot exists to reconcile against, and the backup enqueues the offload for us once it succeeds + OnDeviceStorageOptimizationState.ENABLED -> BackupMessagesJob.enqueue() + else -> Unit } } @@ -184,7 +188,8 @@ class ManageStorageSettingsViewModel : ViewModel() { val storageOptimizationStateChanged: Boolean = false, val isPaidTierPending: Boolean = false, val localBackupsEnabled: Boolean = false, - val isPrimary: Boolean = true + val isPrimary: Boolean = true, + val initialOptimizeStorage: Boolean = false ) { companion object { const val NO_LIMIT = 0 diff --git a/app/src/main/java/org/thoughtcrime/securesms/database/AttachmentTable.kt b/app/src/main/java/org/thoughtcrime/securesms/database/AttachmentTable.kt index 3a39000797..ca9f723c41 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/database/AttachmentTable.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/database/AttachmentTable.kt @@ -426,7 +426,9 @@ class AttachmentTable( } /** - * Filters thumbnail snapshot entries down to only those that have at least one eligible attachment capable of thumbnail upload. + * Already-archived thumbnails are kept even with no local data file to upload from, because offloaded media still legitimately references its thumbnail. + * Dropping those entries would take them out of the snapshot, which marks them for deletion off of the CDN. A restored thumbnail counts as archived too: we + * downloaded it from the CDN, so it is there regardless of what the archive state says. */ fun filterThumbnailsWithoutEligibleAttachment(entries: Set): Set { if (entries.isEmpty()) { @@ -442,16 +444,7 @@ class AttachmentTable( readableDatabase .select(DATA_HASH_END, REMOTE_KEY) .from(TABLE_NAME) - .where( - """ - $DATA_HASH_END NOT NULL AND - $REMOTE_KEY NOT NULL AND - $DATA_FILE NOT NULL AND - $TRANSFER_STATE = $TRANSFER_PROGRESS_DONE AND - $QUOTE = 0 AND - $ARCHIVE_THUMBNAIL_TRANSFER_STATE != ${ArchiveTransferState.PERMANENT_FAILURE.value} - """ - ) + .where(buildThumbnailEligibilityClause()) .run() .forEach { cursor -> val hashEnd = cursor.requireNonNullString(DATA_HASH_END) @@ -666,7 +659,7 @@ class AttachmentTable( fun getLast30DaysOfRestorableAttachments(batchSize: Int): List { val thirtyDaysAgo = System.currentTimeMillis().milliseconds - 30.days return readableDatabase - .select("$TABLE_NAME.$ID", MESSAGE_ID, DATA_SIZE, DATA_HASH_END, REMOTE_KEY, STICKER_PACK_ID) + .select("$TABLE_NAME.$ID", MESSAGE_ID, DATA_SIZE, DATA_HASH_END, REMOTE_KEY, STICKER_PACK_ID, CONTENT_TYPE, QUOTE, STICKER_ID) .from("$TABLE_NAME LEFT JOIN ${MessageTable.TABLE_NAME} ON ${MessageTable.TABLE_NAME}.${MessageTable.ID} = $TABLE_NAME.$MESSAGE_ID") .where("$TRANSFER_STATE = ? AND (${MessageTable.TABLE_NAME}.${MessageTable.DATE_RECEIVED} >= ? OR $MESSAGE_ID = ?)", TRANSFER_NEEDS_RESTORE, thirtyDaysAgo.inWholeMilliseconds, WALLPAPER_MESSAGE_ID) .limit(batchSize) @@ -679,7 +672,10 @@ class AttachmentTable( size = it.requireLong(DATA_SIZE), plaintextHash = it.requireString(DATA_HASH_END)?.let { hash -> Base64.decode(hash) }, remoteKey = it.requireString(REMOTE_KEY)?.let { key -> Base64.decode(key) }, - stickerPackId = it.requireString(STICKER_PACK_ID) + stickerPackId = it.requireString(STICKER_PACK_ID), + contentType = it.requireString(CONTENT_TYPE), + quote = it.requireBoolean(QUOTE), + stickerId = it.requireInt(STICKER_ID) ) } } @@ -691,7 +687,7 @@ class AttachmentTable( fun getOlderRestorableAttachments(batchSize: Int): List { val thirtyDaysAgo = System.currentTimeMillis().milliseconds - 30.days return readableDatabase - .select("$TABLE_NAME.$ID", MESSAGE_ID, DATA_SIZE, DATA_HASH_END, REMOTE_KEY, STICKER_PACK_ID) + .select("$TABLE_NAME.$ID", MESSAGE_ID, DATA_SIZE, DATA_HASH_END, REMOTE_KEY, STICKER_PACK_ID, CONTENT_TYPE, QUOTE, STICKER_ID) .from("$TABLE_NAME LEFT JOIN ${MessageTable.TABLE_NAME} ON ${MessageTable.TABLE_NAME}.${MessageTable.ID} = $TABLE_NAME.$MESSAGE_ID") .where("$TRANSFER_STATE = ? AND (${MessageTable.TABLE_NAME}.${MessageTable.DATE_RECEIVED} < ? OR $MESSAGE_ID = ?)", TRANSFER_NEEDS_RESTORE, thirtyDaysAgo.inWholeMilliseconds, WALLPAPER_MESSAGE_ID) .limit(batchSize) @@ -704,14 +700,17 @@ class AttachmentTable( size = it.requireLong(DATA_SIZE), plaintextHash = it.requireString(DATA_HASH_END)?.let { hash -> Base64.decode(hash) }, remoteKey = it.requireString(REMOTE_KEY)?.let { key -> Base64.decode(key) }, - stickerPackId = it.requireString(STICKER_PACK_ID) + stickerPackId = it.requireString(STICKER_PACK_ID), + contentType = it.requireString(CONTENT_TYPE), + quote = it.requireBoolean(QUOTE), + stickerId = it.requireInt(STICKER_ID) ) } } fun getRestorableOptimizedAttachments(): List { return readableDatabase - .select(ID, MESSAGE_ID, DATA_SIZE, DATA_HASH_END, REMOTE_KEY, STICKER_PACK_ID) + .select(ID, MESSAGE_ID, DATA_SIZE, DATA_HASH_END, REMOTE_KEY, STICKER_PACK_ID, CONTENT_TYPE, QUOTE, STICKER_ID) .from(TABLE_NAME) .where("$TRANSFER_STATE = ? AND $DATA_HASH_END NOT NULL AND $REMOTE_KEY NOT NULL", TRANSFER_RESTORE_OFFLOADED) .orderBy("$ID DESC") @@ -723,7 +722,10 @@ class AttachmentTable( size = it.requireLong(DATA_SIZE), plaintextHash = it.requireString(DATA_HASH_END)?.let { hash -> Base64.decode(hash) }, remoteKey = it.requireString(REMOTE_KEY)?.let { key -> Base64.decode(key) }, - stickerPackId = it.requireString(STICKER_PACK_ID) + stickerPackId = it.requireString(STICKER_PACK_ID), + contentType = it.requireString(CONTENT_TYPE), + quote = it.requireBoolean(QUOTE), + stickerId = it.requireInt(STICKER_ID) ) } } @@ -948,16 +950,7 @@ class AttachmentTable( fun doAnyThumbnailsNeedArchiveUpload(): Boolean { return readableDatabase .exists("$TABLE_NAME INNER JOIN ${MessageTable.TABLE_NAME} ON $TABLE_NAME.$MESSAGE_ID = ${MessageTable.TABLE_NAME}.${MessageTable.ID}") - .where( - """ - ${buildAttachmentsThatCanArchiveQuery("$ARCHIVE_THUMBNAIL_TRANSFER_STATE IN (${ArchiveTransferState.NONE.value}, ${ArchiveTransferState.TEMPORARY_FAILURE.value})")} AND - $QUOTE = 0 AND - $STICKER_ID = -1 AND - ($CONTENT_TYPE LIKE 'image/%' OR $CONTENT_TYPE LIKE 'video/%') AND - $CONTENT_TYPE != 'image/svg+xml' AND - $MESSAGE_ID != $WALLPAPER_MESSAGE_ID - """ - ) + .where(buildThumbnailsThatNeedArchiveWorkQuery()) .run() } @@ -996,16 +989,7 @@ class AttachmentTable( return readableDatabase .select("$TABLE_NAME.$ID") .from("$TABLE_NAME INNER JOIN ${MessageTable.TABLE_NAME} ON $TABLE_NAME.$MESSAGE_ID = ${MessageTable.TABLE_NAME}.${MessageTable.ID}") - .where( - """ - ${buildAttachmentsThatCanArchiveQuery("$ARCHIVE_THUMBNAIL_TRANSFER_STATE IN (${ArchiveTransferState.NONE.value}, ${ArchiveTransferState.TEMPORARY_FAILURE.value})")} AND - $QUOTE = 0 AND - $STICKER_ID = -1 AND - ($CONTENT_TYPE LIKE 'image/%' OR $CONTENT_TYPE LIKE 'video/%') AND - $CONTENT_TYPE != 'image/svg+xml' AND - $MESSAGE_ID != $WALLPAPER_MESSAGE_ID - """ - ) + .where(buildThumbnailsThatNeedArchiveWorkQuery()) .run() .readToList { AttachmentId(it.requireLong(ID)) } } @@ -1034,6 +1018,13 @@ class AttachmentTable( .readToSingleObject { ArchiveTransferState.deserialize(it.requireInt(ARCHIVE_THUMBNAIL_TRANSFER_STATE)) } } + fun hasThumbnailFile(id: AttachmentId): Boolean { + return readableDatabase + .exists(TABLE_NAME) + .where("$ID = ? AND $THUMBNAIL_FILE NOT NULL", id.id) + .run() + } + /** * Sets the archive transfer state for the given attachment and all other attachments that share the same data file. */ @@ -1161,28 +1152,63 @@ class AttachmentTable( /** * Resets the archive upload state by hash/key if we believe the attachment should have been uploaded already. */ - fun resetArchiveTransferStateByPlaintextHashAndRemoteKeyIfNecessary(plaintextHash: ByteArray, remoteKey: ByteArray): Boolean { - return writableDatabase - .update(TABLE_NAME) - .values( + fun resetArchiveTransferStateByPlaintextHashAndRemoteKeyIfNecessary(plaintextHash: ByteArray, remoteKey: ByteArray): ArchiveTransferStateResetResult { + return resetArchiveTransferStateIfNecessary( + plaintextHash = plaintextHash, + remoteKey = remoteKey, + stateColumn = ARCHIVE_TRANSFER_STATE, + values = contentValuesOf( ARCHIVE_TRANSFER_STATE to ArchiveTransferState.NONE.value, ARCHIVE_CDN to null ) - .where("$DATA_HASH_END = ? AND $REMOTE_KEY = ? AND $ARCHIVE_TRANSFER_STATE = ${ArchiveTransferState.FINISHED.value}", Base64.encodeWithPadding(plaintextHash), Base64.encodeWithPadding(remoteKey)) - .run() > 0 + ) } /** * Resets the archive thumbnail upload state by hash/key if we believe the thumbnail should have been uploaded already. */ - fun resetArchiveThumbnailTransferStateByPlaintextHashAndRemoteKeyIfNecessary(plaintextHash: ByteArray, remoteKey: ByteArray): Boolean { - return writableDatabase - .update(TABLE_NAME) - .values( - ARCHIVE_THUMBNAIL_TRANSFER_STATE to ArchiveTransferState.NONE.value - ) - .where("$DATA_HASH_END = ? AND $REMOTE_KEY = ? AND $ARCHIVE_THUMBNAIL_TRANSFER_STATE = ${ArchiveTransferState.FINISHED.value}", Base64.encodeWithPadding(plaintextHash), Base64.encodeWithPadding(remoteKey)) - .run() > 0 + fun resetArchiveThumbnailTransferStateByPlaintextHashAndRemoteKeyIfNecessary(plaintextHash: ByteArray, remoteKey: ByteArray): ArchiveTransferStateResetResult { + return resetArchiveTransferStateIfNecessary( + plaintextHash = plaintextHash, + remoteKey = remoteKey, + stateColumn = ARCHIVE_THUMBNAIL_TRANSFER_STATE, + values = contentValuesOf(ARCHIVE_THUMBNAIL_TRANSFER_STATE to ArchiveTransferState.NONE.value) + ) + } + + private fun resetArchiveTransferStateIfNecessary(plaintextHash: ByteArray, remoteKey: ByteArray, stateColumn: String, values: ContentValues): ArchiveTransferStateResetResult { + val encodedHash = Base64.encodeWithPadding(plaintextHash) + val encodedKey = Base64.encodeWithPadding(remoteKey) + val finishedClause = "$DATA_HASH_END = ? AND $REMOTE_KEY = ? AND $stateColumn = ${ArchiveTransferState.FINISHED.value}" + + return writableDatabase.withinTransaction { db -> + val isFinished = db + .exists(TABLE_NAME) + .where(finishedClause, encodedHash, encodedKey) + .run() + + if (!isFinished) { + return@withinTransaction ArchiveTransferStateResetResult.NOT_NEEDED + } + + val hasLocalData = db + .exists(TABLE_NAME) + .where("$DATA_HASH_END = ? AND $REMOTE_KEY = ? AND $DATA_FILE NOT NULL", encodedHash, encodedKey) + .run() + + // With no local data file there is nothing to re-upload, and clearing the state makes the export emit a tombstone instead of a locator, which in turn marks + // the media for deletion off of the CDN. + if (!hasLocalData) { + return@withinTransaction ArchiveTransferStateResetResult.SKIPPED_NO_LOCAL_DATA + } + + db.update(TABLE_NAME) + .values(values) + .where(finishedClause, encodedHash, encodedKey) + .run() + + ArchiveTransferStateResetResult.RESET + } } /** @@ -1233,49 +1259,128 @@ class AttachmentTable( * * Marking offloaded only clears the strong references to the on disk file and clears other local file data like hashes. * Another operation must run to actually delete the data from disk. See [deleteAbandonedAttachmentFiles]. + * + * @param lastCompletedCrawlVersion The snapshot version of the most recent reconciliation that finished crawling the archive CDN, or negative if none ever has. + * @param pageSize How many candidates to derive mediaIds for at a time. */ - fun markEligibleAttachmentsAsOptimized(minimumAge: Duration = 30.days) { - val now = System.currentTimeMillis() + fun markEligibleAttachmentsAsOptimized(lastCompletedCrawlVersion: Long, minimumAge: Duration = 30.days, now: Long = System.currentTimeMillis(), pageSize: Int = 500) { + if (lastCompletedCrawlVersion < 0) { + Log.w(TAG, "No reconciliation has ever completed on this device. Refusing to offload anything.", true) + return + } - val subSelect = """ - SELECT $TABLE_NAME.$ID - FROM $TABLE_NAME - INNER JOIN ${MessageTable.TABLE_NAME} ON ${MessageTable.TABLE_NAME}.${MessageTable.ID} = $TABLE_NAME.$MESSAGE_ID - WHERE + // Everything that looks offloadable locally, saying nothing about whether the archive CDN actually has the bytes + fun eligibilityClause(prefix: String): String = """ + $prefix$OFFLOAD_RESTORED_AT < ${now - 7.days.inWholeMilliseconds} AND + $prefix$TRANSFER_STATE = $TRANSFER_PROGRESS_DONE AND + $prefix$ARCHIVE_TRANSFER_STATE = ${ArchiveTransferState.FINISHED.value} AND + $prefix$DATA_FILE IS NOT NULL AND + $prefix$STICKER_ID = -1 AND + $prefix$REMOTE_KEY IS NOT NULL AND + $prefix$DATA_HASH_END IS NOT NULL AND ( - $TABLE_NAME.$OFFLOAD_RESTORED_AT < ${now - 7.days.inWholeMilliseconds} AND - $TABLE_NAME.$TRANSFER_STATE = $TRANSFER_PROGRESS_DONE AND - $TABLE_NAME.$ARCHIVE_TRANSFER_STATE = ${ArchiveTransferState.FINISHED.value} AND - $TABLE_NAME.$DATA_FILE IS NOT NULL AND - $TABLE_NAME.$STICKER_ID = -1 AND - $TABLE_NAME.$REMOTE_KEY IS NOT NULL AND - $TABLE_NAME.$DATA_HASH_END IS NOT NULL AND - ( - $TABLE_NAME.$THUMBNAIL_FILE IS NOT NULL OR - NOT ($TABLE_NAME.$CONTENT_TYPE LIKE 'image/%' OR $TABLE_NAME.$CONTENT_TYPE LIKE 'video/%') OR - $TABLE_NAME.$CONTENT_TYPE = 'image/svg+xml' - ) - ) - AND - ( - ${MessageTable.TABLE_NAME}.${MessageTable.DATE_RECEIVED} < ${now - minimumAge.inWholeMilliseconds} + $prefix$THUMBNAIL_FILE IS NOT NULL OR + NOT ($prefix$CONTENT_TYPE LIKE 'image/%' OR $prefix$CONTENT_TYPE LIKE 'video/%') OR + $prefix$CONTENT_TYPE = 'image/svg+xml' ) """ - val count = writableDatabase - .update(TABLE_NAME) - .values( - TRANSFER_STATE to TRANSFER_RESTORE_OFFLOADED, - DATA_FILE to null, - DATA_RANDOM to null, - TRANSFORM_PROPERTIES to null, - DATA_HASH_START to null, - OFFLOAD_RESTORED_AT to 0 - ) - .where("$ID in ($subSelect)") - .run() + val oldestAllowedDateReceived = now - minimumAge.inWholeMilliseconds + val mediaRootBackupKey = SignalStore.backup.mediaRootBackupKey - Log.i(TAG, "Marked $count attachments as optimized") + var lastId = 0L + var offloadedCount = 0 + var candidateCount = 0 + var unconfirmedCount = 0 + + while (true) { + // Paged by id rather than looping until the candidate query runs dry: candidates the CDN never confirms stay eligible forever, so only a strictly advancing + // cursor terminates. + val candidates: List> = readableDatabase + .rawQuery( + """ + SELECT $TABLE_NAME.$ID, $TABLE_NAME.$DATA_HASH_END, $TABLE_NAME.$REMOTE_KEY + FROM $TABLE_NAME + INNER JOIN ${MessageTable.TABLE_NAME} ON ${MessageTable.TABLE_NAME}.${MessageTable.ID} = $TABLE_NAME.$MESSAGE_ID + WHERE + $TABLE_NAME.$ID > $lastId AND + ${eligibilityClause("$TABLE_NAME.")} AND + ${MessageTable.TABLE_NAME}.${MessageTable.DATE_RECEIVED} < $oldestAllowedDateReceived + ORDER BY $TABLE_NAME.$ID ASC + LIMIT $pageSize + """ + ) + .readToList { cursor -> + cursor.requireLong(ID) to MediaNameParts( + plaintextHash = cursor.requireNonNullString(DATA_HASH_END), + remoteKey = cursor.requireNonNullString(REMOTE_KEY) + ) + } + + if (candidates.isEmpty()) { + break + } + + lastId = candidates.last().first + + // Group by mediaId, which is the only thing the snapshot table can be joined on. Scoped to the page so the derivation cache can't grow without bound. + val mediaIdsByMediaName: MutableMap = mutableMapOf() + val candidateIdsByMediaId: MutableMap> = mutableMapOf() + + candidates.forEach { (id, mediaName) -> + val mediaId = mediaIdsByMediaName.getOrPut(mediaName) { + MediaName.fromPlaintextHashAndRemoteKey(Base64.decode(mediaName.plaintextHash), Base64.decode(mediaName.remoteKey)) + .toMediaId(mediaRootBackupKey) + .encode() + } + + candidateIdsByMediaId.getOrPut(mediaId) { mutableListOf() } += id + } + + // Narrow to media the server itself told us about. This is the only step backed by evidence that didn't originate on this device. + val confirmedOnCdn = SignalDatabase.backupMediaSnapshots.getMediaIdsConfirmedOnCdn(candidateIdsByMediaId.keys, lastCompletedCrawlVersion) + val idsToOffload: List = confirmedOnCdn.flatMap { candidateIdsByMediaId[it] ?: emptyList() } + + candidateCount += candidateIdsByMediaId.size + unconfirmedCount += candidateIdsByMediaId.size - confirmedOnCdn.size + + if (idsToOffload.isNotEmpty()) { + // Eligibility is re-checked at write time, since anything could have changed while we were deriving mediaIds outside of a transaction. + val idQuery = SqlUtil.buildFastCollectionQuery(ID, idsToOffload) + + offloadedCount += writableDatabase + .update(TABLE_NAME) + .values( + TRANSFER_STATE to TRANSFER_RESTORE_OFFLOADED, + DATA_FILE to null, + DATA_RANDOM to null, + TRANSFORM_PROPERTIES to null, + DATA_HASH_START to null, + OFFLOAD_RESTORED_AT to 0 + ) + .where( + """ + ${idQuery.where} AND + ${eligibilityClause("")} AND + EXISTS ( + SELECT 1 + FROM ${MessageTable.TABLE_NAME} + WHERE + ${MessageTable.TABLE_NAME}.${MessageTable.ID} = $TABLE_NAME.$MESSAGE_ID AND + ${MessageTable.TABLE_NAME}.${MessageTable.DATE_RECEIVED} < $oldestAllowedDateReceived + ) + """, + idQuery.whereArgs + ) + .run() + } + } + + if (unconfirmedCount > 0) { + Log.w(TAG, "Skipping $unconfirmedCount/$candidateCount candidate media objects that the archive CDN has not confirmed since snapshot version $lastCompletedCrawlVersion.", true) + } + + Log.i(TAG, "Marked $offloadedCount attachments as optimized") } /** @@ -1659,12 +1764,23 @@ class AttachmentTable( writableDatabase .update(TABLE_NAME) .values(THUMBNAIL_RESTORE_STATE to ThumbnailRestoreState.PERMANENT_FAILURE.value) - .where("$ID = ? AND $THUMBNAIL_RESTORE_STATE != ?", attachmentId.id, ThumbnailRestoreState.FINISHED) + .where("$ID = ? AND $THUMBNAIL_RESTORE_STATE != ?", attachmentId.id, ThumbnailRestoreState.FINISHED.value) .run() notifyConversationListeners(messages.getThreadIdForMessage(mmsId)) } + /** + * Records that a thumbnail can't be built from the local data file. Skips rows with a restore pending or completed. + */ + fun markThumbnailPermanentlyFailedIfUnrestorable(attachmentId: AttachmentId) { + writableDatabase + .update(TABLE_NAME) + .values(THUMBNAIL_RESTORE_STATE to ThumbnailRestoreState.PERMANENT_FAILURE.value) + .where("$ID = ? AND $THUMBNAIL_RESTORE_STATE = ?", attachmentId.id, ThumbnailRestoreState.NONE.value) + .run() + } + fun setTransferProgressPermanentFailure(attachmentId: AttachmentId, mmsId: Long) { writableDatabase .update(TABLE_NAME) @@ -2571,6 +2687,7 @@ class AttachmentTable( } val objectsByMediaId: Map = objects.associateBy { it.mediaId } + val mediaRootBackupKey = SignalStore.backup.mediaRootBackupKey // Collect updates grouped by CDN: Map>> val updatesByCdn: MutableMap>> = mutableMapOf() @@ -2588,7 +2705,7 @@ class AttachmentTable( val plaintextHash = Base64.decode(plaintextHashStr) val mediaId = MediaName.fromPlaintextHashAndRemoteKey(plaintextHash, remoteKey) - .toMediaId(SignalStore.backup.mediaRootBackupKey) + .toMediaId(mediaRootBackupKey) .encode() val matchingObject = objectsByMediaId[mediaId] @@ -2631,6 +2748,83 @@ class AttachmentTable( return updatedCount } + /** + * A restore only carries a claim about the full-size object, so a CDN listing is the only proof we ever get that a *thumbnail* object exists. Without this a + * restored device both drops the thumbnail from its snapshot and re-uploads a thumbnail the CDN already has. + * + * @return the number of unique (plaintextHash, remoteKey) pairs that were updated + */ + fun setArchiveThumbnailFinishedForMatchingMediaObjects(objects: Set): Int { + if (objects.isEmpty()) { + return 0 + } + + val candidateClause = """ + $DATA_HASH_END NOT NULL AND + $REMOTE_KEY NOT NULL AND + $ARCHIVE_TRANSFER_STATE = ${ArchiveTransferState.FINISHED.value} AND + $ARCHIVE_THUMBNAIL_TRANSFER_STATE != ${ArchiveTransferState.FINISHED.value} + """ + + // Skips the group-by scan below in the common case, where every thumbnail state is already correct and there is nothing to promote. + if (!readableDatabase.exists(TABLE_NAME).where(candidateClause).run()) { + return 0 + } + + val mediaIds: Set = objects.map { it.mediaId }.toSet() + val mediaRootBackupKey = SignalStore.backup.mediaRootBackupKey + val matches: MutableList = mutableListOf() + + readableDatabase + .select(DATA_HASH_END, REMOTE_KEY) + .from(TABLE_NAME) + .where(candidateClause) + .groupBy("$DATA_HASH_END, $REMOTE_KEY") + .run() + .readToList { cursor -> + MediaNameParts( + plaintextHash = cursor.requireNonNullString(DATA_HASH_END), + remoteKey = cursor.requireNonNullString(REMOTE_KEY) + ) + } + .forEach { parts -> + val thumbnailMediaId = MediaName + .fromPlaintextHashAndRemoteKeyForThumbnail(Base64.decode(parts.plaintextHash), Base64.decode(parts.remoteKey)) + .toMediaId(mediaRootBackupKey) + .encode() + + if (thumbnailMediaId in mediaIds) { + matches += parts + } + } + + if (matches.isEmpty()) { + return 0 + } + + var updatedCount = 0 + + writableDatabase.withinTransaction { db -> + for (batch in matches.chunked(250)) { + val whereClause = batch.joinToString(" OR ") { "($DATA_HASH_END = ? AND $REMOTE_KEY = ?)" } + val whereArgs = batch.flatMap { listOf(it.plaintextHash, it.remoteKey) }.toTypedArray() + + db.update(TABLE_NAME) + .values(ARCHIVE_THUMBNAIL_TRANSFER_STATE to ArchiveTransferState.FINISHED.value) + .where(whereClause, *whereArgs) + .run() + + updatedCount += batch.size + } + } + + if (updatedCount > 0) { + AppDependencies.databaseObserver.notifyAttachmentUpdatedObservers() + } + + return updatedCount + } + fun clearArchiveData(attachmentId: AttachmentId) { writableDatabase .update(TABLE_NAME) @@ -3453,6 +3647,31 @@ class AttachmentTable( ) } + /** + * Thumbnails that need work, which is either "the archive has no copy" or "we have no local copy". + */ + private fun buildThumbnailsThatNeedArchiveWorkQuery(): String { + val stateFilter = """ + ( + $ARCHIVE_THUMBNAIL_TRANSFER_STATE IN (${ArchiveTransferState.NONE.value}, ${ArchiveTransferState.TEMPORARY_FAILURE.value}) OR + ( + $ARCHIVE_THUMBNAIL_TRANSFER_STATE = ${ArchiveTransferState.FINISHED.value} AND + $THUMBNAIL_FILE IS NULL AND + $THUMBNAIL_RESTORE_STATE != ${ThumbnailRestoreState.PERMANENT_FAILURE.value} + ) + ) + """ + + return """ + ${buildAttachmentsThatCanArchiveQuery(stateFilter)} AND + $QUOTE = 0 AND + $STICKER_ID = -1 AND + ($CONTENT_TYPE LIKE 'image/%' OR $CONTENT_TYPE LIKE 'video/%') AND + $CONTENT_TYPE != 'image/svg+xml' AND + $MESSAGE_ID != $WALLPAPER_MESSAGE_ID + """ + } + /** * IMPORTANT: This query may match against rows that have no associated message (like a wallpaper attachment). * This needs to be accounted by allowing nulls when reading any message table row. @@ -3593,6 +3812,7 @@ class AttachmentTable( fun getAttachmentDataForMediaIds(mediaIds: Collection): List { if (mediaIds.isEmpty()) return emptyList() val mediaIdByteStrings = mediaIds.map { it.value.toByteString() }.toSet() + val mediaRootBackupKey = SignalStore.backup.mediaRootBackupKey val found: MutableList = mutableListOf() @@ -3609,13 +3829,13 @@ class AttachmentTable( val mediaId = MediaName .fromPlaintextHashAndRemoteKey(plaintextHash, remoteKey) - .toMediaId(SignalStore.backup.mediaRootBackupKey) + .toMediaId(mediaRootBackupKey) .value .toByteString() val mediaIdThumbnail = MediaName .fromPlaintextHashAndRemoteKeyForThumbnail(plaintextHash, remoteKey) - .toMediaId(SignalStore.backup.mediaRootBackupKey) + .toMediaId(mediaRootBackupKey) .value .toByteString() @@ -3635,27 +3855,124 @@ class AttachmentTable( return found } + /** + * Whether an attachment still makes its thumbnail worth having on the archive CDN: we can either upload one from local data, or one is already up there. + * Mirrors the snapshot write side, except stickers, which it doesn't exclude either, so filtering them here would delete thumbnails it still wants. + */ + private fun buildThumbnailEligibilityClause(): String { + return """ + $DATA_HASH_END NOT NULL AND + $REMOTE_KEY NOT NULL AND + $QUOTE = 0 AND + $MESSAGE_ID != $WALLPAPER_MESSAGE_ID AND + ($CONTENT_TYPE LIKE 'image/%' OR $CONTENT_TYPE LIKE 'video/%') AND + $CONTENT_TYPE != 'image/svg+xml' AND + $ARCHIVE_THUMBNAIL_TRANSFER_STATE != ${ArchiveTransferState.PERMANENT_FAILURE.value} AND + ( + ($DATA_FILE NOT NULL AND $TRANSFER_STATE = $TRANSFER_PROGRESS_DONE) OR + $ARCHIVE_THUMBNAIL_TRANSFER_STATE = ${ArchiveTransferState.FINISHED.value} OR + $THUMBNAIL_RESTORE_STATE = ${ThumbnailRestoreState.FINISHED.value} + ) + """ + } + + /** + * Stops an attachment from counting as a referent when the over-size-limit cutoff left its message out of the backup, matching the exporter's + * `$DATE_RECEIVED >= cutoff`. Attachments with no message row at all, like wallpapers, are deliberately still referents. + * + * @param messageInclusionCutoffTime Zero when every message made it into the backup, which is the overwhelmingly common case. + */ + private fun buildMessageIncludedInBackupClause(messageInclusionCutoffTime: Long): String { + if (messageInclusionCutoffTime <= 0) { + return "" + } + + return """ + AND NOT EXISTS ( + SELECT 1 + FROM ${MessageTable.TABLE_NAME} + WHERE + ${MessageTable.TABLE_NAME}.${MessageTable.ID} = $TABLE_NAME.$MESSAGE_ID AND + ${MessageTable.TABLE_NAME}.${MessageTable.DATE_RECEIVED} < $messageInclusionCutoffTime + ) + """ + } + + /** + * The thumbnail counterpart to [getMediaNamesWithNoAttachment]: of the given media names, the ones no attachment still wants a thumbnail for. + */ + fun getMediaNamesWithNoEligibleThumbnail(mediaNames: Set, messageInclusionCutoffTime: Long = 0): Set { + if (mediaNames.isEmpty()) { + return emptySet() + } + + val gone: MutableSet = mediaNames.toMutableSet() + + for (batch in mediaNames.chunked(250)) { + val whereClause = batch.joinToString(" OR ") { "($DATA_HASH_END = ? AND $REMOTE_KEY = ?)" } + val whereArgs = batch.flatMap { listOf(it.plaintextHash, it.remoteKey) }.toTypedArray() + + readableDatabase + .select(DATA_HASH_END, REMOTE_KEY) + .from(TABLE_NAME) + .where("($whereClause) AND ${buildThumbnailEligibilityClause()}${buildMessageIncludedInBackupClause(messageInclusionCutoffTime)}", *whereArgs) + .groupBy("$DATA_HASH_END, $REMOTE_KEY") + .run() + .forEach { cursor -> + gone -= MediaNameParts(plaintextHash = cursor.requireNonNullString(DATA_HASH_END), remoteKey = cursor.requireNonNullString(REMOTE_KEY)) + } + } + + return gone + } + + fun getMediaNamesWithNoAttachment(mediaNames: Set, messageInclusionCutoffTime: Long = 0): Set { + if (mediaNames.isEmpty()) { + return emptySet() + } + + val gone: MutableSet = mediaNames.toMutableSet() + + for (batch in mediaNames.chunked(250)) { + val whereClause = batch.joinToString(" OR ") { "($DATA_HASH_END = ? AND $REMOTE_KEY = ?)" } + val whereArgs = batch.flatMap { listOf(it.plaintextHash, it.remoteKey) }.toTypedArray() + + readableDatabase + .select(DATA_HASH_END, REMOTE_KEY) + .from(TABLE_NAME) + .where("($whereClause)${buildMessageIncludedInBackupClause(messageInclusionCutoffTime)}", *whereArgs) + .groupBy("$DATA_HASH_END, $REMOTE_KEY") + .run() + .forEach { cursor -> + gone -= MediaNameParts(plaintextHash = cursor.requireNonNullString(DATA_HASH_END), remoteKey = cursor.requireNonNullString(REMOTE_KEY)) + } + } + + return gone + } + /** * Given a set of media objects, this will return all of the items in the set that could not be found locally. */ - fun getMediaObjectsThatCantBeFound(objects: Set): Set { + fun getMediaObjectsThatCantBeFound(objects: Set, messageInclusionCutoffTime: Long = 0): Set { if (objects.isEmpty()) { return emptySet() } val objectsByMediaId: MutableMap = objects.associateBy { it.mediaId }.toMutableMap() + val mediaRootBackupKey = SignalStore.backup.mediaRootBackupKey readableDatabase .select(*PROJECTION) .from(TABLE_NAME) - .where("$REMOTE_KEY NOT NULL AND $DATA_HASH_END NOT NULL") + .where("$REMOTE_KEY NOT NULL AND $DATA_HASH_END NOT NULL${buildMessageIncludedInBackupClause(messageInclusionCutoffTime)}") .groupBy("$DATA_HASH_END, $REMOTE_KEY") .run() .forEach { cursor -> val remoteKey = Base64.decode(cursor.requireNonNullString(REMOTE_KEY)) val plaintextHash = Base64.decode(cursor.requireNonNullString(DATA_HASH_END)) - val mediaId = MediaName.fromPlaintextHashAndRemoteKey(plaintextHash, remoteKey).toMediaId(SignalStore.backup.mediaRootBackupKey).encode() - val mediaIdThumbnail = MediaName.fromPlaintextHashAndRemoteKeyForThumbnail(plaintextHash, remoteKey).toMediaId(SignalStore.backup.mediaRootBackupKey).encode() + val mediaId = MediaName.fromPlaintextHashAndRemoteKey(plaintextHash, remoteKey).toMediaId(mediaRootBackupKey).encode() + val mediaIdThumbnail = MediaName.fromPlaintextHashAndRemoteKeyForThumbnail(plaintextHash, remoteKey).toMediaId(mediaRootBackupKey).encode() objectsByMediaId.remove(mediaId) objectsByMediaId.remove(mediaIdThumbnail) @@ -4039,8 +4356,24 @@ class AttachmentTable( val size: Long, val plaintextHash: ByteArray?, val remoteKey: ByteArray?, - val stickerPackId: String? + val stickerPackId: String?, + val contentType: String?, + val quote: Boolean, + val stickerId: Int ) { + /** + * Whether a thumbnail for this attachment could ever have been uploaded, matches [buildThumbnailsThatNeedArchiveWorkQuery] and + * `BackupMessagesJob.toThumbnailMediaEntries`. + */ + val couldHaveArchivedThumbnail: Boolean + get() { + return mmsId != WALLPAPER_MESSAGE_ID && + !quote && + stickerId == -1 && + contentType != "image/svg+xml" && + MediaUtil.isImageOrVideoType(contentType) + } + override fun equals(other: Any?): Boolean { return this === other || attachmentId == (other as? RestorableAttachment)?.attachmentId } @@ -4165,4 +4498,20 @@ class AttachmentTable( return "attachmentId=${attachment.attachmentId}, mediaId=$mediaId, messageId=${attachment.mmsId}, isThumbnail=$isThumbnail, contentType=${attachment.contentType}, quote=${attachment.quote}, wallpaper=$isWallpaper, transferState=${attachment.transferState}, archiveTransferState=${attachment.archiveTransferState}, hasData=${attachment.hasData}, dateSent=${messageRecord?.dateSent}, messageType=${messageRecord?.type}, messageFrom=${messageRecord?.fromRecipient?.id}, messageTo=${messageRecord?.toRecipient?.id}, expiresIn=${messageRecord?.expiresIn}, expireStarted=${messageRecord?.expireStarted}" } } + + /** + * The base64 [DATA_HASH_END] and [REMOTE_KEY] that a [MediaName] is derived from. + */ + data class MediaNameParts(val plaintextHash: String, val remoteKey: String) + + enum class ArchiveTransferStateResetResult { + /** We cleared the state, so the media will be re-uploaded. */ + RESET, + + /** We left the state alone because there are no local bytes to re-upload, so the media is unrecoverable rather than merely out of date. */ + SKIPPED_NO_LOCAL_DATA, + + /** There was nothing to clear. The attachment is gone, or an upload is already underway. */ + NOT_NEEDED + } } diff --git a/app/src/main/java/org/thoughtcrime/securesms/database/BackupMediaSnapshotTable.kt b/app/src/main/java/org/thoughtcrime/securesms/database/BackupMediaSnapshotTable.kt index a2a7ba4448..ac3f98e9b6 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/database/BackupMediaSnapshotTable.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/database/BackupMediaSnapshotTable.kt @@ -20,6 +20,7 @@ import org.signal.core.util.readToSingleLong import org.signal.core.util.requireBoolean import org.signal.core.util.requireInt import org.signal.core.util.requireIntOrNull +import org.signal.core.util.requireLong import org.signal.core.util.requireNonNullBlob import org.signal.core.util.requireNonNullString import org.signal.core.util.select @@ -195,22 +196,44 @@ class BackupMediaSnapshotTable(context: Context, database: SignalDatabase) : Dat .readToSingleLong(0) } - fun getPageOfOldMediaObjects(pageSize: Int): Set { - return readableDatabase.select(MEDIA_ID, CDN) + /** + * Media objects that have dropped out of the most recent snapshot. Carries the hash/key so callers can check whether anything in the [AttachmentTable] still + * refers to them before deleting them off of the CDN. + */ + fun getPageOfOldMediaEntries(pageSize: Int, afterId: Long = 0): List { + return readableDatabase.select(ID, MEDIA_ID, CDN, PLAINTEXT_HASH, REMOTE_KEY, IS_THUMBNAIL) .from(TABLE_NAME) - .where("$SNAPSHOT_VERSION < $MAX_VERSION AND $IS_PENDING = 0") + .where("$SNAPSHOT_VERSION < $MAX_VERSION AND $IS_PENDING = 0 AND $ID > $afterId") + .orderBy("$ID ASC") .limit(pageSize) .run() - .readToSet { - ArchivedMediaObject(mediaId = it.requireNonNullString(MEDIA_ID), cdn = it.requireInt(CDN)) - } + .readToList { ExistingMediaEntry.fromCursor(it) } + } + + /** + * Of the given mediaIds, the ones present in the most recent snapshot that a reconciliation crawl has since seen on the archive CDN. Being in the snapshot + * only reflects our own bookkeeping, so [LAST_SEEN_ON_REMOTE_SNAPSHOT_VERSION] > 0 is the only part of this that the server told us. + */ + fun getMediaIdsConfirmedOnCdn(mediaIds: Collection, minimumLastSeenSnapshotVersion: Long): Set { + if (mediaIds.isEmpty()) { + return emptySet() + } + + val query = SqlUtil.buildFastCollectionQuery(MEDIA_ID, mediaIds) + + return readableDatabase + .select(MEDIA_ID) + .from(TABLE_NAME) + .where("${query.where} AND $SNAPSHOT_VERSION = $MAX_VERSION AND $LAST_SEEN_ON_REMOTE_SNAPSHOT_VERSION > 0 AND $LAST_SEEN_ON_REMOTE_SNAPSHOT_VERSION >= $minimumLastSeenSnapshotVersion", query.whereArgs) + .run() + .readToSet { it.requireNonNullString(MEDIA_ID) } } /** * This will remove any old snapshot entries with matching mediaId's. No pending entries or entries in the latest snapshot will be affected. */ - fun deleteOldMediaObjects(mediaObjects: Collection) { - val query = SqlUtil.buildFastCollectionQuery(MEDIA_ID, mediaObjects.map { it.mediaId }) + fun deleteOldMediaObjects(mediaIds: Collection) { + val query = SqlUtil.buildFastCollectionQuery(MEDIA_ID, mediaIds) writableDatabase.delete(TABLE_NAME) .where("$SNAPSHOT_VERSION < $MAX_VERSION AND $IS_PENDING = 0 AND " + query.where, query.whereArgs) @@ -251,32 +274,6 @@ class BackupMediaSnapshotTable(context: Context, database: SignalDatabase) : Dat return objectsByMediaId.values.toSet() } - fun getMediaEntriesForObjects(objects: List): Set { - if (objects.isEmpty()) { - return emptySet() - } - - val queries: List = SqlUtil.buildCollectionQuery( - column = MEDIA_ID, - values = objects.map { it.mediaId }, - collectionOperator = SqlUtil.CollectionOperator.IN, - prefix = "$SNAPSHOT_VERSION = $MAX_VERSION AND " - ) - - val entries: MutableSet = mutableSetOf() - - for (query in queries) { - entries += readableDatabase - .select(MEDIA_ID, CDN, PLAINTEXT_HASH, REMOTE_KEY, IS_THUMBNAIL) - .from("$TABLE_NAME JOIN ${AttachmentTable.TABLE_NAME}") - .where(query.where, query.whereArgs) - .run() - .readToList { MediaEntry.fromCursor(it) } - } - - return entries.toSet() - } - /** * Given a list of media objects, find the ones that are present in the most recent snapshot, but have a different CDN than the one passed in. * This will ignore thumbnails, as the results are intended to be used to update CDNs, which we do not track for thumbnails. @@ -308,8 +305,25 @@ class BackupMediaSnapshotTable(context: Context, database: SignalDatabase) : Dat } } + /** + * Limited to rows *no* crawl has ever confirmed, so we never contradict our own earlier evidence that an object existed. Only valid after a *completed* crawl, + * since on a partial one an unstamped row may simply not have been paged yet, and deleting it would orphan the object it tracks. + * + * @param crawlSnapshotVersion The version the crawl pinned itself to. Rows above it were committed after the crawl fixed its view, so an upload that landed + * behind the cursor still looks unconfirmed. Dropping those would leave an object on the CDN with nothing left tracking it. + */ + fun deleteOldMediaObjectsNeverSeenOnCdn(crawlSnapshotVersion: Long): Int { + return writableDatabase + .delete(TABLE_NAME) + .where("$SNAPSHOT_VERSION < $MAX_VERSION AND $SNAPSHOT_VERSION <= $crawlSnapshotVersion AND $IS_PENDING = 0 AND $LAST_SEEN_ON_REMOTE_SNAPSHOT_VERSION = 0") + .run() + } + /** * Indicate the time that the set of media objects were seen on the archive CDN. Can be used to reconcile our local state with the server state. + * + * The write only moves forward: a resumed crawl can carry a version older than what later backups have committed, and 0 doubles as the "never confirmed" + * default, so a stale write would make confirmed rows look like they had never been seen. */ fun markSeenOnRemote(mediaIdBatch: Collection, snapshotVersion: Long) { if (mediaIdBatch.isEmpty()) { @@ -320,7 +334,7 @@ class BackupMediaSnapshotTable(context: Context, database: SignalDatabase) : Dat writableDatabase .update(TABLE_NAME) .values(LAST_SEEN_ON_REMOTE_SNAPSHOT_VERSION to snapshotVersion) - .where(query.where, query.whereArgs) + .where("${query.where} AND $LAST_SEEN_ON_REMOTE_SNAPSHOT_VERSION < $snapshotVersion", query.whereArgs) .run() } @@ -374,6 +388,31 @@ class BackupMediaSnapshotTable(context: Context, database: SignalDatabase) : Dat val cdn: Int ) + class ExistingMediaEntry( + val id: Long, + val mediaEntry: MediaEntry + ) { + val mediaId: String + get() = mediaEntry.mediaId + val cdn: Int? + get() = mediaEntry.cdn + val plaintextHash: ByteArray + get() = mediaEntry.plaintextHash + val remoteKey: ByteArray + get() = mediaEntry.remoteKey + val isThumbnail: Boolean + get() = mediaEntry.isThumbnail + + companion object { + fun fromCursor(cursor: Cursor): ExistingMediaEntry { + return ExistingMediaEntry( + id = cursor.requireLong(ID), + mediaEntry = MediaEntry.fromCursor(cursor) + ) + } + } + } + class MediaEntry( val mediaId: String, val cdn: Int?, diff --git a/app/src/main/java/org/thoughtcrime/securesms/jobs/ArchiveAttachmentReconciliationJob.kt b/app/src/main/java/org/thoughtcrime/securesms/jobs/ArchiveAttachmentReconciliationJob.kt index 3780423e32..24424d7ba8 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/jobs/ArchiveAttachmentReconciliationJob.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/jobs/ArchiveAttachmentReconciliationJob.kt @@ -27,6 +27,7 @@ import org.signal.network.service.ArchiveError import org.thoughtcrime.securesms.R import org.thoughtcrime.securesms.backup.v2.ArchivedMediaObject import org.thoughtcrime.securesms.database.AttachmentTable +import org.thoughtcrime.securesms.database.AttachmentTable.ArchiveTransferStateResetResult import org.thoughtcrime.securesms.database.BackupMediaSnapshotTable import org.thoughtcrime.securesms.database.SignalDatabase import org.thoughtcrime.securesms.dependencies.AppDependencies @@ -40,8 +41,10 @@ import org.thoughtcrime.securesms.notifications.NotificationChannels import org.thoughtcrime.securesms.notifications.NotificationIds import org.thoughtcrime.securesms.util.RemoteConfig import org.thoughtcrime.securesms.wallpaper.WallpaperStorage +import kotlin.time.Duration import kotlin.time.Duration.Companion.days import kotlin.time.Duration.Companion.hours +import kotlin.time.Duration.Companion.milliseconds /** * We do our best to keep our local attachments in sync with the archive CDN, but we still want to have a backstop that periodically @@ -84,6 +87,21 @@ class ArchiveAttachmentReconciliationJob private constructor( } } + /** + * Rate-limited because the caller runs after every backup. + */ + fun enqueueToVerifyMediaBeforeOffloading(minTimeBetweenAttempts: Duration) { + val sinceLastAttempt = (System.currentTimeMillis() - SignalStore.backup.lastForcedReconciliationAttemptTime).milliseconds + + if (sinceLastAttempt > Duration.ZERO && sinceLastAttempt < minTimeBetweenAttempts) { + Log.i(TAG, "Already forced a reconciliation $sinceLastAttempt ago. Waiting before forcing another.") + return + } + + Log.i(TAG, "Forcing a reconciliation so we can verify our media against the archive CDN.", true) + AppDependencies.jobManager.add(ArchiveAttachmentReconciliationJob(forced = true)) + } + /** * Reconcile-first entry point for after a local restore. Runs a [BackupMessagesJob] (to capture a snapshot) chained into a forced reconciliation. * Sets [BackupValues.localRestoreReconcilePending] so the backup holds off on the bulk attachment backfill, letting reconciliation run first. Reconciliation @@ -152,6 +170,12 @@ class ArchiveAttachmentReconciliationJob private constructor( // we use to determine which attachments need to be re-uploaded will possibly result in us unnecessarily re-uploading attachments. snapshotVersion = snapshotVersion ?: SignalDatabase.backupMediaSnapshots.getCurrentSnapshotVersion() + // Only spend the forced-attempt budget on a crawl that could actually open the offload gate. Without a snapshot it can't, and the rate limit would then keep + // us from retrying once a backup has built one. + if (forced && snapshotVersion!! > 0) { + SignalStore.backup.lastForcedReconciliationAttemptTime = System.currentTimeMillis() + } + syncDataFromCdn(snapshotVersion!!)?.let { return it } clearPendingLocalRestoreReconcile() @@ -239,6 +263,7 @@ class ArchiveAttachmentReconciliationJob private constructor( var newBackupJobRequired = false var bookkeepingErrorCount = 0 + var unrecoverableCount = 0 var fullSizeMismatchFound = false var thumbnailMismatchFound = false @@ -251,26 +276,37 @@ class ArchiveAttachmentReconciliationJob private constructor( } val mediaIdLog = if (internalUser) "[${MediaId(entry.mediaId)}]" else "" + val logPrefix = if (entry.isThumbnail) "[Thumbnail]$mediaIdLog" else "[Fullsize]$mediaIdLog" - if (entry.isThumbnail) { - thumbnailMismatchFound = true - val wasReset = SignalDatabase.attachments.resetArchiveThumbnailTransferStateByPlaintextHashAndRemoteKeyIfNecessary(entry.plaintextHash, entry.remoteKey) - if (wasReset) { - Log.w(TAG, "[Thumbnail]$mediaIdLog Reset transfer state by hash/key.", true) - newBackupJobRequired = true - bookkeepingErrorCount++ - } else { - Log.i(TAG, "[Thumbnail]$mediaIdLog Did not need to reset the transfer state by hash/key because the thumbnail either no longer exists or the upload is already in-progress.", true) - } + val resetResult = if (entry.isThumbnail) { + SignalDatabase.attachments.resetArchiveThumbnailTransferStateByPlaintextHashAndRemoteKeyIfNecessary(entry.plaintextHash, entry.remoteKey) } else { - fullSizeMismatchFound = true - val wasReset = SignalDatabase.attachments.resetArchiveTransferStateByPlaintextHashAndRemoteKeyIfNecessary(entry.plaintextHash, entry.remoteKey) - if (wasReset) { - Log.w(TAG, "[Fullsize]$mediaIdLog Reset transfer state by hash/key.", true) + SignalDatabase.attachments.resetArchiveTransferStateByPlaintextHashAndRemoteKeyIfNecessary(entry.plaintextHash, entry.remoteKey) + } + + when (resetResult) { + ArchiveTransferStateResetResult.RESET -> { + Log.w(TAG, "$logPrefix Reset transfer state by hash/key.", true) newBackupJobRequired = true bookkeepingErrorCount++ - } else { - Log.i(TAG, "[Fullsize]$mediaIdLog Did not need to reset the transfer state by hash/key because the attachment either no longer exists or the upload is already in-progress.", true) + } + + ArchiveTransferStateResetResult.SKIPPED_NO_LOCAL_DATA -> { + if (internalUser) { + Log.w(TAG, "$logPrefix Missing from the CDN with no local data to re-upload. Leaving the archive state alone so we keep the locator.", true) + } + unrecoverableCount++ + } + + ArchiveTransferStateResetResult.NOT_NEEDED -> { + Log.i(TAG, "$logPrefix Did not need to reset the transfer state by hash/key because it either no longer exists or the upload is already in-progress.", true) + + // Deliberately not set for SKIPPED_NO_LOCAL_DATA, since the precautionary backfills these drive could never upload media that has no local bytes. + if (entry.isThumbnail) { + thumbnailMismatchFound = true + } else { + fullSizeMismatchFound = true + } } } } @@ -282,6 +318,10 @@ class ArchiveAttachmentReconciliationJob private constructor( Log.i(TAG, "None of the $mayNeedReUploadCount CDN mismatches were bookkeeping errors.", true) } + if (unrecoverableCount > 0) { + Log.w(TAG, "Found that $unrecoverableCount/$mayNeedReUploadCount of the CDN mismatches have no local data to re-upload. That media is not recoverable from this device.", true) + } + Log.d(TAG, "AFTER:\n" + SignalDatabase.attachments.debugGetAttachmentStats().shortPrettyString(), true) stopwatch.split("stats-after") @@ -320,8 +360,24 @@ class ArchiveAttachmentReconciliationJob private constructor( Log.d(TAG, "No attachments need to be repaired.", true) } + if (snapshotVersion > 0) { + val prunedCount = SignalDatabase.backupMediaSnapshots.deleteOldMediaObjectsNeverSeenOnCdn(snapshotVersion) + if (prunedCount > 0) { + Log.i(TAG, "Pruned $prunedCount snapshot entries that left the latest snapshot and have never been seen on the CDN.", true) + } + } + stopwatch.split("prune-absent") + + val completionTime = System.currentTimeMillis() + SignalStore.backup.remoteStorageGarbageCollectionPending = false - SignalStore.backup.lastAttachmentReconciliationTime = System.currentTimeMillis() + SignalStore.backup.lastAttachmentReconciliationTime = completionTime + + // A crawl with no snapshot to compare against verified nothing, so it must not satisfy the gate that lets us delete local copies of media. + if (snapshotVersion > 0) { + SignalStore.backup.lastCompletedReconciliationSnapshotVersion = snapshotVersion + SignalStore.backup.lastCompletedReconciliationTime = completionTime + } stopwatch.stop(TAG) @@ -369,6 +425,13 @@ class ArchiveAttachmentReconciliationJob private constructor( } } + // TODO [cody] Fix perf problems of calling setArchiveThumbnailFinishedForMatchingMediaObjects + // Takes the whole listing, since a restored device has no thumbnail snapshot rows yet +// val thumbnailsMarkedFinished = SignalDatabase.attachments.setArchiveThumbnailFinishedForMatchingMediaObjects(mediaObjects.toSet()) +// if (thumbnailsMarkedFinished > 0) { +// Log.i(TAG, "Marked $thumbnailsMarkedFinished thumbnail group(s) as finished after finding them on the CDN.", true) +// } + return mediaOnRemoteButNotLocal } @@ -422,8 +485,8 @@ class ArchiveAttachmentReconciliationJob private constructor( } val stopwatch = Stopwatch("remote-delete") - val validatedDeletes: MutableSet = SignalDatabase.attachments.getMediaObjectsThatCantBeFound(deletes).toMutableSet() - Log.d(TAG, "Found that ${validatedDeletes.size}/${deletes.size} requested remote deletes have no data at all locally, and are therefore safe to delete.", true) + val validatedDeletes: MutableSet = SignalDatabase.attachments.getMediaObjectsThatCantBeFound(deletes, SignalStore.backup.lastUsedMessageCutoffTime).toMutableSet() + Log.d(TAG, "Found that ${validatedDeletes.size}/${deletes.size} requested remote deletes are no longer referenced by any attachment, and are therefore safe to delete.", true) stopwatch.split("validate") // Fix archive state for attachments that are found locally but weren't in the latest snapshot. @@ -474,6 +537,9 @@ class ArchiveAttachmentReconciliationJob private constructor( } stopwatch.split("network") + // Any snapshot row left behind here would keep claiming an object we just removed, and reviving that row would present it as still CDN-confirmed. + SignalDatabase.backupMediaSnapshots.deleteOldMediaObjects(validatedDeletes.map { it.mediaId }) + stopwatch.stop(TAG) return null diff --git a/app/src/main/java/org/thoughtcrime/securesms/jobs/ArchiveCommitAttachmentDeletesJob.kt b/app/src/main/java/org/thoughtcrime/securesms/jobs/ArchiveCommitAttachmentDeletesJob.kt index 6dd2041557..b7c47b4fb2 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/jobs/ArchiveCommitAttachmentDeletesJob.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/jobs/ArchiveCommitAttachmentDeletesJob.kt @@ -5,6 +5,7 @@ package org.thoughtcrime.securesms.jobs +import androidx.annotation.VisibleForTesting import arrow.core.Either import org.signal.core.models.backup.MediaId import org.signal.core.util.Base64 @@ -12,6 +13,7 @@ import org.signal.core.util.logging.Log import org.signal.network.service.ArchiveError import org.thoughtcrime.securesms.attachments.Cdn import org.thoughtcrime.securesms.backup.v2.ArchivedMediaObject +import org.thoughtcrime.securesms.database.AttachmentTable import org.thoughtcrime.securesms.database.BackupMediaSnapshotTable import org.thoughtcrime.securesms.database.SignalDatabase import org.thoughtcrime.securesms.dependencies.AppDependencies @@ -19,7 +21,6 @@ import org.thoughtcrime.securesms.jobmanager.CoroutineJob import org.thoughtcrime.securesms.jobmanager.Job import org.thoughtcrime.securesms.keyvalue.SignalStore import org.thoughtcrime.securesms.util.RemoteConfig -import java.lang.RuntimeException import kotlin.time.Duration.Companion.days import kotlin.time.Duration.Companion.hours @@ -36,7 +37,8 @@ class ArchiveCommitAttachmentDeletesJob private constructor(parameters: Paramete const val KEY = "ArchiveCommitAttachmentDeletesJob" const val ARCHIVE_ATTACHMENT_QUEUE = "ArchiveAttachmentQueue" - private const val REMOTE_DELETE_BATCH_SIZE = 1_000 + @VisibleForTesting + internal const val REMOTE_DELETE_BATCH_SIZE = 1_000 /** * Deletes the provided attachments from the CDN. @@ -116,18 +118,61 @@ class ArchiveCommitAttachmentDeletesJob private constructor(parameters: Paramete return Result.success() } - var mediaObjects = SignalDatabase.backupMediaSnapshots.getPageOfOldMediaObjects(REMOTE_DELETE_BATCH_SIZE) + // Read once so every page judges against the same threshold, even if a backup finishes while we're paging. + val messageInclusionCutoffTime = SignalStore.backup.lastUsedMessageCutoffTime - while (mediaObjects.isNotEmpty()) { + var retainedCount = 0 + var unknownCdnCount = 0 + var lastId = 0L + var page = SignalDatabase.backupMediaSnapshots.getPageOfOldMediaEntries(pageSize = REMOTE_DELETE_BATCH_SIZE, afterId = lastId) + + while (page.isNotEmpty()) { if (isCanceled) { Log.w(TAG, "Job cancelled while processing media objects for deletion.") return Result.failure() } - deleteMediaObjectsFromCdn(TAG, mediaObjects, this::defaultBackoff, this::isCanceled)?.let { result -> return result } - SignalDatabase.backupMediaSnapshots.deleteOldMediaObjects(mediaObjects) + lastId = page.last().id - mediaObjects = SignalDatabase.backupMediaSnapshots.getPageOfOldMediaObjects(REMOTE_DELETE_BATCH_SIZE) + // Full-size and thumbnail rows share a mediaName, so they have to be judged separately. A surviving attachment keeps its full-size object alive but says + // nothing about whether that attachment still wants a thumbnail archived. + val (thumbnailPage, fullSizePage) = page.partition { it.isThumbnail } + + val fullSizeByMediaName = fullSizePage.groupBy { it.mediaNameParts() } + val unreferencedFullSizeNames = SignalDatabase.attachments.getMediaNamesWithNoAttachment(fullSizeByMediaName.keys, messageInclusionCutoffTime) + val unreferencedFullSize = fullSizeByMediaName.filterKeys { it in unreferencedFullSizeNames }.values.flatten() + + val thumbnailsByMediaName = thumbnailPage.groupBy { it.mediaNameParts() } + val unreferencedThumbnailNames = SignalDatabase.attachments.getMediaNamesWithNoEligibleThumbnail(thumbnailsByMediaName.keys, messageInclusionCutoffTime) + val unreferencedThumbnails = thumbnailsByMediaName.filterKeys { it in unreferencedThumbnailNames }.values.flatten() + + val unreferencedEntries = unreferencedFullSize + unreferencedThumbnails + + val safeToDelete = unreferencedEntries.mapNotNull { entry -> entry.cdn?.let { ArchivedMediaObject(mediaId = entry.mediaId, cdn = it) } }.toSet() + + retainedCount += page.size - unreferencedEntries.size + unknownCdnCount += unreferencedEntries.size - safeToDelete.size + + if (safeToDelete.isNotEmpty()) { + deleteMediaObjectsFromCdn( + tag = TAG, + attachmentsToDelete = safeToDelete, + backoffGenerator = this::defaultBackoff, + cancellationSignal = this::isCanceled + )?.let { result -> return result } + + SignalDatabase.backupMediaSnapshots.deleteOldMediaObjects(safeToDelete.map { it.mediaId }) + } + + page = SignalDatabase.backupMediaSnapshots.getPageOfOldMediaEntries(pageSize = REMOTE_DELETE_BATCH_SIZE, afterId = lastId) + } + + if (retainedCount > 0) { + Log.w(TAG, "Retained $retainedCount media objects that dropped out of the latest snapshot but are still referenced by an attachment. They stay tracked and will be reconsidered after the next backup.", true) + } + + if (unknownCdnCount > 0) { + Log.w(TAG, "Retained $unknownCdnCount unreferenced media objects that have no recorded CDN, will get resolved during reconciliation rather than here.", true) } return Result.success() @@ -135,6 +180,13 @@ class ArchiveCommitAttachmentDeletesJob private constructor(parameters: Paramete override fun onFailure() = Unit + private fun BackupMediaSnapshotTable.ExistingMediaEntry.mediaNameParts(): AttachmentTable.MediaNameParts { + return AttachmentTable.MediaNameParts( + plaintextHash = Base64.encodeWithPadding(plaintextHash), + remoteKey = Base64.encodeWithPadding(remoteKey) + ) + } + class Factory : Job.Factory { override fun create(parameters: Parameters, serializedData: ByteArray?): ArchiveCommitAttachmentDeletesJob { return ArchiveCommitAttachmentDeletesJob(parameters) diff --git a/app/src/main/java/org/thoughtcrime/securesms/jobs/ArchiveThumbnailUploadJob.kt b/app/src/main/java/org/thoughtcrime/securesms/jobs/ArchiveThumbnailUploadJob.kt index 2e99031e65..6ae868768b 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/jobs/ArchiveThumbnailUploadJob.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/jobs/ArchiveThumbnailUploadJob.kt @@ -157,13 +157,36 @@ class ArchiveThumbnailUploadJob private constructor( return Result.success() } - // TODO [backups] Determine if we actually need to upload or are reusing a thumbnail from another attachment + val alreadyArchived = SignalDatabase.attachments.getArchiveThumbnailTransferState(attachmentId) == AttachmentTable.ArchiveTransferState.FINISHED + + // Attachments sharing a mediaName are enqueued individually, but finalizing fills in the whole group, so by now a sibling may have done the work for us. + if (alreadyArchived && SignalDatabase.attachments.hasThumbnailFile(attachmentId)) { + Log.i(TAG, "[$attachmentId] Thumbnail is already archived and present locally. Nothing to do.") + return Result.success() + } val thumbnailResult = generateThumbnailIfPossible(attachment) if (thumbnailResult == null) { Log.w(TAG, "Unable to generate a thumbnail result for $attachmentId") ArchiveDatabaseExecutor.runBlocking { - SignalDatabase.attachments.setArchiveThumbnailTransferState(attachmentId, AttachmentTable.ArchiveTransferState.PERMANENT_FAILURE) + if (alreadyArchived) { + SignalDatabase.attachments.markThumbnailPermanentlyFailedIfUnrestorable(attachmentId) + } else { + SignalDatabase.attachments.setArchiveThumbnailTransferState(attachmentId, AttachmentTable.ArchiveTransferState.PERMANENT_FAILURE) + } + } + return Result.success() + } + + if (alreadyArchived) { + Log.i(TAG, "[$attachmentId] Thumbnail is already on the archive CDN. Saving the generated copy locally rather than uploading it again.") + ArchiveDatabaseExecutor.runBlocking { + SignalDatabase.attachments.finalizeAttachmentThumbnailAfterUpload( + attachmentId = attachmentId, + attachmentPlaintextHash = attachment.dataHash, + attachmentRemoteKey = attachment.remoteKey, + data = thumbnailResult.data + ) } return Result.success() } @@ -306,6 +329,11 @@ class ArchiveThumbnailUploadJob private constructor( } override fun onFailure() { + if (SignalDatabase.attachments.getArchiveThumbnailTransferState(attachmentId) == AttachmentTable.ArchiveTransferState.FINISHED) { + Log.w(TAG, "[$attachmentId] Job didn't finish, but the thumbnail is already archived. Leaving the transfer state alone so we don't re-upload it.") + return + } + if (this.isCanceled) { Log.w(TAG, "[$attachmentId] Job was canceled, updating archive thumbnail transfer state to ${AttachmentTable.ArchiveTransferState.NONE}.") ArchiveDatabaseExecutor.runBlocking { diff --git a/app/src/main/java/org/thoughtcrime/securesms/jobs/BackupRestoreMediaJob.kt b/app/src/main/java/org/thoughtcrime/securesms/jobs/BackupRestoreMediaJob.kt index a4d96cb105..89e9d38742 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/jobs/BackupRestoreMediaJob.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/jobs/BackupRestoreMediaJob.kt @@ -5,6 +5,7 @@ package org.thoughtcrime.securesms.jobs +import androidx.annotation.VisibleForTesting import org.signal.core.models.database.AttachmentId import org.signal.core.util.logging.Log import org.signal.core.util.withinTransaction @@ -57,9 +58,23 @@ class BackupRestoreMediaJob private constructor(parameters: Parameters) : BaseJo throw NotPushRegisteredException() } + enqueueRestoreJobs(restoreTime = System.currentTimeMillis()) + + BackupMediaRestoreService.start(context, context.getString(R.string.BackupStatus__restoring_media)) + ArchiveRestoreProgress.onRestoringMedia() + + RestoreAttachmentJob.Queues.INITIAL_RESTORE.forEach { queue -> + AppDependencies.jobManager.add(CheckRestoreMediaLeftJob(queue)) + } + } + + /** + * Walks every restorable attachment, enqueuing a full-size or thumbnail restore for each and moving it out of [AttachmentTable.TRANSFER_NEEDS_RESTORE] so the + * next batch makes progress. + */ + @VisibleForTesting + internal fun enqueueRestoreJobs(restoreTime: Long, batchSize: Int = 500) { val jobManager = AppDependencies.jobManager - val batchSize = 500 - val restoreTime = System.currentTimeMillis() val orphanedCount = SignalDatabase.attachments.markRestorableAttachmentsWithoutMessageAsFailed() if (orphanedCount > 0) { @@ -71,6 +86,8 @@ class BackupRestoreMediaJob private constructor(parameters: Parameters) : BaseJo Log.w(TAG, "$stalledCount attachments were stuck mid-restore; reset to needs-restore so they can be re-enqueued") } + var batchWasEmpty: Boolean + do { val restoreThumbnailJobs: MutableList = mutableListOf() val restoreFullAttachmentJobs: MutableList = mutableListOf() @@ -88,6 +105,7 @@ class BackupRestoreMediaJob private constructor(parameters: Parameters) : BaseJo } val attachmentBatch = last30DaysAttachments + remaining + batchWasEmpty = attachmentBatch.isEmpty() val messageIds = attachmentBatch.map { it.mmsId }.toSet() val messageMap = SignalDatabase.messages.getMessages(messageIds).associate { it.id to (it as MmsMessageRecord) } @@ -109,11 +127,13 @@ class BackupRestoreMediaJob private constructor(parameters: Parameters) : BaseJo queueHash = attachment.plaintextHash?.contentHashCode() ?: attachment.remoteKey?.contentHashCode() ) } else { - restoreThumbnailJobs += RestoreAttachmentThumbnailJob( - messageId = attachment.mmsId, - attachmentId = attachment.attachmentId, - highPriority = false - ) + if (attachment.couldHaveArchivedThumbnail) { + restoreThumbnailJobs += RestoreAttachmentThumbnailJob( + messageId = attachment.mmsId, + attachmentId = attachment.attachmentId, + highPriority = false + ) + } restoreThumbnailOnlyAttachmentsIds += attachment.attachmentId } @@ -133,14 +153,7 @@ class BackupRestoreMediaJob private constructor(parameters: Parameters) : BaseJo // Intentionally enqueues one at a time for safer attachment transfer state management restoreThumbnailJobs.forEach { jobManager.add(it) } restoreFullAttachmentJobs.forEach { jobManager.add(it) } - } while (restoreThumbnailJobs.isNotEmpty() || restoreFullAttachmentJobs.isNotEmpty() || notRestorable.isNotEmpty()) - - BackupMediaRestoreService.start(context, context.getString(R.string.BackupStatus__restoring_media)) - ArchiveRestoreProgress.onRestoringMedia() - - RestoreAttachmentJob.Queues.INITIAL_RESTORE.forEach { queue -> - jobManager.add(CheckRestoreMediaLeftJob(queue)) - } + } while (!batchWasEmpty) } private fun shouldRestoreFullSize(message: MmsMessageRecord, restoreTime: Long, optimizeStorage: Boolean): Boolean { diff --git a/app/src/main/java/org/thoughtcrime/securesms/jobs/OptimizeMediaJob.kt b/app/src/main/java/org/thoughtcrime/securesms/jobs/OptimizeMediaJob.kt index 5fdb58c364..95a0a4cc32 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/jobs/OptimizeMediaJob.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/jobs/OptimizeMediaJob.kt @@ -5,6 +5,7 @@ package org.thoughtcrime.securesms.jobs +import androidx.annotation.VisibleForTesting import org.signal.core.util.DiskUtil import org.signal.core.util.logging.Log import org.thoughtcrime.securesms.backup.v2.ArchiveRestoreProgress @@ -12,7 +13,10 @@ import org.thoughtcrime.securesms.database.SignalDatabase import org.thoughtcrime.securesms.dependencies.AppDependencies import org.thoughtcrime.securesms.jobmanager.Job import org.thoughtcrime.securesms.keyvalue.SignalStore +import org.thoughtcrime.securesms.util.RemoteConfig +import kotlin.time.Duration import kotlin.time.Duration.Companion.days +import kotlin.time.Duration.Companion.milliseconds /** * Optimizes media storage by relying on backups for full copies of files and only keeping thumbnails locally. @@ -23,6 +27,11 @@ class OptimizeMediaJob private constructor(parameters: Parameters) : Job(paramet private val TAG = Log.tag(OptimizeMediaJob::class) const val KEY = "OptimizeMediaJob" + private const val LOW_STORAGE_THRESHOLD_PERCENT = 5f + + /** How many reconciliation intervals a completed crawl stays trustworthy for. */ + private const val EVIDENCE_AGE_INTERVAL_MULTIPLIER = 4 + fun enqueue() { if (!SignalStore.backup.optimizeStorage || !SignalStore.backup.backsUpMedia) { Log.i(TAG, "Optimize media is not enabled, skipping. backsUpMedia: ${SignalStore.backup.backsUpMedia} optimizeStorage: ${SignalStore.backup.optimizeStorage}") @@ -61,18 +70,16 @@ class OptimizeMediaJob private constructor(parameters: Parameters) : Job(paramet RestoreAttachmentJob.Queues.OFFLOAD_RESTORE.forEach { queue -> AppDependencies.jobManager.add(CheckRestoreMediaLeftJob(queue)) } } - Log.i(TAG, "Optimizing media in the db") - val available = DiskUtil.getAvailableSpace(context).bytes.toFloat() val total = DiskUtil.getTotalDiskSize(context).bytes.toFloat() - val remaining = (total - available) / total * 100 + val percentAvailable = if (total > 0f) available / total * 100 else 100f + val minimumAge = if (percentAvailable > LOW_STORAGE_THRESHOLD_PERCENT) 30.days else 15.days - val minimumAge = if (remaining > 5f) 30.days else 15.days + Log.i(TAG, "${"%.1f".format(percentAvailable)}% storage available") - Log.i(TAG, "${"%.1f".format(remaining)}% storage available, optimizing attachments older than $minimumAge") - - SignalDatabase.attachments.markEligibleAttachmentsAsOptimized(minimumAge) + offloadVerifiedMedia(minimumAge, RemoteConfig.archiveReconciliationSyncInterval) + // Reclaims files nothing references anymore, which doesn't depend on CDN confirmation, so it has to run even when offloading is gated off. Log.i(TAG, "Deleting abandoned attachment files") val count = SignalDatabase.attachments.deleteAbandonedAttachmentFiles() Log.i(TAG, "Deleted $count attachments") @@ -80,6 +87,39 @@ class OptimizeMediaJob private constructor(parameters: Parameters) : Job(paramet return Result.success() } + /** Offloads the local copy of media the archive CDN confirmed during a completed crawl, and nothing else. */ + @VisibleForTesting + internal fun offloadVerifiedMedia(minimumAge: Duration, reconciliationInterval: Duration) { + val crawlInterval = reconciliationInterval.coerceAtLeast(1.days) + val lastCompletedCrawlVersion = SignalStore.backup.lastCompletedReconciliationSnapshotVersion + + if (lastCompletedCrawlVersion < 0) { + Log.w(TAG, "No archive reconciliation has completed on this device yet. Not offloading anything until our media has been verified against the archive CDN.", true) + ArchiveAttachmentReconciliationJob.enqueueToVerifyMediaBeforeOffloading(crawlInterval) + return + } + + val maxEvidenceAge = crawlInterval * EVIDENCE_AGE_INTERVAL_MULTIPLIER + val evidenceAge = (System.currentTimeMillis() - SignalStore.backup.lastCompletedReconciliationTime).milliseconds + + // Refusing alone isn't enough here: a far-future timestamp keeps reading as untrustworthy until the clock catches up to it, which could be years. + if (evidenceAge.isNegative()) { + Log.w(TAG, "The last completed archive reconciliation is timestamped ${-evidenceAge} in the future, likely from a clock change. Discarding our verification state so a fresh crawl has to confirm our media again.", true) + SignalStore.backup.clearArchiveVerificationState() + ArchiveAttachmentReconciliationJob.enqueueToVerifyMediaBeforeOffloading(crawlInterval) + return + } + + if (evidenceAge > maxEvidenceAge) { + Log.w(TAG, "The last completed archive reconciliation is $evidenceAge old, past the $maxEvidenceAge we trust. Not offloading anything until our media has been verified again.", true) + ArchiveAttachmentReconciliationJob.enqueueToVerifyMediaBeforeOffloading(crawlInterval) + return + } + + Log.i(TAG, "Optimizing attachments older than $minimumAge") + SignalDatabase.attachments.markEligibleAttachmentsAsOptimized(lastCompletedCrawlVersion, minimumAge) + } + override fun serialize(): ByteArray? = null override fun getFactoryKey(): String = KEY override fun onFailure() = Unit diff --git a/app/src/main/java/org/thoughtcrime/securesms/keyvalue/BackupValues.kt b/app/src/main/java/org/thoughtcrime/securesms/keyvalue/BackupValues.kt index 93cecaf440..0a7b6835b2 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/keyvalue/BackupValues.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/keyvalue/BackupValues.kt @@ -52,6 +52,9 @@ class BackupValues(store: KeyValueStore) : SignalStoreValues(store) { private const val KEY_NEXT_BACKUP_TIME = "backup.nextBackupTime" private const val KEY_LAST_BACKUP_TIME = "backup.lastBackupTime" private const val KEY_LAST_ATTACHMENT_RECONCILIATION_TIME = "backup.lastBackupMediaSyncTime" + private const val KEY_LAST_COMPLETED_RECONCILIATION_SNAPSHOT_VERSION = "backup.lastCompletedReconciliationSnapshotVersion" + private const val KEY_LAST_COMPLETED_RECONCILIATION_TIME = "backup.lastCompletedReconciliationTime" + private const val KEY_LAST_FORCED_RECONCILIATION_ATTEMPT_TIME = "backup.lastForcedReconciliationAttemptTime" private const val KEY_TOTAL_RESTORABLE_ATTACHMENT_SIZE = "backup.totalRestorableAttachmentSize" private const val KEY_LAST_BACKUP_PROTO_VERSION = "backup.lastBackupProtoVersion" @@ -193,6 +196,33 @@ class BackupValues(store: KeyValueStore) : SignalStoreValues(store) { var lastAttachmentReconciliationTime: Long by longValue(KEY_LAST_ATTACHMENT_RECONCILIATION_TIME, -1) + /** + * Negative if no crawl has ever completed on this device. Must be cleared by anything that invalidates our view of the CDN (new media root backup key, tier + * change), because it gates deleting local copies of media. + */ + var lastCompletedReconciliationSnapshotVersion: Long by longValue(KEY_LAST_COMPLETED_RECONCILIATION_SNAPSHOT_VERSION, -1) + + /** + * When the crawl behind [lastCompletedReconciliationSnapshotVersion] finished, or zero if none ever has. Written alongside it. + */ + var lastCompletedReconciliationTime: Long by longValue(KEY_LAST_COMPLETED_RECONCILIATION_TIME, 0) + + /** + * Advances when a forced crawl *starts*, unlike [lastAttachmentReconciliationTime] which only advances on completion, so that a crawl which can never finish + * doesn't get re-forced daily while one that never started still can be. + */ + var lastForcedReconciliationAttemptTime: Long by longValue(KEY_LAST_FORCED_RECONCILIATION_ATTEMPT_TIME, 0) + + /** + * Discards everything we know about having verified our media against the archive CDN, so offloading stays gated until a fresh crawl confirms it again. Call + * this from anything that invalidates that view. + */ + fun clearArchiveVerificationState() { + lastCompletedReconciliationSnapshotVersion = -1 + lastCompletedReconciliationTime = 0 + lastForcedReconciliationAttemptTime = 0 + } + var userManuallySkippedMediaRestore: Boolean by booleanValue(KEY_USER_MANUALLY_SKIPPED_MEDIA_RESTORE, false) /** @@ -258,6 +288,8 @@ class BackupValues(store: KeyValueStore) : SignalStoreValues(store) { store.beginWrite().putBlob(KEY_MEDIA_ROOT_BACKUP_KEY, value.value).commit() mediaCredentials.clearAll() cachedMediaCdnPath = null + + clearArchiveVerificationState() } } @@ -299,6 +331,8 @@ class BackupValues(store: KeyValueStore) : SignalStoreValues(store) { clearMessageBackupFailureSheetWatermark() backupCreationError = null + clearArchiveVerificationState() + if (storedValue == null) { Log.i(TAG, "Enabling backups. Resetting 'finished initial backup' state.") } else if (value == MessageBackupTier.PAID) { diff --git a/app/src/main/java/org/thoughtcrime/securesms/logsubmit/LogSectionRemoteBackups.kt b/app/src/main/java/org/thoughtcrime/securesms/logsubmit/LogSectionRemoteBackups.kt index fd8b53cad8..ecc295c0ba 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/logsubmit/LogSectionRemoteBackups.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/logsubmit/LogSectionRemoteBackups.kt @@ -46,6 +46,9 @@ class LogSectionRemoteBackups : LogSection { output.append("Detected subscription state mismatch: ${SignalStore.backup.subscriptionStateMismatchDetected}\n") output.append("Last verified key time : ${SignalStore.backup.lastVerifyKeyTime}\n") output.append("Local restore reconcile pending : ${SignalStore.backup.localRestoreReconcilePending}\n") + output.append("Last CDN-verified snapshot version : ${SignalStore.backup.lastCompletedReconciliationSnapshotVersion}\n") + output.append("Last CDN-verified time : ${SignalStore.backup.lastCompletedReconciliationTime}\n") + output.append("Last forced reconciliation attempt : ${SignalStore.backup.lastForcedReconciliationAttemptTime}\n") output.append("Restore state : ${ArchiveRestoreProgress.state}\n") output.append("\n -- Subscription State\n") diff --git a/app/src/test/java/org/thoughtcrime/securesms/database/AttachmentTableTest_localRestoreArchiveState.kt b/app/src/test/java/org/thoughtcrime/securesms/database/AttachmentTableTest_localRestoreArchiveState.kt index 3fd20292b0..56d7a1a776 100644 --- a/app/src/test/java/org/thoughtcrime/securesms/database/AttachmentTableTest_localRestoreArchiveState.kt +++ b/app/src/test/java/org/thoughtcrime/securesms/database/AttachmentTableTest_localRestoreArchiveState.kt @@ -18,6 +18,7 @@ import org.junit.runner.RunWith import org.robolectric.RobolectricTestRunner import org.robolectric.annotation.Config import org.signal.core.models.database.AttachmentId +import org.signal.core.util.Base64 import org.signal.core.util.logging.Log import org.thoughtcrime.securesms.attachments.ArchivedAttachment import org.thoughtcrime.securesms.attachments.Attachment @@ -90,6 +91,36 @@ class AttachmentTableTest_localRestoreArchiveState { assertThat(SignalDatabase.attachments.hasArchiveFinishedLocalBackupMedia()).isFalse() } + @Test + fun resetArchiveTransferState_skippedWhenThereIsNoLocalDataFileToReUploadFrom() { + val attachmentId = insertArchivedAttachment(archiveCdn = 3, localBackupKey = Random.nextBytes(32)) + val attachment = SignalDatabase.attachments.getAttachment(attachmentId)!! + + val result = SignalDatabase.attachments.resetArchiveTransferStateByPlaintextHashAndRemoteKeyIfNecessary( + plaintextHash = Base64.decode(attachment.dataHash!!), + remoteKey = Base64.decode(attachment.remoteKey!!) + ) + + assertThat(result).isEqualTo(AttachmentTable.ArchiveTransferStateResetResult.SKIPPED_NO_LOCAL_DATA) + + val after = SignalDatabase.attachments.getAttachment(attachmentId)!! + assertThat(after.archiveTransferState).isEqualTo(AttachmentTable.ArchiveTransferState.FINISHED) + assertThat(after.archiveCdn).isEqualTo(3) + } + + @Test + fun resetArchiveTransferState_notNeededWhenNotFinished() { + val attachmentId = insertArchivedAttachment(archiveCdn = null, localBackupKey = Random.nextBytes(32)) + val attachment = SignalDatabase.attachments.getAttachment(attachmentId)!! + + val result = SignalDatabase.attachments.resetArchiveTransferStateByPlaintextHashAndRemoteKeyIfNecessary( + plaintextHash = Base64.decode(attachment.dataHash!!), + remoteKey = Base64.decode(attachment.remoteKey!!) + ) + + assertThat(result).isEqualTo(AttachmentTable.ArchiveTransferStateResetResult.NOT_NEEDED) + } + private fun insertArchivedAttachment(archiveCdn: Int?, localBackupKey: ByteArray?): AttachmentId { val from = recipients.createRecipient("Some Contact") val threadId = SignalDatabase.threads.getOrCreateThreadIdFor(Recipient.resolved(from)) diff --git a/app/src/test/java/org/thoughtcrime/securesms/database/AttachmentTableTest_thumbnailSnapshotEntries.kt b/app/src/test/java/org/thoughtcrime/securesms/database/AttachmentTableTest_thumbnailSnapshotEntries.kt new file mode 100644 index 0000000000..8a42efc917 --- /dev/null +++ b/app/src/test/java/org/thoughtcrime/securesms/database/AttachmentTableTest_thumbnailSnapshotEntries.kt @@ -0,0 +1,267 @@ +/* + * Copyright 2026 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ + +package org.thoughtcrime.securesms.database + +import android.app.Application +import assertk.assertThat +import assertk.assertions.hasSize +import assertk.assertions.isEmpty +import assertk.assertions.isEqualTo +import org.junit.BeforeClass +import org.junit.Rule +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import org.robolectric.annotation.Config +import org.signal.core.models.database.AttachmentId +import org.signal.core.util.Base64 +import org.signal.core.util.logging.Log +import org.signal.core.util.update +import org.thoughtcrime.securesms.attachments.ArchivedAttachment +import org.thoughtcrime.securesms.attachments.Attachment +import org.thoughtcrime.securesms.mms.IncomingMessage +import org.thoughtcrime.securesms.recipients.Recipient +import org.thoughtcrime.securesms.testutil.RecipientTestRule +import org.thoughtcrime.securesms.testutil.SystemOutLogger +import java.util.UUID +import kotlin.random.Random + +/** + * Coverage for which thumbnail entries survive into the backup media snapshot. An entry that does not survive falls out of the snapshot and is + * therefore marked for deletion off of the archive CDN, so media with no local data file but an archived thumbnail has to be kept. + */ +@Suppress("ClassName") +@RunWith(RobolectricTestRunner::class) +@Config(manifest = Config.NONE, application = Application::class) +class AttachmentTableTest_thumbnailSnapshotEntries { + + @get:Rule val recipients = RecipientTestRule() + + companion object { + @BeforeClass + @JvmStatic + fun setUpClass() { + Log.initialize(SystemOutLogger()) + } + } + + @Test + fun givenNoLocalDataFileButAnArchivedThumbnail_whenIFilter_thenIExpectTheEntryKept() { + val attachmentId = insertArchivedAttachment() + SignalDatabase.attachments.setArchiveThumbnailTransferState(attachmentId, AttachmentTable.ArchiveTransferState.FINISHED) + + val entry = thumbnailEntryFor(attachmentId) + val kept = SignalDatabase.attachments.filterThumbnailsWithoutEligibleAttachment(setOf(entry)) + + assertThat(kept.size).isEqualTo(1) + assertThat(kept.first().mediaId).isEqualTo(entry.mediaId) + } + + @Test + fun givenNoLocalDataFileAndNoArchivedThumbnail_whenIFilter_thenIExpectTheEntryDropped() { + val attachmentId = insertArchivedAttachment() + + val kept = SignalDatabase.attachments.filterThumbnailsWithoutEligibleAttachment(setOf(thumbnailEntryFor(attachmentId))) + + assertThat(kept).isEmpty() + } + + @Test + fun givenAThumbnailRestoredFromTheCdn_whenIFilter_thenIExpectTheEntryKept() { + val attachmentId = insertArchivedAttachment() + SignalDatabase.attachments.setThumbnailRestoreState(listOf(attachmentId), AttachmentTable.ThumbnailRestoreState.FINISHED) + + val entry = thumbnailEntryFor(attachmentId) + val kept = SignalDatabase.attachments.filterThumbnailsWithoutEligibleAttachment(setOf(entry)) + + assertThat(kept.size).isEqualTo(1) + assertThat(kept.first().mediaId).isEqualTo(entry.mediaId) + } + + @Test + fun givenAPermanentlyFailedThumbnail_whenIFilter_thenIExpectTheEntryDropped() { + val attachmentId = insertArchivedAttachment() + SignalDatabase.attachments.setArchiveThumbnailTransferState(attachmentId, AttachmentTable.ArchiveTransferState.PERMANENT_FAILURE) + + val kept = SignalDatabase.attachments.filterThumbnailsWithoutEligibleAttachment(setOf(thumbnailEntryFor(attachmentId))) + + assertThat(kept).isEmpty() + } + + /** + * These four mirror the cases above against [AttachmentTable.getMediaNamesWithNoEligibleThumbnail], which answers the same question with an indexed lookup + * instead of a full scan. The two share a predicate, so a divergence here means the CDN-delete side and the snapshot-write side have drifted apart. + */ + @Test + fun givenNoLocalDataFileButAnArchivedThumbnail_whenIQueryByName_thenIExpectItStillWanted() { + val attachmentId = insertArchivedAttachment() + SignalDatabase.attachments.setArchiveThumbnailTransferState(attachmentId, AttachmentTable.ArchiveTransferState.FINISHED) + + assertThat(SignalDatabase.attachments.getMediaNamesWithNoEligibleThumbnail(setOf(mediaNameFor(attachmentId)))).isEmpty() + } + + @Test + fun givenNoLocalDataFileAndNoArchivedThumbnail_whenIQueryByName_thenIExpectItUnwanted() { + val attachmentId = insertArchivedAttachment() + + assertThat(SignalDatabase.attachments.getMediaNamesWithNoEligibleThumbnail(setOf(mediaNameFor(attachmentId)))).hasSize(1) + } + + @Test + fun givenAThumbnailRestoredFromTheCdn_whenIQueryByName_thenIExpectItStillWanted() { + val attachmentId = insertArchivedAttachment() + SignalDatabase.attachments.setThumbnailRestoreState(listOf(attachmentId), AttachmentTable.ThumbnailRestoreState.FINISHED) + + assertThat(SignalDatabase.attachments.getMediaNamesWithNoEligibleThumbnail(setOf(mediaNameFor(attachmentId)))).isEmpty() + } + + @Test + fun givenAPermanentlyFailedThumbnail_whenIQueryByName_thenIExpectItUnwanted() { + val attachmentId = insertArchivedAttachment() + SignalDatabase.attachments.setArchiveThumbnailTransferState(attachmentId, AttachmentTable.ArchiveTransferState.PERMANENT_FAILURE) + + assertThat(SignalDatabase.attachments.getMediaNamesWithNoEligibleThumbnail(setOf(mediaNameFor(attachmentId)))).hasSize(1) + } + + @Test + fun givenAMediaNameWithNoAttachmentAtAll_whenIQueryByName_thenIExpectItUnwanted() { + val orphan = AttachmentTable.MediaNameParts( + plaintextHash = Base64.encodeWithPadding(Random.nextBytes(32)), + remoteKey = Base64.encodeWithPadding(Random.nextBytes(32)) + ) + + assertThat(SignalDatabase.attachments.getMediaNamesWithNoEligibleThumbnail(setOf(orphan))).hasSize(1) + } + + /** + * These pin the clause against [org.thoughtcrime.securesms.jobs.BackupMessagesJob]'s thumbnail filters. Anything the write side refuses to put in a snapshot + * has to read as unwanted here, or we keep paying to store an object nothing will ever reference again. + */ + @Test + fun givenAWallpaper_whenIFilter_thenIExpectTheEntryDropped() { + val attachmentId = insertArchivedAttachment() + SignalDatabase.attachments.setArchiveThumbnailTransferState(attachmentId, AttachmentTable.ArchiveTransferState.FINISHED) + setColumn(attachmentId, AttachmentTable.MESSAGE_ID, AttachmentTable.WALLPAPER_MESSAGE_ID) + + assertThat(SignalDatabase.attachments.filterThumbnailsWithoutEligibleAttachment(setOf(thumbnailEntryFor(attachmentId)))).isEmpty() + } + + @Test + fun givenAWallpaper_whenIQueryByName_thenIExpectItUnwanted() { + val attachmentId = insertArchivedAttachment() + SignalDatabase.attachments.setArchiveThumbnailTransferState(attachmentId, AttachmentTable.ArchiveTransferState.FINISHED) + setColumn(attachmentId, AttachmentTable.MESSAGE_ID, AttachmentTable.WALLPAPER_MESSAGE_ID) + + assertThat(SignalDatabase.attachments.getMediaNamesWithNoEligibleThumbnail(setOf(mediaNameFor(attachmentId)))).hasSize(1) + } + + @Test + fun givenNonVisualMedia_whenIQueryByName_thenIExpectItUnwanted() { + val attachmentId = insertArchivedAttachment(contentType = "application/pdf") + SignalDatabase.attachments.setArchiveThumbnailTransferState(attachmentId, AttachmentTable.ArchiveTransferState.FINISHED) + + assertThat(SignalDatabase.attachments.getMediaNamesWithNoEligibleThumbnail(setOf(mediaNameFor(attachmentId)))).hasSize(1) + } + + @Test + fun givenAnSvg_whenIQueryByName_thenIExpectItUnwanted() { + val attachmentId = insertArchivedAttachment(contentType = "image/svg+xml") + SignalDatabase.attachments.setArchiveThumbnailTransferState(attachmentId, AttachmentTable.ArchiveTransferState.FINISHED) + + assertThat(SignalDatabase.attachments.getMediaNamesWithNoEligibleThumbnail(setOf(mediaNameFor(attachmentId)))).hasSize(1) + } + + /** + * Stickers are excluded from thumbnail generation upstream, so in practice they never reach the CDN and this branch is unreachable. It stays unfiltered + * deliberately: this clause exists to mirror the snapshot write side, and the write side doesn't special-case stickers either. + */ + @Test + fun givenASticker_whenIQueryByName_thenIExpectItStillWanted() { + val attachmentId = insertArchivedAttachment() + SignalDatabase.attachments.setArchiveThumbnailTransferState(attachmentId, AttachmentTable.ArchiveTransferState.FINISHED) + + // Read before writing the sticker id, since setting it alone leaves a sticker locator the attachment reader can't parse. + val mediaName = mediaNameFor(attachmentId) + setColumn(attachmentId, AttachmentTable.STICKER_ID, 7L) + + assertThat(SignalDatabase.attachments.getMediaNamesWithNoEligibleThumbnail(setOf(mediaName))).isEmpty() + } + + private fun setColumn(attachmentId: AttachmentId, column: String, value: Long) { + SignalDatabase.attachments.writableDatabase + .update(AttachmentTable.TABLE_NAME) + .values(column to value) + .where("${AttachmentTable.ID} = ?", attachmentId.id) + .run() + } + + private fun mediaNameFor(attachmentId: AttachmentId): AttachmentTable.MediaNameParts { + val attachment = SignalDatabase.attachments.getAttachment(attachmentId)!! + + return AttachmentTable.MediaNameParts( + plaintextHash = attachment.dataHash!!, + remoteKey = attachment.remoteKey!! + ) + } + + private fun thumbnailEntryFor(attachmentId: AttachmentId): BackupMediaSnapshotTable.MediaEntry { + val attachment = SignalDatabase.attachments.getAttachment(attachmentId)!! + + return BackupMediaSnapshotTable.MediaEntry( + mediaId = "media-id-${attachment.attachmentId.id}", + cdn = 3, + plaintextHash = Base64.decode(attachment.dataHash!!), + remoteKey = Base64.decode(attachment.remoteKey!!), + isThumbnail = true + ) + } + + private fun insertArchivedAttachment(contentType: String = "image/jpeg"): AttachmentId { + val from = recipients.createRecipient("Some Contact") + val threadId = SignalDatabase.threads.getOrCreateThreadIdFor(Recipient.resolved(from)) + + val message = IncomingMessage( + type = MessageType.NORMAL, + from = from, + body = null, + sentTimeMillis = 100L, + serverTimeMillis = 100L, + receivedTimeMillis = 200L, + attachments = listOf(createArchivedAttachment(contentType)) + ) + + val messageId = SignalDatabase.messages.insertMessageInbox(message, threadId).get().messageId + return SignalDatabase.attachments.getAttachmentsForMessage(messageId).first().attachmentId + } + + private fun createArchivedAttachment(contentType: String): Attachment { + return ArchivedAttachment( + contentType = contentType, + size = 1024, + cdn = 3, + uploadTimestamp = 0, + key = Random.nextBytes(32), + cdnKey = "password", + archiveCdn = 3, + plaintextHash = Random.nextBytes(32), + incrementalMac = Random.nextBytes(8), + incrementalMacChunkSize = 8, + width = 100, + height = 100, + caption = null, + blurHash = null, + voiceNote = false, + borderless = false, + stickerLocator = null, + gif = false, + quote = false, + quoteTargetContentType = null, + uuid = UUID.randomUUID(), + fileName = null, + localBackupKey = null + ) + } +} diff --git a/app/src/test/java/org/thoughtcrime/securesms/database/AttachmentTableTest_thumbnailsThatNeedArchiveWork.kt b/app/src/test/java/org/thoughtcrime/securesms/database/AttachmentTableTest_thumbnailsThatNeedArchiveWork.kt new file mode 100644 index 0000000000..8d14616848 --- /dev/null +++ b/app/src/test/java/org/thoughtcrime/securesms/database/AttachmentTableTest_thumbnailsThatNeedArchiveWork.kt @@ -0,0 +1,275 @@ +/* + * Copyright 2026 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ + +package org.thoughtcrime.securesms.database + +import android.app.Application +import assertk.assertThat +import assertk.assertions.containsExactly +import assertk.assertions.isEmpty +import assertk.assertions.isEqualTo +import assertk.assertions.isFalse +import assertk.assertions.isTrue +import org.junit.BeforeClass +import org.junit.Rule +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import org.robolectric.annotation.Config +import org.signal.core.models.database.AttachmentId +import org.signal.core.util.logging.Log +import org.signal.core.util.update +import org.thoughtcrime.securesms.attachments.ArchivedAttachment +import org.thoughtcrime.securesms.attachments.Attachment +import org.thoughtcrime.securesms.database.AttachmentTable.ArchiveTransferState +import org.thoughtcrime.securesms.mms.IncomingMessage +import org.thoughtcrime.securesms.recipients.Recipient +import org.thoughtcrime.securesms.testutil.RecipientTestRule +import org.thoughtcrime.securesms.testutil.SystemOutLogger +import java.util.UUID +import kotlin.random.Random + +/** + * Coverage for which thumbnails are selected for archive work. A thumbnail qualifies when the archive has no copy, and also when we have no + * local copy: without a local thumbnail file the attachment can never satisfy the offload eligibility check, so a restored device would be stuck. + */ +@Suppress("ClassName") +@RunWith(RobolectricTestRunner::class) +@Config(manifest = Config.NONE, application = Application::class) +class AttachmentTableTest_thumbnailsThatNeedArchiveWork { + + @get:Rule val recipients = RecipientTestRule() + + companion object { + @BeforeClass + @JvmStatic + fun setUpClass() { + Log.initialize(SystemOutLogger()) + } + } + + @Test + fun givenAnArchivedThumbnailWithNoLocalFile_whenIQuery_thenIExpectItSelected() { + val attachmentId = insertAttachment(thumbnailState = ArchiveTransferState.FINISHED, hasLocalThumbnail = false) + + assertThat(SignalDatabase.attachments.getThumbnailsThatNeedArchiveUpload()).containsExactly(attachmentId) + assertThat(SignalDatabase.attachments.doAnyThumbnailsNeedArchiveUpload()).isTrue() + } + + @Test + fun givenAnArchivedThumbnailWithALocalFile_whenIQuery_thenIExpectItSkipped() { + insertAttachment(thumbnailState = ArchiveTransferState.FINISHED, hasLocalThumbnail = true) + + assertThat(SignalDatabase.attachments.getThumbnailsThatNeedArchiveUpload()).isEmpty() + assertThat(SignalDatabase.attachments.doAnyThumbnailsNeedArchiveUpload()).isFalse() + } + + @Test + fun givenAnUnarchivedThumbnailWithALocalFile_whenIQuery_thenIExpectItSelected() { + val attachmentId = insertAttachment(thumbnailState = ArchiveTransferState.NONE, hasLocalThumbnail = true) + + assertThat(SignalDatabase.attachments.getThumbnailsThatNeedArchiveUpload()).containsExactly(attachmentId) + } + + @Test + fun givenAnUnarchivedThumbnailWithNoLocalFile_whenIQuery_thenIExpectItSelected() { + val attachmentId = insertAttachment(thumbnailState = ArchiveTransferState.NONE, hasLocalThumbnail = false) + + assertThat(SignalDatabase.attachments.getThumbnailsThatNeedArchiveUpload()).containsExactly(attachmentId) + } + + @Test + fun givenATemporarilyFailedThumbnail_whenIQuery_thenIExpectItSelected() { + val attachmentId = insertAttachment(thumbnailState = ArchiveTransferState.TEMPORARY_FAILURE, hasLocalThumbnail = true) + + assertThat(SignalDatabase.attachments.getThumbnailsThatNeedArchiveUpload()).containsExactly(attachmentId) + } + + /** + * An upload writes the thumbnail file only once it finishes, so an in-flight row always looks like it has no local copy. Selecting it would enqueue a second + * upload for work already underway. + */ + @Test + fun givenAnInFlightUploadWithNoLocalFile_whenIQuery_thenIExpectItSkipped() { + insertAttachment(thumbnailState = ArchiveTransferState.UPLOAD_IN_PROGRESS, hasLocalThumbnail = false) + + assertThat(SignalDatabase.attachments.getThumbnailsThatNeedArchiveUpload()).isEmpty() + assertThat(SignalDatabase.attachments.doAnyThumbnailsNeedArchiveUpload()).isFalse() + } + + @Test + fun givenACopyPendingThumbnailWithNoLocalFile_whenIQuery_thenIExpectItSkipped() { + insertAttachment(thumbnailState = ArchiveTransferState.COPY_PENDING, hasLocalThumbnail = false) + + assertThat(SignalDatabase.attachments.getThumbnailsThatNeedArchiveUpload()).isEmpty() + } + + /** + * If we already established we can't produce a local thumbnail for this media, selecting it again just fails again after every backup. + */ + @Test + fun givenAnArchivedThumbnailWeCannotRestoreLocally_whenIQuery_thenIExpectItSkipped() { + val attachmentId = insertAttachment(thumbnailState = ArchiveTransferState.FINISHED, hasLocalThumbnail = false) + SignalDatabase.attachments.setThumbnailRestoreState(attachmentId, AttachmentTable.ThumbnailRestoreState.PERMANENT_FAILURE) + + assertThat(SignalDatabase.attachments.getThumbnailsThatNeedArchiveUpload()).isEmpty() + assertThat(SignalDatabase.attachments.doAnyThumbnailsNeedArchiveUpload()).isFalse() + } + + @Test + fun givenAPermanentlyFailedThumbnailWithNoLocalFile_whenIQuery_thenIExpectItSkipped() { + insertAttachment(thumbnailState = ArchiveTransferState.PERMANENT_FAILURE, hasLocalThumbnail = false) + + assertThat(SignalDatabase.attachments.getThumbnailsThatNeedArchiveUpload()).isEmpty() + assertThat(SignalDatabase.attachments.doAnyThumbnailsNeedArchiveUpload()).isFalse() + } + + @Test + fun givenNoLocalDataFile_whenIQuery_thenIExpectItSkipped() { + insertAttachment(thumbnailState = ArchiveTransferState.FINISHED, hasLocalThumbnail = false, hasLocalData = false) + + assertThat(SignalDatabase.attachments.getThumbnailsThatNeedArchiveUpload()).isEmpty() + } + + @Test + fun givenNonVisualMedia_whenIQuery_thenIExpectItSkipped() { + insertAttachment(thumbnailState = ArchiveTransferState.FINISHED, hasLocalThumbnail = false, contentType = "application/pdf") + + assertThat(SignalDatabase.attachments.getThumbnailsThatNeedArchiveUpload()).isEmpty() + } + + @Test + fun givenAMixOfStates_whenIQuery_thenIExpectOnlyTheOnesNeedingWork() { + val archivedWithNoFile = insertAttachment(thumbnailState = ArchiveTransferState.FINISHED, hasLocalThumbnail = false) + val unarchived = insertAttachment(thumbnailState = ArchiveTransferState.NONE, hasLocalThumbnail = true) + insertAttachment(thumbnailState = ArchiveTransferState.FINISHED, hasLocalThumbnail = true) + insertAttachment(thumbnailState = ArchiveTransferState.PERMANENT_FAILURE, hasLocalThumbnail = false) + + assertThat(SignalDatabase.attachments.getThumbnailsThatNeedArchiveUpload()).containsExactly(archivedWithNoFile, unarchived) + } + + @Test + fun givenALocalThumbnailFile_whenIAskIfItHasOne_thenIExpectTrue() { + val attachmentId = insertAttachment(thumbnailState = ArchiveTransferState.FINISHED, hasLocalThumbnail = true) + + assertThat(SignalDatabase.attachments.hasThumbnailFile(attachmentId)).isTrue() + } + + @Test + fun givenNoLocalThumbnailFile_whenIAskIfItHasOne_thenIExpectFalse() { + val attachmentId = insertAttachment(thumbnailState = ArchiveTransferState.FINISHED, hasLocalThumbnail = false) + + assertThat(SignalDatabase.attachments.hasThumbnailFile(attachmentId)).isFalse() + } + + /** + * [AttachmentTable.ThumbnailRestoreState.PERMANENT_FAILURE] is also what stops the restore job from downloading, so tombstoning a row a restore could still + * recover would cost the user a thumbnail the archive CDN is holding. + */ + @Test + fun givenARestoreIsPending_whenIMarkItUnrestorable_thenIExpectTheRestoreStateUntouched() { + val attachmentId = insertAttachment(thumbnailState = ArchiveTransferState.FINISHED, hasLocalThumbnail = false) + SignalDatabase.attachments.setThumbnailRestoreState(attachmentId, AttachmentTable.ThumbnailRestoreState.NEEDS_RESTORE) + + SignalDatabase.attachments.markThumbnailPermanentlyFailedIfUnrestorable(attachmentId) + + assertThat(restoreStateOf(attachmentId)).isEqualTo(AttachmentTable.ThumbnailRestoreState.NEEDS_RESTORE) + } + + @Test + fun givenARestoreAlreadyFinished_whenIMarkItUnrestorable_thenIExpectTheRestoreStateUntouched() { + val attachmentId = insertAttachment(thumbnailState = ArchiveTransferState.FINISHED, hasLocalThumbnail = false) + SignalDatabase.attachments.setThumbnailRestoreState(attachmentId, AttachmentTable.ThumbnailRestoreState.FINISHED) + + SignalDatabase.attachments.markThumbnailPermanentlyFailedIfUnrestorable(attachmentId) + + assertThat(restoreStateOf(attachmentId)).isEqualTo(AttachmentTable.ThumbnailRestoreState.FINISHED) + } + + @Test + fun givenNoRestoreIsPossible_whenIMarkItUnrestorable_thenIExpectItTombstonedAndSkipped() { + val attachmentId = insertAttachment(thumbnailState = ArchiveTransferState.FINISHED, hasLocalThumbnail = false) + SignalDatabase.attachments.setThumbnailRestoreState(attachmentId, AttachmentTable.ThumbnailRestoreState.NONE) + + SignalDatabase.attachments.markThumbnailPermanentlyFailedIfUnrestorable(attachmentId) + + assertThat(restoreStateOf(attachmentId)).isEqualTo(AttachmentTable.ThumbnailRestoreState.PERMANENT_FAILURE) + assertThat(SignalDatabase.attachments.getThumbnailsThatNeedArchiveUpload()).isEmpty() + } + + private fun restoreStateOf(attachmentId: AttachmentId): AttachmentTable.ThumbnailRestoreState { + return SignalDatabase.attachments.getAttachment(attachmentId)!!.thumbnailRestoreState + } + + private fun insertAttachment( + thumbnailState: ArchiveTransferState, + hasLocalThumbnail: Boolean, + hasLocalData: Boolean = true, + contentType: String = "image/jpeg" + ): AttachmentId { + val attachmentId = insertArchivedAttachment(contentType) + + SignalDatabase.attachments.writableDatabase + .update(AttachmentTable.TABLE_NAME) + .values( + AttachmentTable.DATA_FILE to if (hasLocalData) "/fake/path/data-${attachmentId.id}" else null, + AttachmentTable.DATA_RANDOM to if (hasLocalData) Random.nextBytes(32) else null, + AttachmentTable.TRANSFER_STATE to AttachmentTable.TRANSFER_PROGRESS_DONE, + AttachmentTable.THUMBNAIL_FILE to if (hasLocalThumbnail) "/fake/path/thumb-${attachmentId.id}" else null, + AttachmentTable.THUMBNAIL_RANDOM to if (hasLocalThumbnail) Random.nextBytes(32) else null, + AttachmentTable.ARCHIVE_THUMBNAIL_TRANSFER_STATE to thumbnailState.value + ) + .where("${AttachmentTable.ID} = ?", attachmentId.id) + .run() + + return attachmentId + } + + private fun insertArchivedAttachment(contentType: String): AttachmentId { + val from = recipients.createRecipient("Some Contact") + val threadId = SignalDatabase.threads.getOrCreateThreadIdFor(Recipient.resolved(from)) + + val message = IncomingMessage( + type = MessageType.NORMAL, + from = from, + body = null, + sentTimeMillis = 100L, + serverTimeMillis = 100L, + receivedTimeMillis = 200L, + attachments = listOf(createArchivedAttachment(contentType)) + ) + + val messageId = SignalDatabase.messages.insertMessageInbox(message, threadId).get().messageId + return SignalDatabase.attachments.getAttachmentsForMessage(messageId).first().attachmentId + } + + private fun createArchivedAttachment(contentType: String): Attachment { + return ArchivedAttachment( + contentType = contentType, + size = 1024, + cdn = 3, + uploadTimestamp = 0, + key = Random.nextBytes(32), + cdnKey = "password", + archiveCdn = 3, + plaintextHash = Random.nextBytes(32), + incrementalMac = Random.nextBytes(8), + incrementalMacChunkSize = 8, + width = 100, + height = 100, + caption = null, + blurHash = null, + voiceNote = false, + borderless = false, + stickerLocator = null, + gif = false, + quote = false, + quoteTargetContentType = null, + uuid = UUID.randomUUID(), + fileName = null, + localBackupKey = null + ) + } +} diff --git a/app/src/test/java/org/thoughtcrime/securesms/database/BackupMediaSnapshotTableTest.kt b/app/src/test/java/org/thoughtcrime/securesms/database/BackupMediaSnapshotTableTest.kt index 8813989f2f..94a3ae82f8 100644 --- a/app/src/test/java/org/thoughtcrime/securesms/database/BackupMediaSnapshotTableTest.kt +++ b/app/src/test/java/org/thoughtcrime/securesms/database/BackupMediaSnapshotTableTest.kt @@ -2,6 +2,7 @@ package org.thoughtcrime.securesms.database import android.app.Application import assertk.assertThat +import assertk.assertions.isEmpty import assertk.assertions.isEqualTo import org.junit.Rule import org.junit.Test @@ -108,8 +109,8 @@ class BackupMediaSnapshotTableTest { SignalDatabase.backupMediaSnapshots.writePendingMediaEntries(generateArchiveMediaItemSequence(count = additionalCount)) SignalDatabase.backupMediaSnapshots.commitPendingRows() - val page = SignalDatabase.backupMediaSnapshots.getPageOfOldMediaObjects(pageSize = 1_000) - SignalDatabase.backupMediaSnapshots.deleteOldMediaObjects(page) + val page = SignalDatabase.backupMediaSnapshots.getPageOfOldMediaEntries(pageSize = 1_000) + SignalDatabase.backupMediaSnapshots.deleteOldMediaObjects(page.map { it.mediaId }) val total = getTotalItemCount(includeThumbnails = false) @@ -251,6 +252,71 @@ class BackupMediaSnapshotTableTest { assertThat(notSeenCount).isEqualTo(expectedOldCount) } + /** + * A reconciliation that completes while the snapshot is empty records version 0, and 0 is also the never-confirmed default, so the minimum must not be + * satisfiable by rows the CDN has never confirmed. Otherwise offloading would delete the only local copy of unconfirmed media. + */ + @Test + fun getMediaIdsConfirmedOnCdn_neverSeenOnRemoteIsNotConfirmedEvenWhenMinimumIsZero() { + val items = generateArchiveMediaItemSequence(count = 10) + + SignalDatabase.backupMediaSnapshots.writePendingMediaEntries(items) + SignalDatabase.backupMediaSnapshots.commitPendingRows() + + val confirmed = SignalDatabase.backupMediaSnapshots.getMediaIdsConfirmedOnCdn(items.map { it.mediaId }, minimumLastSeenSnapshotVersion = 0) + + assertThat(confirmed).isEmpty() + } + + @Test + fun getMediaIdsConfirmedOnCdn_seenOnRemoteIsConfirmed() { + val items = generateArchiveMediaItemSequence(count = 10) + + SignalDatabase.backupMediaSnapshots.writePendingMediaEntries(items) + SignalDatabase.backupMediaSnapshots.commitPendingRows() + SignalDatabase.backupMediaSnapshots.markSeenOnRemote(items.map { it.mediaId }, 1) + + val confirmed = SignalDatabase.backupMediaSnapshots.getMediaIdsConfirmedOnCdn(items.map { it.mediaId }, minimumLastSeenSnapshotVersion = 1) + + assertThat(confirmed.size).isEqualTo(items.size) + } + + /** + * A reconciliation captures its snapshot version once and can resume days later, after further backups have committed. Replaying the version it captured must + * not walk a confirmation backwards, and least of all to 0, which the prune reads as "the CDN has never had this". + */ + @Test + fun markSeenOnRemote_staleVersionDoesNotEraseNewerConfirmation() { + val items = generateArchiveMediaItemSequence(count = 10) + val mediaIds = items.map { it.mediaId } + + SignalDatabase.backupMediaSnapshots.writePendingMediaEntries(items) + SignalDatabase.backupMediaSnapshots.commitPendingRows() + SignalDatabase.backupMediaSnapshots.markSeenOnRemote(mediaIds, 1) + + SignalDatabase.backupMediaSnapshots.markSeenOnRemote(mediaIds, 0) + + val confirmed = SignalDatabase.backupMediaSnapshots.getMediaIdsConfirmedOnCdn(mediaIds, minimumLastSeenSnapshotVersion = 1) + + assertThat(confirmed.size).isEqualTo(items.size) + } + + @Test + fun markSeenOnRemote_newerVersionAdvancesConfirmation() { + val items = generateArchiveMediaItemSequence(count = 10) + val mediaIds = items.map { it.mediaId } + + SignalDatabase.backupMediaSnapshots.writePendingMediaEntries(items) + SignalDatabase.backupMediaSnapshots.commitPendingRows() + SignalDatabase.backupMediaSnapshots.markSeenOnRemote(mediaIds, 1) + + SignalDatabase.backupMediaSnapshots.markSeenOnRemote(mediaIds, 2) + + val confirmed = SignalDatabase.backupMediaSnapshots.getMediaIdsConfirmedOnCdn(mediaIds, minimumLastSeenSnapshotVersion = 2) + + assertThat(confirmed.size).isEqualTo(items.size) + } + private fun getTotalItemCount(includeThumbnails: Boolean): Int { return if (includeThumbnails) { SignalDatabase.backupMediaSnapshots.readableDatabase diff --git a/app/src/test/java/org/thoughtcrime/securesms/jobs/BackupRestoreMediaJobTest.kt b/app/src/test/java/org/thoughtcrime/securesms/jobs/BackupRestoreMediaJobTest.kt new file mode 100644 index 0000000000..ef5a821675 --- /dev/null +++ b/app/src/test/java/org/thoughtcrime/securesms/jobs/BackupRestoreMediaJobTest.kt @@ -0,0 +1,289 @@ +/* + * Copyright 2026 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ + +package org.thoughtcrime.securesms.jobs + +import android.app.Application +import assertk.assertThat +import assertk.assertions.containsExactlyInAnyOrder +import assertk.assertions.isEmpty +import assertk.assertions.isEqualTo +import io.mockk.Runs +import io.mockk.every +import io.mockk.just +import io.mockk.mockkObject +import io.mockk.unmockkObject +import org.junit.After +import org.junit.Before +import org.junit.BeforeClass +import org.junit.Rule +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import org.robolectric.annotation.Config +import org.signal.core.models.database.AttachmentId +import org.signal.core.util.logging.Log +import org.signal.core.util.readToList +import org.signal.core.util.readToSingleInt +import org.signal.core.util.requireLong +import org.signal.core.util.select +import org.signal.core.util.update +import org.thoughtcrime.securesms.attachments.ArchivedAttachment +import org.thoughtcrime.securesms.attachments.Attachment +import org.thoughtcrime.securesms.backup.RestoreState +import org.thoughtcrime.securesms.backup.v2.ArchiveRestoreProgress +import org.thoughtcrime.securesms.database.AttachmentTable +import org.thoughtcrime.securesms.database.MessageTable +import org.thoughtcrime.securesms.database.MessageType +import org.thoughtcrime.securesms.database.SignalDatabase +import org.thoughtcrime.securesms.dependencies.AppDependencies +import org.thoughtcrime.securesms.jobmanager.Job +import org.thoughtcrime.securesms.mms.IncomingMessage +import org.thoughtcrime.securesms.recipients.Recipient +import org.thoughtcrime.securesms.testutil.RecipientTestRule +import org.thoughtcrime.securesms.testutil.SystemOutLogger +import java.util.UUID +import kotlin.random.Random +import kotlin.time.Duration.Companion.days + +/** + * Coverage for which attachments get a thumbnail restore enqueued. Asking the CDN for a thumbnail that was never generated is guaranteed to 404, so the + * enqueue side has to agree with the upload side about what can have one. + */ +@RunWith(RobolectricTestRunner::class) +@Config(manifest = Config.NONE, application = Application::class) +class BackupRestoreMediaJobTest { + + @get:Rule val recipients = RecipientTestRule() + + companion object { + @BeforeClass + @JvmStatic + fun setUpClass() { + Log.initialize(SystemOutLogger()) + } + } + + private val enqueuedJobs = mutableListOf() + + @Before + fun setUp() { + every { recipients.signalStore.backup.optimizeStorage } returns true + + // ArchiveRestoreProgress reads these while initializing, and BackupValues is a strict mock, so they have to exist before the object is touched at all. + every { recipients.signalStore.backup.restoreState } returns RestoreState.NONE + every { recipients.signalStore.backup.totalRestorableAttachmentSize } returns 0L + mockkObject(ArchiveRestoreProgress) + every { ArchiveRestoreProgress.onProcessStart() } just Runs + + every { AppDependencies.jobManager.add(capture(enqueuedJobs)) } returns Unit + } + + @After + fun tearDown() { + unmockkObject(ArchiveRestoreProgress) + } + + @Test + fun givenAnImage_whenIEnqueue_thenIExpectAThumbnailRestore() { + val image = givenRestorableAttachment(contentType = "image/jpeg") + + enqueue() + + assertThat(thumbnailRestoreTargets()).containsExactlyInAnyOrder(image) + } + + @Test + fun givenAVideo_whenIEnqueue_thenIExpectAThumbnailRestore() { + val video = givenRestorableAttachment(contentType = "video/mp4") + + enqueue() + + assertThat(thumbnailRestoreTargets()).containsExactlyInAnyOrder(video) + } + + /** A document never had a thumbnail generated, so the upload side never produced one to fetch. */ + @Test + fun givenADocument_whenIEnqueue_thenIExpectNoThumbnailRestore() { + givenRestorableAttachment(contentType = "application/pdf") + + enqueue() + + assertThat(thumbnailRestoreTargets()).isEmpty() + } + + /** Stickers are image types, so only the sticker id keeps them out of thumbnail work on the upload side. */ + @Test + fun givenASticker_whenIEnqueue_thenIExpectNoThumbnailRestore() { + val sticker = givenRestorableAttachment(contentType = "image/webp") + markAsSticker(sticker) + + enqueue() + + assertThat(thumbnailRestoreTargets()).isEmpty() + } + + @Test + fun givenAnSvg_whenIEnqueue_thenIExpectNoThumbnailRestore() { + givenRestorableAttachment(contentType = "image/svg+xml") + + enqueue() + + assertThat(thumbnailRestoreTargets()).isEmpty() + } + + @Test + fun givenAQuote_whenIEnqueue_thenIExpectNoThumbnailRestore() { + val quote = givenRestorableAttachment(contentType = "image/jpeg") + markAsQuote(quote) + + enqueue() + + assertThat(thumbnailRestoreTargets()).isEmpty() + } + + /** Skipping the job must not skip the state change, or the row would be re-selected by every later batch. */ + @Test + fun givenAnIneligibleAttachment_whenIEnqueue_thenIExpectItStillOffloaded() { + val document = givenRestorableAttachment(contentType = "application/pdf") + + enqueue() + + assertThat(transferStateOf(document)).isEqualTo(AttachmentTable.TRANSFER_RESTORE_OFFLOADED) + } + + /** + * The batch that drains no jobs is the regression this guards. With a page size of 1 and three ineligible attachments, a loop keyed off the jobs it created + * would stop after the first batch and silently leave the rest of the restore undone. + */ + @Test + fun givenMoreIneligibleAttachmentsThanOnePage_whenIEnqueue_thenIExpectEveryRowDrained() { + repeat(3) { givenRestorableAttachment(contentType = "application/pdf") } + + enqueue(batchSize = 1) + + assertThat(needsRestoreCount()).isEqualTo(0) + } + + @Test + fun givenAMixOfEligibleAndNot_whenIEnqueue_thenIExpectOnlyTheEligibleOnesRequested() { + val image = givenRestorableAttachment(contentType = "image/jpeg") + givenRestorableAttachment(contentType = "application/pdf") + val video = givenRestorableAttachment(contentType = "video/mp4") + givenRestorableAttachment(contentType = "audio/mpeg") + + enqueue(batchSize = 2) + + assertThat(thumbnailRestoreTargets()).containsExactlyInAnyOrder(image, video) + assertThat(needsRestoreCount()).isEqualTo(0) + } + + private fun enqueue(batchSize: Int = 500) { + BackupRestoreMediaJob().enqueueRestoreJobs(restoreTime = System.currentTimeMillis(), batchSize = batchSize) + } + + private fun thumbnailRestoreTargets(): List { + return enqueuedJobs.filterIsInstance().map { it.attachmentId } + } + + /** + * Inserts an attachment that needs restoring on a message old enough that optimize storage routes it down the thumbnail-only branch rather than a full-size + * restore. + */ + private fun givenRestorableAttachment(contentType: String): AttachmentId { + val from = recipients.createRecipient("Contact ${UUID.randomUUID()}") + val threadId = SignalDatabase.threads.getOrCreateThreadIdFor(Recipient.resolved(from)) + + val message = IncomingMessage( + type = MessageType.NORMAL, + from = from, + body = null, + sentTimeMillis = 100L, + serverTimeMillis = 100L, + receivedTimeMillis = 200L, + attachments = listOf(createAttachment(contentType)) + ) + + val messageId = SignalDatabase.messages.insertMessageInbox(message, threadId).get().messageId + val attachmentId = SignalDatabase.attachments.getAttachmentsForMessage(messageId).first().attachmentId + + SignalDatabase.messages.writableDatabase + .update(MessageTable.TABLE_NAME) + .values(MessageTable.DATE_RECEIVED to System.currentTimeMillis() - 60.days.inWholeMilliseconds) + .where("${MessageTable.ID} = ?", messageId) + .run() + + SignalDatabase.attachments.writableDatabase + .update(AttachmentTable.TABLE_NAME) + .values(AttachmentTable.TRANSFER_STATE to AttachmentTable.TRANSFER_NEEDS_RESTORE) + .where("${AttachmentTable.ID} = ?", attachmentId.id) + .run() + + return attachmentId + } + + private fun markAsSticker(attachmentId: AttachmentId) { + SignalDatabase.attachments.writableDatabase + .update(AttachmentTable.TABLE_NAME) + .values(AttachmentTable.STICKER_ID to 0) + .where("${AttachmentTable.ID} = ?", attachmentId.id) + .run() + } + + private fun markAsQuote(attachmentId: AttachmentId) { + SignalDatabase.attachments.writableDatabase + .update(AttachmentTable.TABLE_NAME) + .values(AttachmentTable.QUOTE to 1) + .where("${AttachmentTable.ID} = ?", attachmentId.id) + .run() + } + + private fun transferStateOf(attachmentId: AttachmentId): Int { + return SignalDatabase.attachments.readableDatabase + .select(AttachmentTable.TRANSFER_STATE) + .from(AttachmentTable.TABLE_NAME) + .where("${AttachmentTable.ID} = ?", attachmentId.id) + .run() + .readToSingleInt(-1) + } + + private fun needsRestoreCount(): Int { + return SignalDatabase.attachments.readableDatabase + .select(AttachmentTable.ID) + .from(AttachmentTable.TABLE_NAME) + .where("${AttachmentTable.TRANSFER_STATE} = ?", AttachmentTable.TRANSFER_NEEDS_RESTORE) + .run() + .readToList { it.requireLong(AttachmentTable.ID) } + .size + } + + private fun createAttachment(contentType: String): Attachment { + return ArchivedAttachment( + contentType = contentType, + size = 1024, + cdn = 3, + uploadTimestamp = 0, + key = Random.nextBytes(32), + cdnKey = "password", + archiveCdn = 3, + plaintextHash = Random.nextBytes(32), + incrementalMac = null, + incrementalMacChunkSize = null, + width = 0, + height = 0, + caption = null, + blurHash = null, + voiceNote = false, + borderless = false, + stickerLocator = null, + gif = false, + quote = false, + quoteTargetContentType = null, + uuid = UUID.randomUUID(), + fileName = null, + localBackupKey = null + ) + } +} diff --git a/app/src/test/java/org/thoughtcrime/securesms/jobs/OptimizeMediaJobTest.kt b/app/src/test/java/org/thoughtcrime/securesms/jobs/OptimizeMediaJobTest.kt new file mode 100644 index 0000000000..a68baeacf6 --- /dev/null +++ b/app/src/test/java/org/thoughtcrime/securesms/jobs/OptimizeMediaJobTest.kt @@ -0,0 +1,379 @@ +/* + * Copyright 2026 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ + +package org.thoughtcrime.securesms.jobs + +import android.app.Application +import assertk.assertThat +import assertk.assertions.isEqualTo +import assertk.assertions.isNotNull +import assertk.assertions.isNull +import io.mockk.every +import io.mockk.verify +import org.junit.Before +import org.junit.BeforeClass +import org.junit.Rule +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import org.robolectric.annotation.Config +import org.signal.core.models.backup.MediaName +import org.signal.core.models.backup.MediaRootBackupKey +import org.signal.core.models.database.AttachmentId +import org.signal.core.util.Base64 +import org.signal.core.util.logging.Log +import org.signal.core.util.readToSingleInt +import org.signal.core.util.readToSingleObject +import org.signal.core.util.requireString +import org.signal.core.util.select +import org.signal.core.util.update +import org.thoughtcrime.securesms.attachments.ArchivedAttachment +import org.thoughtcrime.securesms.attachments.Attachment +import org.thoughtcrime.securesms.database.AttachmentTable +import org.thoughtcrime.securesms.database.BackupMediaSnapshotTable.MediaEntry +import org.thoughtcrime.securesms.database.MessageTable +import org.thoughtcrime.securesms.database.MessageType +import org.thoughtcrime.securesms.database.SignalDatabase +import org.thoughtcrime.securesms.dependencies.AppDependencies +import org.thoughtcrime.securesms.mms.IncomingMessage +import org.thoughtcrime.securesms.recipients.Recipient +import org.thoughtcrime.securesms.testutil.RecipientTestRule +import org.thoughtcrime.securesms.testutil.SystemOutLogger +import java.util.UUID +import kotlin.random.Random +import kotlin.time.Duration +import kotlin.time.Duration.Companion.days +import kotlin.time.Duration.Companion.hours + +/** + * Coverage for the gate that decides whether a local copy may be deleted. The local copy is usually the only copy, so every refusal here is the difference + * between a recoverable mistake and permanent data loss. + */ +@RunWith(RobolectricTestRunner::class) +@Config(manifest = Config.NONE, application = Application::class) +class OptimizeMediaJobTest { + + @get:Rule val recipients = RecipientTestRule() + + companion object { + private const val NO_COMPLETED_CRAWL = -1L + private val RECONCILIATION_INTERVAL = 7.days + private val MEDIA_ROOT_BACKUP_KEY = MediaRootBackupKey(Random.nextBytes(32)) + private val PLAINTEXT_HASH = Random.nextBytes(32) + private val REMOTE_KEY = Random.nextBytes(32) + + @BeforeClass + @JvmStatic + fun setUpClass() { + Log.initialize(SystemOutLogger()) + } + } + + @Before + fun setUp() { + every { recipients.signalStore.backup.mediaRootBackupKey } returns MEDIA_ROOT_BACKUP_KEY + } + + @Test + fun givenNoCrawlHasEverCompleted_whenIOffload_thenIExpectNothingOffloaded() { + val attachmentId = givenAnEligibleAttachmentConfirmedOnTheCdn() + givenCompletedCrawl(snapshotVersion = NO_COMPLETED_CRAWL, age = 1.days) + + offload() + + assertNotOffloaded(attachmentId) + } + + /** + * Refusing is safe but permanent on its own, since only a completed crawl can produce the evidence that lifts the refusal. Without this the feature would + * appear to do nothing until the periodic crawl happened to run. + */ + @Test + fun givenNoCrawlHasEverCompleted_whenIOffload_thenIExpectACrawlRequested() { + givenAnEligibleAttachmentConfirmedOnTheCdn() + givenCompletedCrawl(snapshotVersion = NO_COMPLETED_CRAWL, age = 1.days) + + offload() + + assertACrawlWasRequested() + } + + @Test + fun givenFreshEvidence_whenIOffload_thenIExpectNoCrawlRequested() { + givenAnEligibleAttachmentConfirmedOnTheCdn() + givenCompletedCrawl(snapshotVersion = 1, age = 1.days) + + offload() + + assertNoCrawlWasRequested() + } + + /** The caller runs after every backup, so without a rate limit an unverified device would force a full walk of the server listing daily. */ + @Test + fun givenACrawlWasForcedRecently_whenIOffload_thenIExpectNoCrawlRequested() { + givenAnEligibleAttachmentConfirmedOnTheCdn() + givenCompletedCrawl(snapshotVersion = NO_COMPLETED_CRAWL, age = 1.days) + givenLastForcedAttempt(age = 1.days) + + offload() + + assertNoCrawlWasRequested() + } + + @Test + fun givenTheLastForcedCrawlIsOlderThanTheInterval_whenIOffload_thenIExpectACrawlRequested() { + givenAnEligibleAttachmentConfirmedOnTheCdn() + givenCompletedCrawl(snapshotVersion = NO_COMPLETED_CRAWL, age = 1.days) + givenLastForcedAttempt(age = RECONCILIATION_INTERVAL * 2) + + offload() + + assertACrawlWasRequested() + } + + /** Media can disappear from the CDN after a crawl confirmed it, so a confirmation has a shelf life even though it was genuine when recorded. */ + @Test + fun givenTheEvidenceIsTooOld_whenIOffload_thenIExpectNothingOffloaded() { + val attachmentId = givenAnEligibleAttachmentConfirmedOnTheCdn() + givenCompletedCrawl(snapshotVersion = 1, age = RECONCILIATION_INTERVAL * 5) + + offload() + + assertNotOffloaded(attachmentId) + } + + @Test + fun givenTheEvidenceIsTooOld_whenIOffload_thenIExpectACrawlRequested() { + givenAnEligibleAttachmentConfirmedOnTheCdn() + givenCompletedCrawl(snapshotVersion = 1, age = RECONCILIATION_INTERVAL * 5) + + offload() + + assertACrawlWasRequested() + } + + /** + * A remotely configured interval of zero parses successfully rather than falling back to the default, so without a floor it would make every confirmation + * instantly stale and stop offloading on every device at once. + */ + @Test + fun givenTheConfiguredIntervalIsZero_whenIOffload_thenIExpectItOffloadedAnyway() { + val attachmentId = givenAnEligibleAttachmentConfirmedOnTheCdn() + givenCompletedCrawl(snapshotVersion = 1, age = 1.days) + + OptimizeMediaJob().offloadVerifiedMedia(minimumAge = 30.days, reconciliationInterval = Duration.ZERO) + + assertThat(transferStateOf(attachmentId)).isEqualTo(AttachmentTable.TRANSFER_RESTORE_OFFLOADED) + } + + /** The same floor has to reach the rate limit, or a zero interval would force a full walk of the server listing after every backup. */ + @Test + fun givenTheConfiguredIntervalIsZeroAndACrawlWasJustForced_whenIOffload_thenIExpectNoCrawlRequested() { + givenAnEligibleAttachmentConfirmedOnTheCdn() + givenCompletedCrawl(snapshotVersion = NO_COMPLETED_CRAWL, age = 1.days) + givenLastForcedAttempt(age = 1.hours) + + OptimizeMediaJob().offloadVerifiedMedia(minimumAge = 30.days, reconciliationInterval = Duration.ZERO) + + assertNoCrawlWasRequested() + } + + /** A clock that moved backwards makes the evidence age negative, which has to read as untrustworthy rather than as brand new. */ + @Test + fun givenTheClockRolledBack_whenIOffload_thenIExpectNothingOffloaded() { + val attachmentId = givenAnEligibleAttachmentConfirmedOnTheCdn() + givenCompletedCrawl(snapshotVersion = 1, age = -(30.days)) + + offload() + + assertNotOffloaded(attachmentId) + } + + /** Refusing on its own would leave a far-future timestamp gating offloading until the clock caught up to it, so the evidence has to be discarded outright. */ + @Test + fun givenTheClockRolledBack_whenIOffload_thenIExpectTheVerificationStateCleared() { + givenAnEligibleAttachmentConfirmedOnTheCdn() + givenCompletedCrawl(snapshotVersion = 1, age = -(30.days)) + + offload() + + val backup = recipients.signalStore.backup + verify { backup.clearArchiveVerificationState() } + } + + @Test + fun givenTheClockRolledBack_whenIOffload_thenIExpectACrawlRequested() { + givenAnEligibleAttachmentConfirmedOnTheCdn() + givenCompletedCrawl(snapshotVersion = 1, age = -(30.days)) + + offload() + + assertACrawlWasRequested() + } + + @Test + fun givenFreshEvidenceForTheMedia_whenIOffload_thenIExpectItOffloaded() { + val attachmentId = givenAnEligibleAttachmentConfirmedOnTheCdn() + givenCompletedCrawl(snapshotVersion = 1, age = 1.days) + + offload() + + assertThat(transferStateOf(attachmentId)).isEqualTo(AttachmentTable.TRANSFER_RESTORE_OFFLOADED) + assertThat(dataFileOf(attachmentId)).isNull() + } + + /** A snapshot row on its own is local bookkeeping. Only the crawl marking it seen makes it evidence the server produced. */ + @Test + fun givenTheMediaWasNeverSeenOnTheCdn_whenIOffload_thenIExpectNothingOffloaded() { + val attachmentId = givenAnEligibleAttachment() + commitSnapshotFor(PLAINTEXT_HASH, REMOTE_KEY, markSeen = false) + givenCompletedCrawl(snapshotVersion = 1, age = 1.days) + + offload() + + assertNotOffloaded(attachmentId) + } + + private fun offload() { + OptimizeMediaJob().offloadVerifiedMedia(minimumAge = 30.days, reconciliationInterval = RECONCILIATION_INTERVAL) + } + + private fun givenCompletedCrawl(snapshotVersion: Long, age: Duration) { + every { recipients.signalStore.backup.lastCompletedReconciliationSnapshotVersion } returns snapshotVersion + every { recipients.signalStore.backup.lastCompletedReconciliationTime } returns System.currentTimeMillis() - age.inWholeMilliseconds + givenLastForcedAttempt(age = null) + } + + /** A null [age] means no crawl has ever been forced, which is what a device that just enabled this looks like. */ + private fun givenLastForcedAttempt(age: Duration?) { + val timestamp = if (age == null) 0 else System.currentTimeMillis() - age.inWholeMilliseconds + every { recipients.signalStore.backup.lastForcedReconciliationAttemptTime } returns timestamp + } + + private fun givenAnEligibleAttachmentConfirmedOnTheCdn(): AttachmentId { + val attachmentId = givenAnEligibleAttachment() + commitSnapshotFor(PLAINTEXT_HASH, REMOTE_KEY, markSeen = true) + return attachmentId + } + + /** The offload path derives the media id from the hash and key, so the snapshot row has to carry the same derived value or it can never match. */ + private fun commitSnapshotFor(plaintextHash: ByteArray, remoteKey: ByteArray, markSeen: Boolean) { + val mediaId = MediaName.fromPlaintextHashAndRemoteKey(plaintextHash, remoteKey).toMediaId(MEDIA_ROOT_BACKUP_KEY).encode() + + SignalDatabase.backupMediaSnapshots.writePendingMediaEntries( + listOf(MediaEntry(mediaId = mediaId, cdn = 3, plaintextHash = plaintextHash, remoteKey = remoteKey, isThumbnail = false)) + ) + SignalDatabase.backupMediaSnapshots.commitPendingRows() + + if (markSeen) { + SignalDatabase.backupMediaSnapshots.markSeenOnRemote(listOf(mediaId), 1) + } + } + + private fun givenAnEligibleAttachment(): AttachmentId { + val from = recipients.createRecipient("Some Contact") + val threadId = SignalDatabase.threads.getOrCreateThreadIdFor(Recipient.resolved(from)) + + val message = IncomingMessage( + type = MessageType.NORMAL, + from = from, + body = null, + sentTimeMillis = 100L, + serverTimeMillis = 100L, + receivedTimeMillis = 200L, + attachments = listOf(createAttachment()) + ) + + val messageId = SignalDatabase.messages.insertMessageInbox(message, threadId).get().messageId + val attachmentId = SignalDatabase.attachments.getAttachmentsForMessage(messageId).first().attachmentId + + SignalDatabase.messages.writableDatabase + .update(MessageTable.TABLE_NAME) + .values(MessageTable.DATE_RECEIVED to System.currentTimeMillis() - 60.days.inWholeMilliseconds) + .where("${MessageTable.ID} = ?", messageId) + .run() + + SignalDatabase.attachments.writableDatabase + .update(AttachmentTable.TABLE_NAME) + .values( + AttachmentTable.TRANSFER_STATE to AttachmentTable.TRANSFER_PROGRESS_DONE, + AttachmentTable.ARCHIVE_TRANSFER_STATE to AttachmentTable.ArchiveTransferState.FINISHED.value, + AttachmentTable.DATA_FILE to "/not/a/real/file/${attachmentId.id}", + AttachmentTable.DATA_HASH_END to Base64.encodeWithPadding(PLAINTEXT_HASH), + AttachmentTable.REMOTE_KEY to Base64.encodeWithPadding(REMOTE_KEY), + AttachmentTable.OFFLOAD_RESTORED_AT to 0 + ) + .where("${AttachmentTable.ID} = ?", attachmentId.id) + .run() + + return attachmentId + } + + /** Deliberately not an image or video, so eligibility does not additionally depend on a local thumbnail existing. */ + private fun createAttachment(): Attachment { + return ArchivedAttachment( + contentType = "application/pdf", + size = 1024, + cdn = 3, + uploadTimestamp = 0, + key = Random.nextBytes(32), + cdnKey = "password", + archiveCdn = 3, + plaintextHash = Random.nextBytes(32), + incrementalMac = null, + incrementalMacChunkSize = null, + width = 0, + height = 0, + caption = null, + blurHash = null, + voiceNote = false, + borderless = false, + stickerLocator = null, + gif = false, + quote = false, + quoteTargetContentType = null, + uuid = UUID.randomUUID(), + fileName = null, + localBackupKey = null + ) + } + + /** + * The job manager is hoisted out of the verify block deliberately. [AppDependencies] is statically mocked, so referencing it inside the block would record + * the accessor itself as a call to verify. + */ + private fun assertACrawlWasRequested() { + val jobManager = AppDependencies.jobManager + verify { jobManager.add(ofType()) } + } + + private fun assertNoCrawlWasRequested() { + val jobManager = AppDependencies.jobManager + verify(exactly = 0) { jobManager.add(ofType()) } + } + + private fun assertNotOffloaded(attachmentId: AttachmentId) { + assertThat(transferStateOf(attachmentId)).isEqualTo(AttachmentTable.TRANSFER_PROGRESS_DONE) + assertThat(dataFileOf(attachmentId)).isNotNull() + } + + private fun transferStateOf(attachmentId: AttachmentId): Int { + return SignalDatabase.attachments.readableDatabase + .select(AttachmentTable.TRANSFER_STATE) + .from(AttachmentTable.TABLE_NAME) + .where("${AttachmentTable.ID} = ?", attachmentId.id) + .run() + .readToSingleInt(-1) + } + + private fun dataFileOf(attachmentId: AttachmentId): String? { + return SignalDatabase.attachments.readableDatabase + .select(AttachmentTable.DATA_FILE) + .from(AttachmentTable.TABLE_NAME) + .where("${AttachmentTable.ID} = ?", attachmentId.id) + .run() + .readToSingleObject { it.requireString(AttachmentTable.DATA_FILE) } + } +}