mirror of
https://github.com/signalapp/Signal-Android.git
synced 2026-09-21 17:24:38 +01:00
Improve logic around optimize media selection.
Use snapshot table to inform optimize eligibility.
This commit is contained in:
Cody Henthorne
+212
-2
@@ -9,6 +9,7 @@ import assertk.assertThat
|
||||
import assertk.assertions.hasSize
|
||||
import assertk.assertions.isEmpty
|
||||
import assertk.assertions.isEqualTo
|
||||
import assertk.assertions.isFalse
|
||||
import assertk.assertions.isNotEmpty
|
||||
import assertk.assertions.isNotEqualTo
|
||||
import assertk.assertions.isNull
|
||||
@@ -27,6 +28,7 @@ import org.signal.core.util.Base64
|
||||
import org.signal.core.util.Base64.decodeBase64OrThrow
|
||||
import org.signal.core.util.copyTo
|
||||
import org.signal.core.util.stream.NullOutputStream
|
||||
import org.signal.core.util.update
|
||||
import org.signal.mediasend.SentMediaQuality
|
||||
import org.thoughtcrime.securesms.attachments.ArchivedAttachment
|
||||
import org.thoughtcrime.securesms.attachments.Attachment
|
||||
@@ -58,6 +60,10 @@ import kotlin.time.Duration.Companion.seconds
|
||||
@RunWith(AndroidJUnit4::class)
|
||||
class AttachmentTableTest {
|
||||
|
||||
companion object {
|
||||
private const val TEST_PAGE_SIZE = 3
|
||||
}
|
||||
|
||||
@get:Rule
|
||||
val harness = SignalActivityRule(othersCount = 10)
|
||||
|
||||
@@ -594,6 +600,210 @@ class AttachmentTableTest {
|
||||
assertThat(updatedCount).isEqualTo(0)
|
||||
}
|
||||
|
||||
/**
|
||||
* Offloading deletes the only local copy, so it must be gated on evidence that came from the server. Being in the snapshot is our own bookkeeping. Only
|
||||
* last_seen_on_remote_snapshot_version is the part the CDN told us, so only it can authorize the delete.
|
||||
*/
|
||||
@Test
|
||||
fun givenAnOffloadCandidateConfirmedOnTheCdn_whenIOptimize_thenIExpectItOffloaded() {
|
||||
val attachmentId = seedOffloadCandidate()
|
||||
val snapshotVersion = commitSnapshotFor(attachmentId, markSeenOnRemote = true)
|
||||
|
||||
SignalDatabase.attachments.markEligibleAttachmentsAsOptimized(lastCompletedCrawlVersion = snapshotVersion, minimumAge = 30.days, now = pastTheOffloadWindow())
|
||||
|
||||
assertThat(SignalDatabase.attachments.getAttachment(attachmentId)!!.transferState).isEqualTo(AttachmentTable.TRANSFER_RESTORE_OFFLOADED)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun givenAnOffloadCandidateNeverSeenOnTheCdn_whenIOptimize_thenIExpectItLeftAlone() {
|
||||
val attachmentId = seedOffloadCandidate()
|
||||
val snapshotVersion = commitSnapshotFor(attachmentId, markSeenOnRemote = false)
|
||||
|
||||
SignalDatabase.attachments.markEligibleAttachmentsAsOptimized(lastCompletedCrawlVersion = snapshotVersion, minimumAge = 30.days, now = pastTheOffloadWindow())
|
||||
|
||||
assertThat(SignalDatabase.attachments.getAttachment(attachmentId)!!.transferState).isEqualTo(AttachmentTable.TRANSFER_PROGRESS_DONE)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun givenNoCompletedReconciliation_whenIOptimize_thenIExpectNothingOffloaded() {
|
||||
val attachmentId = seedOffloadCandidate()
|
||||
commitSnapshotFor(attachmentId, markSeenOnRemote = true)
|
||||
|
||||
SignalDatabase.attachments.markEligibleAttachmentsAsOptimized(lastCompletedCrawlVersion = -1, minimumAge = 30.days, now = pastTheOffloadWindow())
|
||||
|
||||
assertThat(SignalDatabase.attachments.getAttachment(attachmentId)!!.transferState).isEqualTo(AttachmentTable.TRANSFER_PROGRESS_DONE)
|
||||
}
|
||||
|
||||
/**
|
||||
* Coverage across pages: if only the first page were processed, the overflow would silently never be reclaimed.
|
||||
*/
|
||||
@Test
|
||||
fun givenMoreConfirmedCandidatesThanOnePage_whenIOptimize_thenIExpectEveryPageOffloaded() {
|
||||
val attachmentIds = List(TEST_PAGE_SIZE * 3) { seedOffloadCandidate() }
|
||||
val snapshotVersion = commitSnapshotForAll(attachmentIds, markSeenOnRemote = true)
|
||||
|
||||
SignalDatabase.attachments.markEligibleAttachmentsAsOptimized(lastCompletedCrawlVersion = snapshotVersion, minimumAge = 30.days, now = pastTheOffloadWindow(), pageSize = TEST_PAGE_SIZE)
|
||||
|
||||
assertThat(attachmentIds.count { isOffloaded(it) }).isEqualTo(attachmentIds.size)
|
||||
}
|
||||
|
||||
/**
|
||||
* Termination when nothing is offloadable. Unconfirmed candidates stay eligible forever, so a loop that ran until the candidate query drained would spin here
|
||||
* and this test would hang rather than fail. Only the advancing id cursor ends it.
|
||||
*/
|
||||
@Test
|
||||
fun givenMoreUnconfirmedCandidatesThanOnePage_whenIOptimize_thenIExpectItToTerminateAndOffloadNothing() {
|
||||
val attachmentIds = List(TEST_PAGE_SIZE * 3) { seedOffloadCandidate() }
|
||||
val snapshotVersion = commitSnapshotForAll(attachmentIds, markSeenOnRemote = false)
|
||||
|
||||
SignalDatabase.attachments.markEligibleAttachmentsAsOptimized(lastCompletedCrawlVersion = snapshotVersion, minimumAge = 30.days, now = pastTheOffloadWindow(), pageSize = TEST_PAGE_SIZE)
|
||||
|
||||
assertThat(attachmentIds.count { isOffloaded(it) }).isEqualTo(0)
|
||||
}
|
||||
|
||||
/**
|
||||
* Stickers come back from their pack rather than the archive, so they are never offloaded no matter what the CDN confirms.
|
||||
*/
|
||||
@Test
|
||||
fun givenAStickerOffloadCandidate_whenIOptimize_thenIExpectItLeftAlone() {
|
||||
val attachmentId = seedOffloadCandidate()
|
||||
val snapshotVersion = commitSnapshotFor(attachmentId, markSeenOnRemote = true)
|
||||
markAsSticker(attachmentId)
|
||||
|
||||
SignalDatabase.attachments.markEligibleAttachmentsAsOptimized(lastCompletedCrawlVersion = snapshotVersion, minimumAge = 30.days, now = pastTheOffloadWindow())
|
||||
|
||||
assertThat(isOffloaded(attachmentId)).isFalse()
|
||||
}
|
||||
|
||||
/**
|
||||
* Offloading an image with no thumbnail would leave nothing at all to render in the conversation, so visual media has to keep a local thumbnail to qualify.
|
||||
*/
|
||||
@Test
|
||||
fun givenAnImageOffloadCandidateWithNoThumbnail_whenIOptimize_thenIExpectItLeftAlone() {
|
||||
val attachmentId = seedOffloadCandidate(contentType = MediaUtil.IMAGE_JPEG)
|
||||
val snapshotVersion = commitSnapshotFor(attachmentId, markSeenOnRemote = true)
|
||||
|
||||
SignalDatabase.attachments.markEligibleAttachmentsAsOptimized(lastCompletedCrawlVersion = snapshotVersion, minimumAge = 30.days, now = pastTheOffloadWindow())
|
||||
|
||||
assertThat(isOffloaded(attachmentId)).isFalse()
|
||||
}
|
||||
|
||||
/**
|
||||
* Media the user just pulled back down would otherwise be offloaded again immediately, undoing the restore they asked for.
|
||||
*/
|
||||
@Test
|
||||
fun givenARecentlyRestoredOffloadCandidate_whenIOptimize_thenIExpectItLeftAlone() {
|
||||
val attachmentId = seedOffloadCandidate()
|
||||
val snapshotVersion = commitSnapshotFor(attachmentId, markSeenOnRemote = true)
|
||||
val evaluatedAt = pastTheOffloadWindow()
|
||||
markRestoredAt(attachmentId, evaluatedAt)
|
||||
|
||||
SignalDatabase.attachments.markEligibleAttachmentsAsOptimized(lastCompletedCrawlVersion = snapshotVersion, minimumAge = 30.days, now = evaluatedAt)
|
||||
|
||||
assertThat(isOffloaded(attachmentId)).isFalse()
|
||||
}
|
||||
|
||||
@Test
|
||||
fun givenAnOffloadCandidateOnATooRecentMessage_whenIOptimize_thenIExpectItLeftAlone() {
|
||||
val attachmentId = seedOffloadCandidate()
|
||||
val snapshotVersion = commitSnapshotFor(attachmentId, markSeenOnRemote = true)
|
||||
|
||||
SignalDatabase.attachments.markEligibleAttachmentsAsOptimized(lastCompletedCrawlVersion = snapshotVersion, minimumAge = 30.days, now = System.currentTimeMillis())
|
||||
|
||||
assertThat(isOffloaded(attachmentId)).isFalse()
|
||||
}
|
||||
|
||||
/** Far enough ahead of the seeded message that both the offload age and the 7-day offload-restore window are satisfied. */
|
||||
private fun pastTheOffloadWindow(): Long = System.currentTimeMillis() + 60.days.inWholeMilliseconds
|
||||
|
||||
/** A full locator, because a partial one makes reading the attachment back blow up on the sticker fields. */
|
||||
private fun markAsSticker(attachmentId: AttachmentId) {
|
||||
SignalDatabase.rawDatabase
|
||||
.update(AttachmentTable.TABLE_NAME)
|
||||
.values(
|
||||
AttachmentTable.STICKER_ID to 7,
|
||||
AttachmentTable.STICKER_PACK_ID to "aa1111bbcc2222ddee3333ff44445555",
|
||||
AttachmentTable.STICKER_PACK_KEY to "YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXoxMjM0NTY3ODkwYWI=",
|
||||
AttachmentTable.STICKER_EMOJI to ":)"
|
||||
)
|
||||
.where("${AttachmentTable.ID} = ?", attachmentId.id)
|
||||
.run()
|
||||
}
|
||||
|
||||
private fun markRestoredAt(attachmentId: AttachmentId, restoredAt: Long) {
|
||||
SignalDatabase.rawDatabase
|
||||
.update(AttachmentTable.TABLE_NAME)
|
||||
.values(AttachmentTable.OFFLOAD_RESTORED_AT to restoredAt)
|
||||
.where("${AttachmentTable.ID} = ?", attachmentId.id)
|
||||
.run()
|
||||
}
|
||||
|
||||
/** Defaults to a non-media content type, which keeps the candidate eligible without also having to generate a thumbnail file. */
|
||||
private fun seedOffloadCandidate(contentType: String = "application/pdf"): AttachmentId {
|
||||
val data = byteArrayOf(1, 2, 3, 4, 5)
|
||||
val attachment = createAttachmentPointer(Random.nextBytes(32), data.size, contentType = contentType)
|
||||
val messageResult = SignalDatabase.messages.insertMessageInbox(createIncomingMessage(serverTime = System.currentTimeMillis().milliseconds, attachment = attachment)).get()
|
||||
val attachmentId = messageResult.insertedAttachments!![attachment]!!
|
||||
|
||||
SignalDatabase.attachments.setTransferState(messageResult.messageId, attachmentId, AttachmentTable.TRANSFER_PROGRESS_STARTED)
|
||||
SignalDatabase.attachments.finalizeAttachmentAfterDownload(messageResult.messageId, attachmentId, ByteArrayInputStream(data))
|
||||
SignalDatabase.attachments.setArchiveTransferState(attachmentId, AttachmentTable.ArchiveTransferState.FINISHED)
|
||||
|
||||
return attachmentId
|
||||
}
|
||||
|
||||
/**
|
||||
* Commits every entry in a single snapshot version. Calling [commitSnapshotFor] in a loop would not work: each commit bumps the version, leaving all but the
|
||||
* last entry below MAX_VERSION and therefore unconfirmable.
|
||||
*/
|
||||
private fun commitSnapshotForAll(attachmentIds: List<AttachmentId>, markSeenOnRemote: Boolean): Long {
|
||||
val mediaIds: MutableList<String> = mutableListOf()
|
||||
val entries: MutableList<BackupMediaSnapshotTable.MediaEntry> = mutableListOf()
|
||||
|
||||
for (attachmentId in attachmentIds) {
|
||||
val attachment = SignalDatabase.attachments.getAttachment(attachmentId)!!
|
||||
val plaintextHash = attachment.dataHash!!.decodeBase64OrThrow()
|
||||
val remoteKey = attachment.remoteKey!!.decodeBase64OrThrow()
|
||||
val mediaId = MediaName.fromPlaintextHashAndRemoteKey(plaintextHash, remoteKey).toMediaId(SignalStore.backup.mediaRootBackupKey).encode()
|
||||
|
||||
mediaIds += mediaId
|
||||
entries += BackupMediaSnapshotTable.MediaEntry(mediaId = mediaId, cdn = 3, plaintextHash = plaintextHash, remoteKey = remoteKey, isThumbnail = false)
|
||||
}
|
||||
|
||||
SignalDatabase.backupMediaSnapshots.writePendingMediaEntries(entries)
|
||||
SignalDatabase.backupMediaSnapshots.commitPendingRows()
|
||||
|
||||
val snapshotVersion = SignalDatabase.backupMediaSnapshots.getCurrentSnapshotVersion()
|
||||
if (markSeenOnRemote) {
|
||||
SignalDatabase.backupMediaSnapshots.markSeenOnRemote(mediaIds, snapshotVersion)
|
||||
}
|
||||
|
||||
return snapshotVersion
|
||||
}
|
||||
|
||||
private fun isOffloaded(attachmentId: AttachmentId): Boolean {
|
||||
return SignalDatabase.attachments.getAttachment(attachmentId)!!.transferState == AttachmentTable.TRANSFER_RESTORE_OFFLOADED
|
||||
}
|
||||
|
||||
private fun commitSnapshotFor(attachmentId: AttachmentId, markSeenOnRemote: Boolean): Long {
|
||||
val attachment = SignalDatabase.attachments.getAttachment(attachmentId)!!
|
||||
val plaintextHash = attachment.dataHash!!.decodeBase64OrThrow()
|
||||
val remoteKey = attachment.remoteKey!!.decodeBase64OrThrow()
|
||||
val mediaId = MediaName.fromPlaintextHashAndRemoteKey(plaintextHash, remoteKey).toMediaId(SignalStore.backup.mediaRootBackupKey).encode()
|
||||
|
||||
SignalDatabase.backupMediaSnapshots.writePendingMediaEntries(
|
||||
listOf(BackupMediaSnapshotTable.MediaEntry(mediaId = mediaId, cdn = 3, plaintextHash = plaintextHash, remoteKey = remoteKey, isThumbnail = false))
|
||||
)
|
||||
SignalDatabase.backupMediaSnapshots.commitPendingRows()
|
||||
|
||||
val snapshotVersion = SignalDatabase.backupMediaSnapshots.getCurrentSnapshotVersion()
|
||||
if (markSeenOnRemote) {
|
||||
SignalDatabase.backupMediaSnapshots.markSeenOnRemote(listOf(mediaId), snapshotVersion)
|
||||
}
|
||||
|
||||
return snapshotVersion
|
||||
}
|
||||
|
||||
private fun createIncomingMessage(
|
||||
serverTime: Duration,
|
||||
attachment: Attachment,
|
||||
@@ -611,13 +821,13 @@ class AttachmentTableTest {
|
||||
)
|
||||
}
|
||||
|
||||
private fun createAttachmentPointer(key: ByteArray, size: Int): Attachment {
|
||||
private fun createAttachmentPointer(key: ByteArray, size: Int, contentType: String = MediaUtil.IMAGE_JPEG): Attachment {
|
||||
return PointerAttachment.forPointer(
|
||||
pointer = Optional.of(
|
||||
SignalServiceAttachmentPointer(
|
||||
cdnNumber = 3,
|
||||
remoteId = SignalServiceAttachmentRemoteId.V4("asdf"),
|
||||
contentType = MediaUtil.IMAGE_JPEG,
|
||||
contentType = contentType,
|
||||
key = key,
|
||||
size = Optional.of(size),
|
||||
preview = Optional.empty(),
|
||||
|
||||
+293
-5
@@ -6,16 +6,22 @@
|
||||
package org.thoughtcrime.securesms.jobs
|
||||
|
||||
import androidx.test.ext.junit.runners.AndroidJUnit4
|
||||
import arrow.core.left
|
||||
import arrow.core.right
|
||||
import assertk.assertThat
|
||||
import assertk.assertions.contains
|
||||
import assertk.assertions.doesNotContain
|
||||
import assertk.assertions.isEqualTo
|
||||
import assertk.assertions.isFalse
|
||||
import assertk.assertions.isGreaterThan
|
||||
import assertk.assertions.isNull
|
||||
import io.mockk.Runs
|
||||
import io.mockk.coEvery
|
||||
import io.mockk.coVerify
|
||||
import io.mockk.every
|
||||
import io.mockk.just
|
||||
import io.mockk.mockkObject
|
||||
import io.mockk.slot
|
||||
import io.mockk.unmockkAll
|
||||
import io.mockk.verify
|
||||
import org.junit.After
|
||||
@@ -27,10 +33,12 @@ import org.signal.core.models.backup.MediaName
|
||||
import org.signal.core.models.database.AttachmentId
|
||||
import org.signal.core.util.Base64.decodeBase64OrThrow
|
||||
import org.signal.network.api.ArchiveApiV2
|
||||
import org.signal.network.service.ArchiveError
|
||||
import org.signal.network.service.ArchiveService
|
||||
import org.thoughtcrime.securesms.attachments.ArchivedAttachment
|
||||
import org.thoughtcrime.securesms.attachments.Attachment
|
||||
import org.thoughtcrime.securesms.attachments.PointerAttachment
|
||||
import org.thoughtcrime.securesms.backup.v2.BackupRepository
|
||||
import org.thoughtcrime.securesms.backup.v2.ArchivedMediaObject
|
||||
import org.thoughtcrime.securesms.backup.v2.MessageBackupTier
|
||||
import org.thoughtcrime.securesms.database.AttachmentTable
|
||||
import org.thoughtcrime.securesms.database.BackupMediaSnapshotTable.MediaEntry
|
||||
@@ -44,7 +52,10 @@ import org.thoughtcrime.securesms.util.MediaUtil
|
||||
import org.whispersystems.signalservice.api.messages.SignalServiceAttachmentPointer
|
||||
import org.whispersystems.signalservice.api.messages.SignalServiceAttachmentRemoteId
|
||||
import java.io.ByteArrayInputStream
|
||||
import java.io.IOException
|
||||
import java.util.Optional
|
||||
import java.util.UUID
|
||||
import kotlin.random.Random
|
||||
import kotlin.time.Duration
|
||||
import kotlin.time.Duration.Companion.days
|
||||
|
||||
@@ -56,16 +67,18 @@ class ArchiveAttachmentReconciliationJobTest {
|
||||
|
||||
private val archiveService: ArchiveService = AppDependencies.archiveService
|
||||
|
||||
private val deletedFromCdn = slot<Set<ArchivedMediaObject>>()
|
||||
|
||||
@Before
|
||||
fun setUp() {
|
||||
SignalStore.backup.backupTier = MessageBackupTier.PAID
|
||||
SignalStore.backup.hasBackupBeenUploaded = true
|
||||
SignalStore.backup.lastAttachmentReconciliationTime = System.currentTimeMillis()
|
||||
SignalStore.backup.localRestoreReconcilePending = false
|
||||
SignalStore.backup.lastUsedMessageCutoffTime = 0
|
||||
|
||||
mockkObject(BackupRepository)
|
||||
mockkObject(ArchiveCommitAttachmentDeletesJob)
|
||||
coEvery { ArchiveCommitAttachmentDeletesJob.deleteMediaObjectsFromCdn(any(), any(), any(), any()) } returns null
|
||||
coEvery { ArchiveCommitAttachmentDeletesJob.deleteMediaObjectsFromCdn(any(), capture(deletedFromCdn), any(), any()) } returns null
|
||||
}
|
||||
|
||||
@After
|
||||
@@ -174,6 +187,36 @@ class ArchiveAttachmentReconciliationJobTest {
|
||||
assertThat(SignalDatabase.attachments.getAttachment(attachmentId)!!.archiveTransferState).isEqualTo(AttachmentTable.ArchiveTransferState.FINISHED)
|
||||
}
|
||||
|
||||
/**
|
||||
* The over-size-limit cutoff leaves old messages out of the export, so nothing in the backup references their media any more. Reconciliation applies the same
|
||||
* rule as [ArchiveCommitAttachmentDeletesJob] so the two reclaim paths can't disagree about what still counts as referenced.
|
||||
*/
|
||||
@Test
|
||||
fun givenCdnMediaWhoseMessagePredatesTheMessageCutoff_whenIReconcile_thenIExpectItDeletedFromTheCdn() {
|
||||
val attachmentId = seedFinalizedAttachment("remote-key-cutoff".toByteArray(), byteArrayOf(21, 22, 23, 24, 25), receivedAt = 10.days)
|
||||
SignalStore.backup.lastUsedMessageCutoffTime = 30.days.inWholeMilliseconds
|
||||
fakeCdnContains(attachmentId, cdn = 3)
|
||||
|
||||
ArchiveAttachmentReconciliationJob(forced = true).run()
|
||||
|
||||
assertThat(deletedFromCdn.captured.map { it.mediaId }).contains(mediaIdFor(attachmentId))
|
||||
}
|
||||
|
||||
/**
|
||||
* A cutoff being set at all must not weaken the protection for media the backup still references, which is what keeps a lost snapshot table from wiping the
|
||||
* archive.
|
||||
*/
|
||||
@Test
|
||||
fun givenCdnMediaWhoseMessageIsWithinTheMessageCutoff_whenIReconcile_thenIExpectItProtected() {
|
||||
val attachmentId = seedFinalizedAttachment("remote-key-recent".toByteArray(), byteArrayOf(26, 27, 28, 29, 30), receivedAt = 40.days)
|
||||
SignalStore.backup.lastUsedMessageCutoffTime = 30.days.inWholeMilliseconds
|
||||
fakeCdnContains(attachmentId, cdn = 3)
|
||||
|
||||
ArchiveAttachmentReconciliationJob(forced = true).run()
|
||||
|
||||
coVerify(exactly = 0) { ArchiveCommitAttachmentDeletesJob.deleteMediaObjectsFromCdn(any(), any(), any(), any()) }
|
||||
}
|
||||
|
||||
@Test
|
||||
fun givenFirstEverReconciliation_whenIForceIt_thenItStillRunsAndRepairs() {
|
||||
SignalStore.backup.lastAttachmentReconciliationTime = -1
|
||||
@@ -198,9 +241,254 @@ class ArchiveAttachmentReconciliationJobTest {
|
||||
assertThat(SignalStore.backup.localRestoreReconcilePending).isFalse()
|
||||
}
|
||||
|
||||
private fun seedFinalizedAttachment(remoteKey: ByteArray, data: ByteArray): AttachmentId {
|
||||
/**
|
||||
* Media that was offloaded (or restored from a backup) has no local data file, so resetting it to NONE cannot lead to a re-upload. It only makes the export
|
||||
* emit a tombstone, which drops the locator and marks the media for deletion off of the CDN. It must stay FINISHED.
|
||||
*/
|
||||
@Test
|
||||
fun givenFinishedMediaMissingFromCdnWithNoLocalDataFile_whenIReconcile_thenItStaysFinishedAndKeepsItsCdn() {
|
||||
val attachmentId = seedArchivedAttachment()
|
||||
commitSnapshotFor(attachmentId, cdn = 3)
|
||||
fakeCdnEmpty()
|
||||
|
||||
ArchiveAttachmentReconciliationJob(forced = true).run()
|
||||
|
||||
val after = SignalDatabase.attachments.getAttachment(attachmentId)!!
|
||||
assertThat(after.archiveTransferState).isEqualTo(AttachmentTable.ArchiveTransferState.FINISHED)
|
||||
assertThat(after.archiveCdn).isEqualTo(3)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun givenACrawlThatCompletes_whenIReconcile_thenIExpectTheCompletedSnapshotVersionRecorded() {
|
||||
SignalStore.backup.lastCompletedReconciliationSnapshotVersion = -1
|
||||
|
||||
val attachmentId = seedFinalizedAttachment("remote-key-recorded".toByteArray(), byteArrayOf(1, 2, 3, 4, 5))
|
||||
commitSnapshotFor(attachmentId, cdn = 3)
|
||||
fakeCdnContains(attachmentId, cdn = 3)
|
||||
|
||||
ArchiveAttachmentReconciliationJob(forced = true).run()
|
||||
|
||||
assertThat(SignalStore.backup.lastCompletedReconciliationSnapshotVersion).isEqualTo(SignalDatabase.backupMediaSnapshots.getCurrentSnapshotVersion())
|
||||
}
|
||||
|
||||
/**
|
||||
* Offloading trusts the recorded version as proof the server confirmed our media, so a crawl that dies part way through must leave it untouched rather than
|
||||
* recording a version it never finished verifying.
|
||||
*/
|
||||
@Test
|
||||
fun givenACrawlThatFailsPartWayThrough_whenIReconcile_thenIExpectNoCompletedSnapshotVersionRecorded() {
|
||||
SignalStore.backup.lastCompletedReconciliationSnapshotVersion = -1
|
||||
|
||||
val attachmentId = seedFinalizedAttachment("remote-key-failed".toByteArray(), byteArrayOf(2, 3, 4, 5, 6))
|
||||
commitSnapshotFor(attachmentId, cdn = 3)
|
||||
coEvery { archiveService.listRemoteMediaObjects(any(), any()) } returns ArchiveError.NetworkError(IOException("boom")).left()
|
||||
|
||||
val result = ArchiveAttachmentReconciliationJob(forced = true).run()
|
||||
|
||||
assertThat(result.isSuccess).isFalse()
|
||||
assertThat(SignalStore.backup.lastCompletedReconciliationSnapshotVersion).isEqualTo(-1L)
|
||||
}
|
||||
|
||||
/**
|
||||
* A completed crawl proves the object isn't on the CDN, so the row is bookkeeping for something that no longer exists and there is nothing left to orphan.
|
||||
*/
|
||||
@Test
|
||||
fun givenOldSnapshotMediaAbsentFromCdn_whenIReconcile_thenIExpectTheRowPruned() {
|
||||
val absentMediaId = commitRandomSnapshotEntry()
|
||||
commitRandomSnapshotEntry()
|
||||
fakeCdnEmpty()
|
||||
|
||||
ArchiveAttachmentReconciliationJob(forced = true).run()
|
||||
|
||||
assertThat(oldSnapshotMediaIds()).doesNotContain(absentMediaId)
|
||||
}
|
||||
|
||||
/**
|
||||
* A crawl confirmed this object once, so a later crawl failing to list it is not enough to conclude it's gone. An upload landing mid-crawl produces the same
|
||||
* evidence. Pruning here would contradict our own earlier proof and orphan the object.
|
||||
*/
|
||||
@Test
|
||||
fun givenOldSnapshotMediaConfirmedByAnEarlierCrawl_whenIReconcile_thenIExpectTheRowKept() {
|
||||
val previouslySeenMediaId = commitRandomSnapshotEntry()
|
||||
SignalDatabase.backupMediaSnapshots.markSeenOnRemote(listOf(previouslySeenMediaId), 1)
|
||||
commitRandomSnapshotEntry()
|
||||
fakeCdnEmpty()
|
||||
|
||||
ArchiveAttachmentReconciliationJob(forced = true).run()
|
||||
|
||||
assertThat(oldSnapshotMediaIds()).contains(previouslySeenMediaId)
|
||||
}
|
||||
|
||||
/** The crawl found the object, so this row is the only thing tracking it and must survive even though it left the latest snapshot. */
|
||||
@Test
|
||||
fun givenOldSnapshotMediaStillOnCdn_whenIReconcile_thenIExpectTheRowKept() {
|
||||
val attachmentId = seedFinalizedAttachment("remote-key-kept".toByteArray(), byteArrayOf(3, 1, 4, 1, 5))
|
||||
commitSnapshotFor(attachmentId, cdn = 3)
|
||||
commitRandomSnapshotEntry()
|
||||
fakeCdnContains(attachmentId, cdn = 3)
|
||||
|
||||
ArchiveAttachmentReconciliationJob(forced = true).run()
|
||||
|
||||
assertThat(oldSnapshotMediaIds()).contains(mediaIdFor(attachmentId))
|
||||
}
|
||||
|
||||
/**
|
||||
* The forced-crawl rate limit must only be consumed by a crawl that actually starts, otherwise a job that returns early leaves offloading blocked for a full
|
||||
* interval without having verified anything.
|
||||
*/
|
||||
@Test
|
||||
fun givenNoBackupUploaded_whenIForceAReconciliation_thenIExpectTheAttemptTimeUnchanged() {
|
||||
SignalStore.backup.hasBackupBeenUploaded = false
|
||||
SignalStore.backup.lastForcedReconciliationAttemptTime = 0
|
||||
fakeCdnEmpty()
|
||||
|
||||
ArchiveAttachmentReconciliationJob(forced = true).run()
|
||||
|
||||
assertThat(SignalStore.backup.lastForcedReconciliationAttemptTime).isEqualTo(0L)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun givenACrawlThatStarts_whenIForceAReconciliation_thenIExpectTheAttemptTimeAdvanced() {
|
||||
SignalStore.backup.lastForcedReconciliationAttemptTime = 0
|
||||
commitRandomSnapshotEntry()
|
||||
fakeCdnEmpty()
|
||||
|
||||
ArchiveAttachmentReconciliationJob(forced = true).run()
|
||||
|
||||
assertThat(SignalStore.backup.lastForcedReconciliationAttemptTime).isGreaterThan(0L)
|
||||
}
|
||||
|
||||
/**
|
||||
* A crawl with no snapshot to compare against can't confirm anything, so it must not consume the forced-attempt budget. Spending it here would rate-limit the
|
||||
* retry that becomes useful once a backup has built a snapshot.
|
||||
*/
|
||||
@Test
|
||||
fun givenNoSnapshotYet_whenIForceAReconciliation_thenIExpectTheAttemptTimeUnchanged() {
|
||||
SignalStore.backup.lastForcedReconciliationAttemptTime = 0
|
||||
fakeCdnEmpty()
|
||||
|
||||
ArchiveAttachmentReconciliationJob(forced = true).run()
|
||||
|
||||
assertThat(SignalStore.backup.lastForcedReconciliationAttemptTime).isEqualTo(0L)
|
||||
}
|
||||
|
||||
/**
|
||||
* Same reasoning for the offload gate: a snapshot-less crawl verified nothing, so it must not record a completed version that would let us start deleting
|
||||
* local copies of media.
|
||||
*/
|
||||
@Test
|
||||
fun givenNoSnapshotYet_whenIReconcile_thenIExpectNoCompletedSnapshotVersionRecorded() {
|
||||
SignalStore.backup.lastCompletedReconciliationSnapshotVersion = -1
|
||||
fakeCdnEmpty()
|
||||
|
||||
ArchiveAttachmentReconciliationJob(forced = true).run()
|
||||
|
||||
assertThat(SignalStore.backup.lastCompletedReconciliationSnapshotVersion).isEqualTo(-1L)
|
||||
}
|
||||
|
||||
// TODO [cody] return after fixing perf problem of restoring thumbnail state
|
||||
// Commented out alongside the skipped thumbnail promotion. See the TODO in ArchiveAttachmentReconciliationJob.syncCdnPage.
|
||||
//
|
||||
// /**
|
||||
// * A restore sets the full-size state from the backup's CDN claim but leaves the thumbnail state NONE, so the listing is the only thing that can promote it.
|
||||
// * Until it does, the thumbnail is dropped from the snapshot on every backup and gets needlessly re-uploaded once the media is downloaded.
|
||||
// */
|
||||
// @Test
|
||||
// fun givenRestoredMediaWhoseThumbnailIsOnTheCdn_whenIReconcile_thenIExpectTheThumbnailMarkedFinished() {
|
||||
// val attachmentId = seedArchivedAttachment()
|
||||
// assertThat(SignalDatabase.attachments.getArchiveThumbnailTransferState(attachmentId)).isEqualTo(AttachmentTable.ArchiveTransferState.NONE)
|
||||
//
|
||||
// fakeCdnContainsThumbnail(attachmentId, cdn = 3)
|
||||
//
|
||||
// ArchiveAttachmentReconciliationJob(forced = true).run()
|
||||
//
|
||||
// assertThat(SignalDatabase.attachments.getArchiveThumbnailTransferState(attachmentId)).isEqualTo(AttachmentTable.ArchiveTransferState.FINISHED)
|
||||
// }
|
||||
//
|
||||
// @Test
|
||||
// fun givenRestoredMediaWhoseThumbnailIsNotOnTheCdn_whenIReconcile_thenIExpectTheThumbnailLeftAlone() {
|
||||
// val attachmentId = seedArchivedAttachment()
|
||||
// fakeCdnEmpty()
|
||||
//
|
||||
// ArchiveAttachmentReconciliationJob(forced = true).run()
|
||||
//
|
||||
// assertThat(SignalDatabase.attachments.getArchiveThumbnailTransferState(attachmentId)).isEqualTo(AttachmentTable.ArchiveTransferState.NONE)
|
||||
// }
|
||||
|
||||
// private fun fakeCdnContainsThumbnail(attachmentId: AttachmentId, cdn: Int) {
|
||||
// val attachment = SignalDatabase.attachments.getAttachment(attachmentId)!!
|
||||
// val plaintextHash = attachment.dataHash!!.decodeBase64OrThrow()
|
||||
// val remoteKey = attachment.remoteKey!!.decodeBase64OrThrow()
|
||||
// val mediaId = MediaName.fromPlaintextHashAndRemoteKeyForThumbnail(plaintextHash, remoteKey).toMediaId(SignalStore.backup.mediaRootBackupKey).encode()
|
||||
//
|
||||
// coEvery { archiveService.listRemoteMediaObjects(any(), any()) } returns ArchiveApiV2.MediaItemsPage(
|
||||
// storedMediaObjects = listOf(ArchiveApiV2.StoredMediaObject(cdn = cdn, mediaId = mediaId, objectLength = attachment.size)),
|
||||
// cursor = null
|
||||
// ).right()
|
||||
// }
|
||||
|
||||
private fun oldSnapshotMediaIds(): List<String> {
|
||||
return SignalDatabase.backupMediaSnapshots.getPageOfOldMediaEntries(pageSize = 1_000).map { it.mediaId }
|
||||
}
|
||||
|
||||
private fun mediaIdFor(attachmentId: AttachmentId): String {
|
||||
val attachment = SignalDatabase.attachments.getAttachment(attachmentId)!!
|
||||
val plaintextHash = attachment.dataHash!!.decodeBase64OrThrow()
|
||||
val remoteKey = attachment.remoteKey!!.decodeBase64OrThrow()
|
||||
|
||||
return MediaName.fromPlaintextHashAndRemoteKey(plaintextHash, remoteKey).toMediaId(SignalStore.backup.mediaRootBackupKey).encode()
|
||||
}
|
||||
|
||||
/**
|
||||
* Committing a second entry is what pushes any previously committed row below MAX(snapshot_version), which is the "fell out of the latest snapshot" state.
|
||||
*/
|
||||
private fun commitRandomSnapshotEntry(): String {
|
||||
val plaintextHash = Random.nextBytes(32)
|
||||
val remoteKey = Random.nextBytes(32)
|
||||
val mediaId = MediaName.fromPlaintextHashAndRemoteKey(plaintextHash, remoteKey).toMediaId(SignalStore.backup.mediaRootBackupKey).encode()
|
||||
|
||||
SignalDatabase.backupMediaSnapshots.writePendingMediaEntries(
|
||||
listOf(MediaEntry(mediaId = mediaId, cdn = 3, plaintextHash = plaintextHash, remoteKey = remoteKey, isThumbnail = false))
|
||||
)
|
||||
SignalDatabase.backupMediaSnapshots.commitPendingRows()
|
||||
|
||||
return mediaId
|
||||
}
|
||||
|
||||
private fun seedArchivedAttachment(): AttachmentId {
|
||||
val attachment = ArchivedAttachment(
|
||||
contentType = MediaUtil.IMAGE_JPEG,
|
||||
size = 1024,
|
||||
cdn = 3,
|
||||
uploadTimestamp = 0,
|
||||
key = Random.nextBytes(8),
|
||||
cdnKey = "password",
|
||||
archiveCdn = 3,
|
||||
plaintextHash = Random.nextBytes(8),
|
||||
incrementalMac = Random.nextBytes(8),
|
||||
incrementalMacChunkSize = 8,
|
||||
width = 100,
|
||||
height = 100,
|
||||
caption = null,
|
||||
blurHash = null,
|
||||
voiceNote = false,
|
||||
borderless = false,
|
||||
stickerLocator = null,
|
||||
gif = false,
|
||||
quote = false,
|
||||
quoteTargetContentType = null,
|
||||
uuid = UUID.randomUUID(),
|
||||
fileName = null,
|
||||
localBackupKey = null
|
||||
)
|
||||
|
||||
val messageId = SignalDatabase.messages.insertMessageInbox(createIncomingMessage(serverTime = 0.days, attachment = attachment)).get().messageId
|
||||
return SignalDatabase.attachments.getAttachmentsForMessage(messageId).first().attachmentId
|
||||
}
|
||||
|
||||
private fun seedFinalizedAttachment(remoteKey: ByteArray, data: ByteArray, receivedAt: Duration = 0.days): AttachmentId {
|
||||
val attachment = createAttachmentPointer(remoteKey, data.size)
|
||||
val messageResult = SignalDatabase.messages.insertMessageInbox(createIncomingMessage(serverTime = 0.days, attachment = attachment)).get()
|
||||
val messageResult = SignalDatabase.messages.insertMessageInbox(createIncomingMessage(serverTime = receivedAt, attachment = attachment)).get()
|
||||
val attachmentId = messageResult.insertedAttachments!![attachment]!!
|
||||
SignalDatabase.attachments.setTransferState(messageResult.messageId, attachmentId, AttachmentTable.TRANSFER_PROGRESS_STARTED)
|
||||
SignalDatabase.attachments.finalizeAttachmentAfterDownload(messageResult.messageId, attachmentId, ByteArrayInputStream(data))
|
||||
|
||||
+411
@@ -0,0 +1,411 @@
|
||||
/*
|
||||
* Copyright 2026 Signal Messenger, LLC
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
package org.thoughtcrime.securesms.jobs
|
||||
|
||||
import androidx.test.ext.junit.runners.AndroidJUnit4
|
||||
import assertk.assertThat
|
||||
import assertk.assertions.contains
|
||||
import assertk.assertions.doesNotContain
|
||||
import assertk.assertions.hasSize
|
||||
import assertk.assertions.isEmpty
|
||||
import assertk.assertions.isEqualTo
|
||||
import io.mockk.coEvery
|
||||
import io.mockk.coVerify
|
||||
import io.mockk.mockkObject
|
||||
import io.mockk.slot
|
||||
import io.mockk.unmockkAll
|
||||
import org.junit.After
|
||||
import org.junit.Before
|
||||
import org.junit.Rule
|
||||
import org.junit.Test
|
||||
import org.junit.runner.RunWith
|
||||
import org.signal.core.models.backup.MediaName
|
||||
import org.signal.core.models.database.AttachmentId
|
||||
import org.signal.core.util.Base64.decodeBase64OrThrow
|
||||
import org.thoughtcrime.securesms.attachments.Attachment
|
||||
import org.thoughtcrime.securesms.attachments.PointerAttachment
|
||||
import org.thoughtcrime.securesms.backup.v2.ArchivedMediaObject
|
||||
import org.thoughtcrime.securesms.backup.v2.MessageBackupTier
|
||||
import org.thoughtcrime.securesms.database.AttachmentTable
|
||||
import org.thoughtcrime.securesms.database.BackupMediaSnapshotTable.MediaEntry
|
||||
import org.thoughtcrime.securesms.database.MessageType
|
||||
import org.thoughtcrime.securesms.database.SignalDatabase
|
||||
import org.thoughtcrime.securesms.keyvalue.SignalStore
|
||||
import org.thoughtcrime.securesms.mms.IncomingMessage
|
||||
import org.thoughtcrime.securesms.testing.SignalActivityRule
|
||||
import org.thoughtcrime.securesms.util.MediaUtil
|
||||
import org.whispersystems.signalservice.api.messages.SignalServiceAttachmentPointer
|
||||
import org.whispersystems.signalservice.api.messages.SignalServiceAttachmentRemoteId
|
||||
import java.io.ByteArrayInputStream
|
||||
import java.util.Optional
|
||||
import kotlin.random.Random
|
||||
import kotlin.time.Duration
|
||||
import kotlin.time.Duration.Companion.days
|
||||
|
||||
@RunWith(AndroidJUnit4::class)
|
||||
class ArchiveCommitAttachmentDeletesJobTest {
|
||||
|
||||
@get:Rule
|
||||
val harness = SignalActivityRule()
|
||||
|
||||
private val deletedFromCdn = slot<Set<ArchivedMediaObject>>()
|
||||
|
||||
private val jobPageSize = ArchiveCommitAttachmentDeletesJob.REMOTE_DELETE_BATCH_SIZE
|
||||
|
||||
@Before
|
||||
fun setUp() {
|
||||
SignalStore.backup.backupTier = MessageBackupTier.PAID
|
||||
SignalStore.backup.lastUsedMessageCutoffTime = 0
|
||||
|
||||
mockkObject(ArchiveCommitAttachmentDeletesJob)
|
||||
coEvery { ArchiveCommitAttachmentDeletesJob.deleteMediaObjectsFromCdn(any(), capture(deletedFromCdn), any(), any()) } returns null
|
||||
}
|
||||
|
||||
@After
|
||||
fun tearDown() {
|
||||
unmockkAll()
|
||||
}
|
||||
|
||||
/**
|
||||
* Media only leaves the snapshot because the backup stopped referencing it, which a bookkeeping bug can cause while the attachment is still very much present.
|
||||
* Deleting on that basis alone is how the only remaining copy of media gets destroyed, so the pass has to discriminate within a page rather than trusting the
|
||||
* snapshot.
|
||||
*/
|
||||
@Test
|
||||
fun givenOldSnapshotMediaBothReferencedAndOrphaned_whenIRun_thenIExpectOnlyTheOrphanDeletedFromTheCdn() {
|
||||
val attachmentId = seedFinalizedAttachment(byteArrayOf(1, 2, 3, 4, 5))
|
||||
val referenced = entryFor(attachmentId)
|
||||
val orphan = randomEntry()
|
||||
|
||||
commit(referenced, orphan)
|
||||
commit(randomEntry())
|
||||
|
||||
ArchiveCommitAttachmentDeletesJob().run()
|
||||
|
||||
val deletedMediaIds = deletedFromCdn.captured.map { it.mediaId }
|
||||
assertThat(deletedMediaIds).contains(orphan.mediaId)
|
||||
assertThat(deletedMediaIds).doesNotContain(referenced.mediaId)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun givenOnlyStillReferencedOldSnapshotMedia_whenIRun_thenIExpectNothingDeletedFromTheCdn() {
|
||||
val attachmentId = seedFinalizedAttachment(byteArrayOf(9, 8, 7, 6, 5))
|
||||
|
||||
commit(entryFor(attachmentId))
|
||||
commit(randomEntry())
|
||||
|
||||
ArchiveCommitAttachmentDeletesJob().run()
|
||||
|
||||
coVerify(exactly = 0) { ArchiveCommitAttachmentDeletesJob.deleteMediaObjectsFromCdn(any(), any(), any(), any()) }
|
||||
}
|
||||
|
||||
/**
|
||||
* A retained row is the only record that its CDN object exists, so pruning it would orphan the object permanently. The run still has to terminate, which it
|
||||
* does by paging past retained rows by id.
|
||||
*/
|
||||
@Test
|
||||
fun givenOnlyStillReferencedOldSnapshotMedia_whenIRun_thenIExpectTheSnapshotRowsRetained() {
|
||||
val attachmentId = seedFinalizedAttachment(byteArrayOf(4, 4, 4, 4, 4))
|
||||
val referenced = entryFor(attachmentId)
|
||||
|
||||
commit(referenced)
|
||||
commit(randomEntry())
|
||||
|
||||
ArchiveCommitAttachmentDeletesJob().run()
|
||||
|
||||
assertThat(remainingOldMediaIds()).contains(referenced.mediaId)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun givenOldSnapshotMediaWithNoAttachment_whenIRun_thenIExpectTheSnapshotRowPruned() {
|
||||
val orphan = randomEntry()
|
||||
|
||||
commit(orphan)
|
||||
commit(randomEntry())
|
||||
|
||||
ArchiveCommitAttachmentDeletesJob().run()
|
||||
|
||||
assertThat(remainingOldMediaIds()).doesNotContain(orphan.mediaId)
|
||||
}
|
||||
|
||||
/**
|
||||
* Coverage across pages: if the loop only ever handled the first page, the overflow rows would stay on the CDN forever with nothing left to reconsider them.
|
||||
*/
|
||||
@Test
|
||||
fun givenMoreOrphansThanOnePage_whenIRun_thenIExpectEveryPageProcessed() {
|
||||
val orphans = List(jobPageSize + 1) { randomEntry() }
|
||||
|
||||
commit(orphans)
|
||||
commit(randomEntry())
|
||||
|
||||
ArchiveCommitAttachmentDeletesJob().run()
|
||||
|
||||
assertThat(remainingOldMediaIds()).isEmpty()
|
||||
}
|
||||
|
||||
/**
|
||||
* Termination when a whole page is retained. Nothing is removed from the table here, so the loop can only end by advancing its cursor past rows it declined to
|
||||
* delete. If it paged from the start each time it would spin forever and this test would hang rather than fail.
|
||||
*/
|
||||
@Test
|
||||
fun givenMoreUndeletableRowsThanOnePage_whenIRun_thenIExpectItToTerminateAndRetainThem() {
|
||||
val unknownCdn = List(jobPageSize + 1) { randomEntry(cdn = null) }
|
||||
|
||||
commit(unknownCdn)
|
||||
commit(randomEntry())
|
||||
|
||||
ArchiveCommitAttachmentDeletesJob().run()
|
||||
|
||||
assertThat(remainingOldMediaIds()).hasSize(unknownCdn.size)
|
||||
coVerify(exactly = 0) { ArchiveCommitAttachmentDeletesJob.deleteMediaObjectsFromCdn(any(), any(), any(), any()) }
|
||||
}
|
||||
|
||||
/**
|
||||
* Users over the backup size limit get old messages left out of the export entirely, so nothing in the backup references their media any more and it should
|
||||
* stop occupying paid archive quota. Before this, an attachment row existing was enough to retain it forever.
|
||||
*/
|
||||
@Test
|
||||
fun givenMediaWhoseMessagePredatesTheMessageCutoff_whenIRun_thenIExpectItDeletedFromTheCdn() {
|
||||
val attachmentId = seedFinalizedAttachment(byteArrayOf(1, 1, 2, 3, 5), receivedAt = 10.days)
|
||||
val agedOut = entryFor(attachmentId)
|
||||
SignalStore.backup.lastUsedMessageCutoffTime = 30.days.inWholeMilliseconds
|
||||
|
||||
commit(agedOut)
|
||||
commit(randomEntry())
|
||||
|
||||
ArchiveCommitAttachmentDeletesJob().run()
|
||||
|
||||
assertThat(deletedFromCdn.captured.map { it.mediaId }).contains(agedOut.mediaId)
|
||||
}
|
||||
|
||||
/**
|
||||
* The exporter includes messages on `date_received >= cutoff`, so one landing exactly on the threshold is still in the backup. Deleting it would take media
|
||||
* the backup still references.
|
||||
*/
|
||||
@Test
|
||||
fun givenMediaWhoseMessageIsExactlyAtTheMessageCutoff_whenIRun_thenIExpectItRetained() {
|
||||
val attachmentId = seedFinalizedAttachment(byteArrayOf(2, 2, 2, 2, 2), receivedAt = 30.days)
|
||||
val boundary = entryFor(attachmentId)
|
||||
SignalStore.backup.lastUsedMessageCutoffTime = 30.days.inWholeMilliseconds
|
||||
|
||||
commit(boundary)
|
||||
commit(randomEntry())
|
||||
|
||||
ArchiveCommitAttachmentDeletesJob().run()
|
||||
|
||||
assertThat(remainingOldMediaIds()).contains(boundary.mediaId)
|
||||
coVerify(exactly = 0) { ArchiveCommitAttachmentDeletesJob.deleteMediaObjectsFromCdn(any(), any(), any(), any()) }
|
||||
}
|
||||
|
||||
/**
|
||||
* A wallpaper has no message row at all. Phrasing the cutoff as "its message is recent enough" would drop wallpapers out of the referent set and delete media
|
||||
* the user still has applied, so the check only disqualifies an attachment whose message exists and is too old.
|
||||
*/
|
||||
@Test
|
||||
fun givenAWallpaperAndAMessageCutoff_whenIRun_thenIExpectItRetained() {
|
||||
val attachmentId = SignalDatabase.attachments.insertWallpaper(ByteArrayInputStream(byteArrayOf(7, 7, 7, 7, 7)))
|
||||
val wallpaper = entryFor(attachmentId)
|
||||
SignalStore.backup.lastUsedMessageCutoffTime = 30.days.inWholeMilliseconds
|
||||
|
||||
commit(wallpaper)
|
||||
commit(randomEntry())
|
||||
|
||||
ArchiveCommitAttachmentDeletesJob().run()
|
||||
|
||||
assertThat(remainingOldMediaIds()).contains(wallpaper.mediaId)
|
||||
coVerify(exactly = 0) { ArchiveCommitAttachmentDeletesJob.deleteMediaObjectsFromCdn(any(), any(), any(), any()) }
|
||||
}
|
||||
|
||||
/**
|
||||
* Cutting off messages only happens for backups over the size limit, which is vanishingly rare, so the ordinary case has to behave exactly as it did before.
|
||||
*/
|
||||
@Test
|
||||
fun givenNoMessageCutoff_whenIRun_thenIExpectAncientMediaRetained() {
|
||||
val attachmentId = seedFinalizedAttachment(byteArrayOf(3, 3, 3, 3, 3), receivedAt = 0.days)
|
||||
val ancient = entryFor(attachmentId)
|
||||
|
||||
commit(ancient)
|
||||
commit(randomEntry())
|
||||
|
||||
ArchiveCommitAttachmentDeletesJob().run()
|
||||
|
||||
assertThat(remainingOldMediaIds()).contains(ancient.mediaId)
|
||||
coVerify(exactly = 0) { ArchiveCommitAttachmentDeletesJob.deleteMediaObjectsFromCdn(any(), any(), any(), any()) }
|
||||
}
|
||||
|
||||
/**
|
||||
* Media is keyed by hash and remote key, so the same bytes can hang off several messages. One of them surviving the cutoff means the backup still references
|
||||
* the media, and it has to be judged referenced rather than per-row.
|
||||
*/
|
||||
@Test
|
||||
fun givenMediaReferencedByBothAnAgedOutAndACurrentMessage_whenIRun_thenIExpectItRetained() {
|
||||
val data = byteArrayOf(4, 5, 6, 7, 8)
|
||||
val agedOutId = seedFinalizedAttachment(data, receivedAt = 10.days)
|
||||
val currentId = seedFinalizedAttachment(data, receivedAt = 40.days)
|
||||
|
||||
val shared = entryFor(agedOutId)
|
||||
assertThat(entryFor(currentId).mediaId).isEqualTo(shared.mediaId)
|
||||
|
||||
SignalStore.backup.lastUsedMessageCutoffTime = 30.days.inWholeMilliseconds
|
||||
|
||||
commit(shared)
|
||||
commit(randomEntry())
|
||||
|
||||
ArchiveCommitAttachmentDeletesJob().run()
|
||||
|
||||
assertThat(remainingOldMediaIds()).contains(shared.mediaId)
|
||||
coVerify(exactly = 0) { ArchiveCommitAttachmentDeletesJob.deleteMediaObjectsFromCdn(any(), any(), any(), any()) }
|
||||
}
|
||||
|
||||
/**
|
||||
* Full-size and thumbnail media share a hash and key but are judged by different rules, so a thumbnail can be reclaimable while its full-size copy is not.
|
||||
*/
|
||||
@Test
|
||||
fun givenAThumbnailWhoseAttachmentStillWantsIt_whenIRun_thenIExpectItRetained() {
|
||||
val attachmentId = seedFinalizedAttachment(byteArrayOf(5, 1, 5, 1, 5))
|
||||
val thumbnail = entryFor(attachmentId, isThumbnail = true)
|
||||
|
||||
commit(thumbnail)
|
||||
commit(randomEntry())
|
||||
|
||||
ArchiveCommitAttachmentDeletesJob().run()
|
||||
|
||||
assertThat(remainingOldMediaIds()).contains(thumbnail.mediaId)
|
||||
coVerify(exactly = 0) { ArchiveCommitAttachmentDeletesJob.deleteMediaObjectsFromCdn(any(), any(), any(), any()) }
|
||||
}
|
||||
|
||||
/**
|
||||
* Wallpapers never get a thumbnail written into the snapshot, so one already on the CDN is unreachable bookkeeping. Its full-size object still has a referent
|
||||
* and has to survive, which is what makes this a thumbnail-only reclaim rather than a blanket delete.
|
||||
*/
|
||||
@Test
|
||||
fun givenAWallpaperThumbnail_whenIRun_thenIExpectOnlyTheThumbnailDeletedFromTheCdn() {
|
||||
val attachmentId = SignalDatabase.attachments.insertWallpaper(ByteArrayInputStream(byteArrayOf(6, 2, 6, 2, 6)))
|
||||
val thumbnail = entryFor(attachmentId, isThumbnail = true)
|
||||
val fullSize = entryFor(attachmentId)
|
||||
|
||||
commit(thumbnail, fullSize)
|
||||
commit(randomEntry())
|
||||
|
||||
ArchiveCommitAttachmentDeletesJob().run()
|
||||
|
||||
val deletedMediaIds = deletedFromCdn.captured.map { it.mediaId }
|
||||
assertThat(deletedMediaIds).contains(thumbnail.mediaId)
|
||||
assertThat(deletedMediaIds).doesNotContain(fullSize.mediaId)
|
||||
}
|
||||
|
||||
/**
|
||||
* The message cutoff has to reach the thumbnail path too, otherwise media dropped from the backup keeps half of its CDN footprint.
|
||||
*/
|
||||
@Test
|
||||
fun givenAThumbnailWhoseMessagePredatesTheMessageCutoff_whenIRun_thenIExpectItDeletedFromTheCdn() {
|
||||
val attachmentId = seedFinalizedAttachment(byteArrayOf(7, 3, 7, 3, 7), receivedAt = 10.days)
|
||||
val thumbnail = entryFor(attachmentId, isThumbnail = true)
|
||||
SignalStore.backup.lastUsedMessageCutoffTime = 30.days.inWholeMilliseconds
|
||||
|
||||
commit(thumbnail)
|
||||
commit(randomEntry())
|
||||
|
||||
ArchiveCommitAttachmentDeletesJob().run()
|
||||
|
||||
assertThat(deletedFromCdn.captured.map { it.mediaId }).contains(thumbnail.mediaId)
|
||||
}
|
||||
|
||||
private fun entryFor(attachmentId: AttachmentId, isThumbnail: Boolean = false): MediaEntry {
|
||||
val attachment = SignalDatabase.attachments.getAttachment(attachmentId)!!
|
||||
val plaintextHash = attachment.dataHash!!.decodeBase64OrThrow()
|
||||
val remoteKey = attachment.remoteKey!!.decodeBase64OrThrow()
|
||||
|
||||
val mediaName = if (isThumbnail) {
|
||||
MediaName.fromPlaintextHashAndRemoteKeyForThumbnail(plaintextHash, remoteKey)
|
||||
} else {
|
||||
MediaName.fromPlaintextHashAndRemoteKey(plaintextHash, remoteKey)
|
||||
}
|
||||
|
||||
return MediaEntry(
|
||||
mediaId = mediaName.toMediaId(SignalStore.backup.mediaRootBackupKey).encode(),
|
||||
cdn = 3,
|
||||
plaintextHash = plaintextHash,
|
||||
remoteKey = remoteKey,
|
||||
isThumbnail = isThumbnail
|
||||
)
|
||||
}
|
||||
|
||||
private fun randomEntry(cdn: Int? = 3): MediaEntry {
|
||||
val plaintextHash = Random.nextBytes(32)
|
||||
val remoteKey = Random.nextBytes(32)
|
||||
|
||||
return MediaEntry(
|
||||
mediaId = MediaName.fromPlaintextHashAndRemoteKey(plaintextHash, remoteKey).toMediaId(SignalStore.backup.mediaRootBackupKey).encode(),
|
||||
cdn = cdn,
|
||||
plaintextHash = plaintextHash,
|
||||
remoteKey = remoteKey,
|
||||
isThumbnail = false
|
||||
)
|
||||
}
|
||||
|
||||
private fun commit(vararg entries: MediaEntry) {
|
||||
commit(entries.toList())
|
||||
}
|
||||
|
||||
private fun commit(entries: List<MediaEntry>) {
|
||||
SignalDatabase.backupMediaSnapshots.writePendingMediaEntries(entries)
|
||||
SignalDatabase.backupMediaSnapshots.commitPendingRows()
|
||||
}
|
||||
|
||||
private fun remainingOldMediaIds(): List<String> {
|
||||
return SignalDatabase.backupMediaSnapshots.getPageOfOldMediaEntries(pageSize = (jobPageSize * 3)).map { it.mediaId }
|
||||
}
|
||||
|
||||
private fun seedFinalizedAttachment(data: ByteArray, receivedAt: Duration = 0.days): AttachmentId {
|
||||
val attachment = createAttachmentPointer(Random.nextBytes(32), data.size)
|
||||
val messageResult = SignalDatabase.messages.insertMessageInbox(createIncomingMessage(serverTime = receivedAt, attachment = attachment)).get()
|
||||
val attachmentId = messageResult.insertedAttachments!![attachment]!!
|
||||
SignalDatabase.attachments.setTransferState(messageResult.messageId, attachmentId, AttachmentTable.TRANSFER_PROGRESS_STARTED)
|
||||
SignalDatabase.attachments.finalizeAttachmentAfterDownload(messageResult.messageId, attachmentId, ByteArrayInputStream(data))
|
||||
return attachmentId
|
||||
}
|
||||
|
||||
private fun createIncomingMessage(serverTime: Duration, attachment: Attachment): IncomingMessage {
|
||||
return IncomingMessage(
|
||||
type = MessageType.NORMAL,
|
||||
from = harness.others[0],
|
||||
body = null,
|
||||
sentTimeMillis = serverTime.inWholeMilliseconds,
|
||||
serverTimeMillis = serverTime.inWholeMilliseconds,
|
||||
receivedTimeMillis = serverTime.inWholeMilliseconds,
|
||||
attachments = listOf(attachment)
|
||||
)
|
||||
}
|
||||
|
||||
private fun createAttachmentPointer(key: ByteArray, size: Int): Attachment {
|
||||
return PointerAttachment.forPointer(
|
||||
pointer = Optional.of(
|
||||
SignalServiceAttachmentPointer(
|
||||
cdnNumber = 3,
|
||||
remoteId = SignalServiceAttachmentRemoteId.V4("asdf"),
|
||||
contentType = MediaUtil.IMAGE_JPEG,
|
||||
key = key,
|
||||
size = Optional.of(size),
|
||||
preview = Optional.empty(),
|
||||
width = 2,
|
||||
height = 2,
|
||||
digest = Optional.of(byteArrayOf()),
|
||||
incrementalDigest = Optional.empty(),
|
||||
incrementalMacChunkSize = 0,
|
||||
fileName = Optional.of("file.jpg"),
|
||||
voiceNote = false,
|
||||
isBorderless = false,
|
||||
isGif = false,
|
||||
caption = Optional.empty(),
|
||||
blurHash = Optional.empty(),
|
||||
uploadTimestamp = 0,
|
||||
uuid = null
|
||||
)
|
||||
)
|
||||
).get()
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user