mirror of
https://github.com/signalapp/Signal-Android.git
synced 2026-09-13 12:07:57 +01:00
Improve logic around optimize media selection.
Use snapshot table to inform optimize eligibility.
This commit is contained in:
Cody Henthorne
+212
-2
@@ -9,6 +9,7 @@ import assertk.assertThat
|
||||
import assertk.assertions.hasSize
|
||||
import assertk.assertions.isEmpty
|
||||
import assertk.assertions.isEqualTo
|
||||
import assertk.assertions.isFalse
|
||||
import assertk.assertions.isNotEmpty
|
||||
import assertk.assertions.isNotEqualTo
|
||||
import assertk.assertions.isNull
|
||||
@@ -27,6 +28,7 @@ import org.signal.core.util.Base64
|
||||
import org.signal.core.util.Base64.decodeBase64OrThrow
|
||||
import org.signal.core.util.copyTo
|
||||
import org.signal.core.util.stream.NullOutputStream
|
||||
import org.signal.core.util.update
|
||||
import org.signal.mediasend.SentMediaQuality
|
||||
import org.thoughtcrime.securesms.attachments.ArchivedAttachment
|
||||
import org.thoughtcrime.securesms.attachments.Attachment
|
||||
@@ -58,6 +60,10 @@ import kotlin.time.Duration.Companion.seconds
|
||||
@RunWith(AndroidJUnit4::class)
|
||||
class AttachmentTableTest {
|
||||
|
||||
companion object {
|
||||
private const val TEST_PAGE_SIZE = 3
|
||||
}
|
||||
|
||||
@get:Rule
|
||||
val harness = SignalActivityRule(othersCount = 10)
|
||||
|
||||
@@ -594,6 +600,210 @@ class AttachmentTableTest {
|
||||
assertThat(updatedCount).isEqualTo(0)
|
||||
}
|
||||
|
||||
/**
|
||||
* Offloading deletes the only local copy, so it must be gated on evidence that came from the server. Being in the snapshot is our own bookkeeping. Only
|
||||
* last_seen_on_remote_snapshot_version is the part the CDN told us, so only it can authorize the delete.
|
||||
*/
|
||||
@Test
|
||||
fun givenAnOffloadCandidateConfirmedOnTheCdn_whenIOptimize_thenIExpectItOffloaded() {
|
||||
val attachmentId = seedOffloadCandidate()
|
||||
val snapshotVersion = commitSnapshotFor(attachmentId, markSeenOnRemote = true)
|
||||
|
||||
SignalDatabase.attachments.markEligibleAttachmentsAsOptimized(lastCompletedCrawlVersion = snapshotVersion, minimumAge = 30.days, now = pastTheOffloadWindow())
|
||||
|
||||
assertThat(SignalDatabase.attachments.getAttachment(attachmentId)!!.transferState).isEqualTo(AttachmentTable.TRANSFER_RESTORE_OFFLOADED)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun givenAnOffloadCandidateNeverSeenOnTheCdn_whenIOptimize_thenIExpectItLeftAlone() {
|
||||
val attachmentId = seedOffloadCandidate()
|
||||
val snapshotVersion = commitSnapshotFor(attachmentId, markSeenOnRemote = false)
|
||||
|
||||
SignalDatabase.attachments.markEligibleAttachmentsAsOptimized(lastCompletedCrawlVersion = snapshotVersion, minimumAge = 30.days, now = pastTheOffloadWindow())
|
||||
|
||||
assertThat(SignalDatabase.attachments.getAttachment(attachmentId)!!.transferState).isEqualTo(AttachmentTable.TRANSFER_PROGRESS_DONE)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun givenNoCompletedReconciliation_whenIOptimize_thenIExpectNothingOffloaded() {
|
||||
val attachmentId = seedOffloadCandidate()
|
||||
commitSnapshotFor(attachmentId, markSeenOnRemote = true)
|
||||
|
||||
SignalDatabase.attachments.markEligibleAttachmentsAsOptimized(lastCompletedCrawlVersion = -1, minimumAge = 30.days, now = pastTheOffloadWindow())
|
||||
|
||||
assertThat(SignalDatabase.attachments.getAttachment(attachmentId)!!.transferState).isEqualTo(AttachmentTable.TRANSFER_PROGRESS_DONE)
|
||||
}
|
||||
|
||||
/**
|
||||
* Coverage across pages: if only the first page were processed, the overflow would silently never be reclaimed.
|
||||
*/
|
||||
@Test
|
||||
fun givenMoreConfirmedCandidatesThanOnePage_whenIOptimize_thenIExpectEveryPageOffloaded() {
|
||||
val attachmentIds = List(TEST_PAGE_SIZE * 3) { seedOffloadCandidate() }
|
||||
val snapshotVersion = commitSnapshotForAll(attachmentIds, markSeenOnRemote = true)
|
||||
|
||||
SignalDatabase.attachments.markEligibleAttachmentsAsOptimized(lastCompletedCrawlVersion = snapshotVersion, minimumAge = 30.days, now = pastTheOffloadWindow(), pageSize = TEST_PAGE_SIZE)
|
||||
|
||||
assertThat(attachmentIds.count { isOffloaded(it) }).isEqualTo(attachmentIds.size)
|
||||
}
|
||||
|
||||
/**
|
||||
* Termination when nothing is offloadable. Unconfirmed candidates stay eligible forever, so a loop that ran until the candidate query drained would spin here
|
||||
* and this test would hang rather than fail. Only the advancing id cursor ends it.
|
||||
*/
|
||||
@Test
|
||||
fun givenMoreUnconfirmedCandidatesThanOnePage_whenIOptimize_thenIExpectItToTerminateAndOffloadNothing() {
|
||||
val attachmentIds = List(TEST_PAGE_SIZE * 3) { seedOffloadCandidate() }
|
||||
val snapshotVersion = commitSnapshotForAll(attachmentIds, markSeenOnRemote = false)
|
||||
|
||||
SignalDatabase.attachments.markEligibleAttachmentsAsOptimized(lastCompletedCrawlVersion = snapshotVersion, minimumAge = 30.days, now = pastTheOffloadWindow(), pageSize = TEST_PAGE_SIZE)
|
||||
|
||||
assertThat(attachmentIds.count { isOffloaded(it) }).isEqualTo(0)
|
||||
}
|
||||
|
||||
/**
|
||||
* Stickers come back from their pack rather than the archive, so they are never offloaded no matter what the CDN confirms.
|
||||
*/
|
||||
@Test
|
||||
fun givenAStickerOffloadCandidate_whenIOptimize_thenIExpectItLeftAlone() {
|
||||
val attachmentId = seedOffloadCandidate()
|
||||
val snapshotVersion = commitSnapshotFor(attachmentId, markSeenOnRemote = true)
|
||||
markAsSticker(attachmentId)
|
||||
|
||||
SignalDatabase.attachments.markEligibleAttachmentsAsOptimized(lastCompletedCrawlVersion = snapshotVersion, minimumAge = 30.days, now = pastTheOffloadWindow())
|
||||
|
||||
assertThat(isOffloaded(attachmentId)).isFalse()
|
||||
}
|
||||
|
||||
/**
|
||||
* Offloading an image with no thumbnail would leave nothing at all to render in the conversation, so visual media has to keep a local thumbnail to qualify.
|
||||
*/
|
||||
@Test
|
||||
fun givenAnImageOffloadCandidateWithNoThumbnail_whenIOptimize_thenIExpectItLeftAlone() {
|
||||
val attachmentId = seedOffloadCandidate(contentType = MediaUtil.IMAGE_JPEG)
|
||||
val snapshotVersion = commitSnapshotFor(attachmentId, markSeenOnRemote = true)
|
||||
|
||||
SignalDatabase.attachments.markEligibleAttachmentsAsOptimized(lastCompletedCrawlVersion = snapshotVersion, minimumAge = 30.days, now = pastTheOffloadWindow())
|
||||
|
||||
assertThat(isOffloaded(attachmentId)).isFalse()
|
||||
}
|
||||
|
||||
/**
|
||||
* Media the user just pulled back down would otherwise be offloaded again immediately, undoing the restore they asked for.
|
||||
*/
|
||||
@Test
|
||||
fun givenARecentlyRestoredOffloadCandidate_whenIOptimize_thenIExpectItLeftAlone() {
|
||||
val attachmentId = seedOffloadCandidate()
|
||||
val snapshotVersion = commitSnapshotFor(attachmentId, markSeenOnRemote = true)
|
||||
val evaluatedAt = pastTheOffloadWindow()
|
||||
markRestoredAt(attachmentId, evaluatedAt)
|
||||
|
||||
SignalDatabase.attachments.markEligibleAttachmentsAsOptimized(lastCompletedCrawlVersion = snapshotVersion, minimumAge = 30.days, now = evaluatedAt)
|
||||
|
||||
assertThat(isOffloaded(attachmentId)).isFalse()
|
||||
}
|
||||
|
||||
@Test
|
||||
fun givenAnOffloadCandidateOnATooRecentMessage_whenIOptimize_thenIExpectItLeftAlone() {
|
||||
val attachmentId = seedOffloadCandidate()
|
||||
val snapshotVersion = commitSnapshotFor(attachmentId, markSeenOnRemote = true)
|
||||
|
||||
SignalDatabase.attachments.markEligibleAttachmentsAsOptimized(lastCompletedCrawlVersion = snapshotVersion, minimumAge = 30.days, now = System.currentTimeMillis())
|
||||
|
||||
assertThat(isOffloaded(attachmentId)).isFalse()
|
||||
}
|
||||
|
||||
/** Far enough ahead of the seeded message that both the offload age and the 7-day offload-restore window are satisfied. */
|
||||
private fun pastTheOffloadWindow(): Long = System.currentTimeMillis() + 60.days.inWholeMilliseconds
|
||||
|
||||
/** A full locator, because a partial one makes reading the attachment back blow up on the sticker fields. */
|
||||
private fun markAsSticker(attachmentId: AttachmentId) {
|
||||
SignalDatabase.rawDatabase
|
||||
.update(AttachmentTable.TABLE_NAME)
|
||||
.values(
|
||||
AttachmentTable.STICKER_ID to 7,
|
||||
AttachmentTable.STICKER_PACK_ID to "aa1111bbcc2222ddee3333ff44445555",
|
||||
AttachmentTable.STICKER_PACK_KEY to "YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXoxMjM0NTY3ODkwYWI=",
|
||||
AttachmentTable.STICKER_EMOJI to ":)"
|
||||
)
|
||||
.where("${AttachmentTable.ID} = ?", attachmentId.id)
|
||||
.run()
|
||||
}
|
||||
|
||||
private fun markRestoredAt(attachmentId: AttachmentId, restoredAt: Long) {
|
||||
SignalDatabase.rawDatabase
|
||||
.update(AttachmentTable.TABLE_NAME)
|
||||
.values(AttachmentTable.OFFLOAD_RESTORED_AT to restoredAt)
|
||||
.where("${AttachmentTable.ID} = ?", attachmentId.id)
|
||||
.run()
|
||||
}
|
||||
|
||||
/** Defaults to a non-media content type, which keeps the candidate eligible without also having to generate a thumbnail file. */
|
||||
private fun seedOffloadCandidate(contentType: String = "application/pdf"): AttachmentId {
|
||||
val data = byteArrayOf(1, 2, 3, 4, 5)
|
||||
val attachment = createAttachmentPointer(Random.nextBytes(32), data.size, contentType = contentType)
|
||||
val messageResult = SignalDatabase.messages.insertMessageInbox(createIncomingMessage(serverTime = System.currentTimeMillis().milliseconds, attachment = attachment)).get()
|
||||
val attachmentId = messageResult.insertedAttachments!![attachment]!!
|
||||
|
||||
SignalDatabase.attachments.setTransferState(messageResult.messageId, attachmentId, AttachmentTable.TRANSFER_PROGRESS_STARTED)
|
||||
SignalDatabase.attachments.finalizeAttachmentAfterDownload(messageResult.messageId, attachmentId, ByteArrayInputStream(data))
|
||||
SignalDatabase.attachments.setArchiveTransferState(attachmentId, AttachmentTable.ArchiveTransferState.FINISHED)
|
||||
|
||||
return attachmentId
|
||||
}
|
||||
|
||||
/**
|
||||
* Commits every entry in a single snapshot version. Calling [commitSnapshotFor] in a loop would not work: each commit bumps the version, leaving all but the
|
||||
* last entry below MAX_VERSION and therefore unconfirmable.
|
||||
*/
|
||||
private fun commitSnapshotForAll(attachmentIds: List<AttachmentId>, markSeenOnRemote: Boolean): Long {
|
||||
val mediaIds: MutableList<String> = mutableListOf()
|
||||
val entries: MutableList<BackupMediaSnapshotTable.MediaEntry> = mutableListOf()
|
||||
|
||||
for (attachmentId in attachmentIds) {
|
||||
val attachment = SignalDatabase.attachments.getAttachment(attachmentId)!!
|
||||
val plaintextHash = attachment.dataHash!!.decodeBase64OrThrow()
|
||||
val remoteKey = attachment.remoteKey!!.decodeBase64OrThrow()
|
||||
val mediaId = MediaName.fromPlaintextHashAndRemoteKey(plaintextHash, remoteKey).toMediaId(SignalStore.backup.mediaRootBackupKey).encode()
|
||||
|
||||
mediaIds += mediaId
|
||||
entries += BackupMediaSnapshotTable.MediaEntry(mediaId = mediaId, cdn = 3, plaintextHash = plaintextHash, remoteKey = remoteKey, isThumbnail = false)
|
||||
}
|
||||
|
||||
SignalDatabase.backupMediaSnapshots.writePendingMediaEntries(entries)
|
||||
SignalDatabase.backupMediaSnapshots.commitPendingRows()
|
||||
|
||||
val snapshotVersion = SignalDatabase.backupMediaSnapshots.getCurrentSnapshotVersion()
|
||||
if (markSeenOnRemote) {
|
||||
SignalDatabase.backupMediaSnapshots.markSeenOnRemote(mediaIds, snapshotVersion)
|
||||
}
|
||||
|
||||
return snapshotVersion
|
||||
}
|
||||
|
||||
private fun isOffloaded(attachmentId: AttachmentId): Boolean {
|
||||
return SignalDatabase.attachments.getAttachment(attachmentId)!!.transferState == AttachmentTable.TRANSFER_RESTORE_OFFLOADED
|
||||
}
|
||||
|
||||
private fun commitSnapshotFor(attachmentId: AttachmentId, markSeenOnRemote: Boolean): Long {
|
||||
val attachment = SignalDatabase.attachments.getAttachment(attachmentId)!!
|
||||
val plaintextHash = attachment.dataHash!!.decodeBase64OrThrow()
|
||||
val remoteKey = attachment.remoteKey!!.decodeBase64OrThrow()
|
||||
val mediaId = MediaName.fromPlaintextHashAndRemoteKey(plaintextHash, remoteKey).toMediaId(SignalStore.backup.mediaRootBackupKey).encode()
|
||||
|
||||
SignalDatabase.backupMediaSnapshots.writePendingMediaEntries(
|
||||
listOf(BackupMediaSnapshotTable.MediaEntry(mediaId = mediaId, cdn = 3, plaintextHash = plaintextHash, remoteKey = remoteKey, isThumbnail = false))
|
||||
)
|
||||
SignalDatabase.backupMediaSnapshots.commitPendingRows()
|
||||
|
||||
val snapshotVersion = SignalDatabase.backupMediaSnapshots.getCurrentSnapshotVersion()
|
||||
if (markSeenOnRemote) {
|
||||
SignalDatabase.backupMediaSnapshots.markSeenOnRemote(listOf(mediaId), snapshotVersion)
|
||||
}
|
||||
|
||||
return snapshotVersion
|
||||
}
|
||||
|
||||
private fun createIncomingMessage(
|
||||
serverTime: Duration,
|
||||
attachment: Attachment,
|
||||
@@ -611,13 +821,13 @@ class AttachmentTableTest {
|
||||
)
|
||||
}
|
||||
|
||||
private fun createAttachmentPointer(key: ByteArray, size: Int): Attachment {
|
||||
private fun createAttachmentPointer(key: ByteArray, size: Int, contentType: String = MediaUtil.IMAGE_JPEG): Attachment {
|
||||
return PointerAttachment.forPointer(
|
||||
pointer = Optional.of(
|
||||
SignalServiceAttachmentPointer(
|
||||
cdnNumber = 3,
|
||||
remoteId = SignalServiceAttachmentRemoteId.V4("asdf"),
|
||||
contentType = MediaUtil.IMAGE_JPEG,
|
||||
contentType = contentType,
|
||||
key = key,
|
||||
size = Optional.of(size),
|
||||
preview = Optional.empty(),
|
||||
|
||||
+293
-5
@@ -6,16 +6,22 @@
|
||||
package org.thoughtcrime.securesms.jobs
|
||||
|
||||
import androidx.test.ext.junit.runners.AndroidJUnit4
|
||||
import arrow.core.left
|
||||
import arrow.core.right
|
||||
import assertk.assertThat
|
||||
import assertk.assertions.contains
|
||||
import assertk.assertions.doesNotContain
|
||||
import assertk.assertions.isEqualTo
|
||||
import assertk.assertions.isFalse
|
||||
import assertk.assertions.isGreaterThan
|
||||
import assertk.assertions.isNull
|
||||
import io.mockk.Runs
|
||||
import io.mockk.coEvery
|
||||
import io.mockk.coVerify
|
||||
import io.mockk.every
|
||||
import io.mockk.just
|
||||
import io.mockk.mockkObject
|
||||
import io.mockk.slot
|
||||
import io.mockk.unmockkAll
|
||||
import io.mockk.verify
|
||||
import org.junit.After
|
||||
@@ -27,10 +33,12 @@ import org.signal.core.models.backup.MediaName
|
||||
import org.signal.core.models.database.AttachmentId
|
||||
import org.signal.core.util.Base64.decodeBase64OrThrow
|
||||
import org.signal.network.api.ArchiveApiV2
|
||||
import org.signal.network.service.ArchiveError
|
||||
import org.signal.network.service.ArchiveService
|
||||
import org.thoughtcrime.securesms.attachments.ArchivedAttachment
|
||||
import org.thoughtcrime.securesms.attachments.Attachment
|
||||
import org.thoughtcrime.securesms.attachments.PointerAttachment
|
||||
import org.thoughtcrime.securesms.backup.v2.BackupRepository
|
||||
import org.thoughtcrime.securesms.backup.v2.ArchivedMediaObject
|
||||
import org.thoughtcrime.securesms.backup.v2.MessageBackupTier
|
||||
import org.thoughtcrime.securesms.database.AttachmentTable
|
||||
import org.thoughtcrime.securesms.database.BackupMediaSnapshotTable.MediaEntry
|
||||
@@ -44,7 +52,10 @@ import org.thoughtcrime.securesms.util.MediaUtil
|
||||
import org.whispersystems.signalservice.api.messages.SignalServiceAttachmentPointer
|
||||
import org.whispersystems.signalservice.api.messages.SignalServiceAttachmentRemoteId
|
||||
import java.io.ByteArrayInputStream
|
||||
import java.io.IOException
|
||||
import java.util.Optional
|
||||
import java.util.UUID
|
||||
import kotlin.random.Random
|
||||
import kotlin.time.Duration
|
||||
import kotlin.time.Duration.Companion.days
|
||||
|
||||
@@ -56,16 +67,18 @@ class ArchiveAttachmentReconciliationJobTest {
|
||||
|
||||
private val archiveService: ArchiveService = AppDependencies.archiveService
|
||||
|
||||
private val deletedFromCdn = slot<Set<ArchivedMediaObject>>()
|
||||
|
||||
@Before
|
||||
fun setUp() {
|
||||
SignalStore.backup.backupTier = MessageBackupTier.PAID
|
||||
SignalStore.backup.hasBackupBeenUploaded = true
|
||||
SignalStore.backup.lastAttachmentReconciliationTime = System.currentTimeMillis()
|
||||
SignalStore.backup.localRestoreReconcilePending = false
|
||||
SignalStore.backup.lastUsedMessageCutoffTime = 0
|
||||
|
||||
mockkObject(BackupRepository)
|
||||
mockkObject(ArchiveCommitAttachmentDeletesJob)
|
||||
coEvery { ArchiveCommitAttachmentDeletesJob.deleteMediaObjectsFromCdn(any(), any(), any(), any()) } returns null
|
||||
coEvery { ArchiveCommitAttachmentDeletesJob.deleteMediaObjectsFromCdn(any(), capture(deletedFromCdn), any(), any()) } returns null
|
||||
}
|
||||
|
||||
@After
|
||||
@@ -174,6 +187,36 @@ class ArchiveAttachmentReconciliationJobTest {
|
||||
assertThat(SignalDatabase.attachments.getAttachment(attachmentId)!!.archiveTransferState).isEqualTo(AttachmentTable.ArchiveTransferState.FINISHED)
|
||||
}
|
||||
|
||||
/**
|
||||
* The over-size-limit cutoff leaves old messages out of the export, so nothing in the backup references their media any more. Reconciliation applies the same
|
||||
* rule as [ArchiveCommitAttachmentDeletesJob] so the two reclaim paths can't disagree about what still counts as referenced.
|
||||
*/
|
||||
@Test
|
||||
fun givenCdnMediaWhoseMessagePredatesTheMessageCutoff_whenIReconcile_thenIExpectItDeletedFromTheCdn() {
|
||||
val attachmentId = seedFinalizedAttachment("remote-key-cutoff".toByteArray(), byteArrayOf(21, 22, 23, 24, 25), receivedAt = 10.days)
|
||||
SignalStore.backup.lastUsedMessageCutoffTime = 30.days.inWholeMilliseconds
|
||||
fakeCdnContains(attachmentId, cdn = 3)
|
||||
|
||||
ArchiveAttachmentReconciliationJob(forced = true).run()
|
||||
|
||||
assertThat(deletedFromCdn.captured.map { it.mediaId }).contains(mediaIdFor(attachmentId))
|
||||
}
|
||||
|
||||
/**
|
||||
* A cutoff being set at all must not weaken the protection for media the backup still references, which is what keeps a lost snapshot table from wiping the
|
||||
* archive.
|
||||
*/
|
||||
@Test
|
||||
fun givenCdnMediaWhoseMessageIsWithinTheMessageCutoff_whenIReconcile_thenIExpectItProtected() {
|
||||
val attachmentId = seedFinalizedAttachment("remote-key-recent".toByteArray(), byteArrayOf(26, 27, 28, 29, 30), receivedAt = 40.days)
|
||||
SignalStore.backup.lastUsedMessageCutoffTime = 30.days.inWholeMilliseconds
|
||||
fakeCdnContains(attachmentId, cdn = 3)
|
||||
|
||||
ArchiveAttachmentReconciliationJob(forced = true).run()
|
||||
|
||||
coVerify(exactly = 0) { ArchiveCommitAttachmentDeletesJob.deleteMediaObjectsFromCdn(any(), any(), any(), any()) }
|
||||
}
|
||||
|
||||
@Test
|
||||
fun givenFirstEverReconciliation_whenIForceIt_thenItStillRunsAndRepairs() {
|
||||
SignalStore.backup.lastAttachmentReconciliationTime = -1
|
||||
@@ -198,9 +241,254 @@ class ArchiveAttachmentReconciliationJobTest {
|
||||
assertThat(SignalStore.backup.localRestoreReconcilePending).isFalse()
|
||||
}
|
||||
|
||||
private fun seedFinalizedAttachment(remoteKey: ByteArray, data: ByteArray): AttachmentId {
|
||||
/**
|
||||
* Media that was offloaded (or restored from a backup) has no local data file, so resetting it to NONE cannot lead to a re-upload. It only makes the export
|
||||
* emit a tombstone, which drops the locator and marks the media for deletion off of the CDN. It must stay FINISHED.
|
||||
*/
|
||||
@Test
|
||||
fun givenFinishedMediaMissingFromCdnWithNoLocalDataFile_whenIReconcile_thenItStaysFinishedAndKeepsItsCdn() {
|
||||
val attachmentId = seedArchivedAttachment()
|
||||
commitSnapshotFor(attachmentId, cdn = 3)
|
||||
fakeCdnEmpty()
|
||||
|
||||
ArchiveAttachmentReconciliationJob(forced = true).run()
|
||||
|
||||
val after = SignalDatabase.attachments.getAttachment(attachmentId)!!
|
||||
assertThat(after.archiveTransferState).isEqualTo(AttachmentTable.ArchiveTransferState.FINISHED)
|
||||
assertThat(after.archiveCdn).isEqualTo(3)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun givenACrawlThatCompletes_whenIReconcile_thenIExpectTheCompletedSnapshotVersionRecorded() {
|
||||
SignalStore.backup.lastCompletedReconciliationSnapshotVersion = -1
|
||||
|
||||
val attachmentId = seedFinalizedAttachment("remote-key-recorded".toByteArray(), byteArrayOf(1, 2, 3, 4, 5))
|
||||
commitSnapshotFor(attachmentId, cdn = 3)
|
||||
fakeCdnContains(attachmentId, cdn = 3)
|
||||
|
||||
ArchiveAttachmentReconciliationJob(forced = true).run()
|
||||
|
||||
assertThat(SignalStore.backup.lastCompletedReconciliationSnapshotVersion).isEqualTo(SignalDatabase.backupMediaSnapshots.getCurrentSnapshotVersion())
|
||||
}
|
||||
|
||||
/**
|
||||
* Offloading trusts the recorded version as proof the server confirmed our media, so a crawl that dies part way through must leave it untouched rather than
|
||||
* recording a version it never finished verifying.
|
||||
*/
|
||||
@Test
|
||||
fun givenACrawlThatFailsPartWayThrough_whenIReconcile_thenIExpectNoCompletedSnapshotVersionRecorded() {
|
||||
SignalStore.backup.lastCompletedReconciliationSnapshotVersion = -1
|
||||
|
||||
val attachmentId = seedFinalizedAttachment("remote-key-failed".toByteArray(), byteArrayOf(2, 3, 4, 5, 6))
|
||||
commitSnapshotFor(attachmentId, cdn = 3)
|
||||
coEvery { archiveService.listRemoteMediaObjects(any(), any()) } returns ArchiveError.NetworkError(IOException("boom")).left()
|
||||
|
||||
val result = ArchiveAttachmentReconciliationJob(forced = true).run()
|
||||
|
||||
assertThat(result.isSuccess).isFalse()
|
||||
assertThat(SignalStore.backup.lastCompletedReconciliationSnapshotVersion).isEqualTo(-1L)
|
||||
}
|
||||
|
||||
/**
|
||||
* A completed crawl proves the object isn't on the CDN, so the row is bookkeeping for something that no longer exists and there is nothing left to orphan.
|
||||
*/
|
||||
@Test
|
||||
fun givenOldSnapshotMediaAbsentFromCdn_whenIReconcile_thenIExpectTheRowPruned() {
|
||||
val absentMediaId = commitRandomSnapshotEntry()
|
||||
commitRandomSnapshotEntry()
|
||||
fakeCdnEmpty()
|
||||
|
||||
ArchiveAttachmentReconciliationJob(forced = true).run()
|
||||
|
||||
assertThat(oldSnapshotMediaIds()).doesNotContain(absentMediaId)
|
||||
}
|
||||
|
||||
/**
|
||||
* A crawl confirmed this object once, so a later crawl failing to list it is not enough to conclude it's gone. An upload landing mid-crawl produces the same
|
||||
* evidence. Pruning here would contradict our own earlier proof and orphan the object.
|
||||
*/
|
||||
@Test
|
||||
fun givenOldSnapshotMediaConfirmedByAnEarlierCrawl_whenIReconcile_thenIExpectTheRowKept() {
|
||||
val previouslySeenMediaId = commitRandomSnapshotEntry()
|
||||
SignalDatabase.backupMediaSnapshots.markSeenOnRemote(listOf(previouslySeenMediaId), 1)
|
||||
commitRandomSnapshotEntry()
|
||||
fakeCdnEmpty()
|
||||
|
||||
ArchiveAttachmentReconciliationJob(forced = true).run()
|
||||
|
||||
assertThat(oldSnapshotMediaIds()).contains(previouslySeenMediaId)
|
||||
}
|
||||
|
||||
/** The crawl found the object, so this row is the only thing tracking it and must survive even though it left the latest snapshot. */
|
||||
@Test
|
||||
fun givenOldSnapshotMediaStillOnCdn_whenIReconcile_thenIExpectTheRowKept() {
|
||||
val attachmentId = seedFinalizedAttachment("remote-key-kept".toByteArray(), byteArrayOf(3, 1, 4, 1, 5))
|
||||
commitSnapshotFor(attachmentId, cdn = 3)
|
||||
commitRandomSnapshotEntry()
|
||||
fakeCdnContains(attachmentId, cdn = 3)
|
||||
|
||||
ArchiveAttachmentReconciliationJob(forced = true).run()
|
||||
|
||||
assertThat(oldSnapshotMediaIds()).contains(mediaIdFor(attachmentId))
|
||||
}
|
||||
|
||||
/**
|
||||
* The forced-crawl rate limit must only be consumed by a crawl that actually starts, otherwise a job that returns early leaves offloading blocked for a full
|
||||
* interval without having verified anything.
|
||||
*/
|
||||
@Test
|
||||
fun givenNoBackupUploaded_whenIForceAReconciliation_thenIExpectTheAttemptTimeUnchanged() {
|
||||
SignalStore.backup.hasBackupBeenUploaded = false
|
||||
SignalStore.backup.lastForcedReconciliationAttemptTime = 0
|
||||
fakeCdnEmpty()
|
||||
|
||||
ArchiveAttachmentReconciliationJob(forced = true).run()
|
||||
|
||||
assertThat(SignalStore.backup.lastForcedReconciliationAttemptTime).isEqualTo(0L)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun givenACrawlThatStarts_whenIForceAReconciliation_thenIExpectTheAttemptTimeAdvanced() {
|
||||
SignalStore.backup.lastForcedReconciliationAttemptTime = 0
|
||||
commitRandomSnapshotEntry()
|
||||
fakeCdnEmpty()
|
||||
|
||||
ArchiveAttachmentReconciliationJob(forced = true).run()
|
||||
|
||||
assertThat(SignalStore.backup.lastForcedReconciliationAttemptTime).isGreaterThan(0L)
|
||||
}
|
||||
|
||||
/**
|
||||
* A crawl with no snapshot to compare against can't confirm anything, so it must not consume the forced-attempt budget. Spending it here would rate-limit the
|
||||
* retry that becomes useful once a backup has built a snapshot.
|
||||
*/
|
||||
@Test
|
||||
fun givenNoSnapshotYet_whenIForceAReconciliation_thenIExpectTheAttemptTimeUnchanged() {
|
||||
SignalStore.backup.lastForcedReconciliationAttemptTime = 0
|
||||
fakeCdnEmpty()
|
||||
|
||||
ArchiveAttachmentReconciliationJob(forced = true).run()
|
||||
|
||||
assertThat(SignalStore.backup.lastForcedReconciliationAttemptTime).isEqualTo(0L)
|
||||
}
|
||||
|
||||
/**
|
||||
* Same reasoning for the offload gate: a snapshot-less crawl verified nothing, so it must not record a completed version that would let us start deleting
|
||||
* local copies of media.
|
||||
*/
|
||||
@Test
|
||||
fun givenNoSnapshotYet_whenIReconcile_thenIExpectNoCompletedSnapshotVersionRecorded() {
|
||||
SignalStore.backup.lastCompletedReconciliationSnapshotVersion = -1
|
||||
fakeCdnEmpty()
|
||||
|
||||
ArchiveAttachmentReconciliationJob(forced = true).run()
|
||||
|
||||
assertThat(SignalStore.backup.lastCompletedReconciliationSnapshotVersion).isEqualTo(-1L)
|
||||
}
|
||||
|
||||
// TODO [cody] return after fixing perf problem of restoring thumbnail state
|
||||
// Commented out alongside the skipped thumbnail promotion. See the TODO in ArchiveAttachmentReconciliationJob.syncCdnPage.
|
||||
//
|
||||
// /**
|
||||
// * A restore sets the full-size state from the backup's CDN claim but leaves the thumbnail state NONE, so the listing is the only thing that can promote it.
|
||||
// * Until it does, the thumbnail is dropped from the snapshot on every backup and gets needlessly re-uploaded once the media is downloaded.
|
||||
// */
|
||||
// @Test
|
||||
// fun givenRestoredMediaWhoseThumbnailIsOnTheCdn_whenIReconcile_thenIExpectTheThumbnailMarkedFinished() {
|
||||
// val attachmentId = seedArchivedAttachment()
|
||||
// assertThat(SignalDatabase.attachments.getArchiveThumbnailTransferState(attachmentId)).isEqualTo(AttachmentTable.ArchiveTransferState.NONE)
|
||||
//
|
||||
// fakeCdnContainsThumbnail(attachmentId, cdn = 3)
|
||||
//
|
||||
// ArchiveAttachmentReconciliationJob(forced = true).run()
|
||||
//
|
||||
// assertThat(SignalDatabase.attachments.getArchiveThumbnailTransferState(attachmentId)).isEqualTo(AttachmentTable.ArchiveTransferState.FINISHED)
|
||||
// }
|
||||
//
|
||||
// @Test
|
||||
// fun givenRestoredMediaWhoseThumbnailIsNotOnTheCdn_whenIReconcile_thenIExpectTheThumbnailLeftAlone() {
|
||||
// val attachmentId = seedArchivedAttachment()
|
||||
// fakeCdnEmpty()
|
||||
//
|
||||
// ArchiveAttachmentReconciliationJob(forced = true).run()
|
||||
//
|
||||
// assertThat(SignalDatabase.attachments.getArchiveThumbnailTransferState(attachmentId)).isEqualTo(AttachmentTable.ArchiveTransferState.NONE)
|
||||
// }
|
||||
|
||||
// private fun fakeCdnContainsThumbnail(attachmentId: AttachmentId, cdn: Int) {
|
||||
// val attachment = SignalDatabase.attachments.getAttachment(attachmentId)!!
|
||||
// val plaintextHash = attachment.dataHash!!.decodeBase64OrThrow()
|
||||
// val remoteKey = attachment.remoteKey!!.decodeBase64OrThrow()
|
||||
// val mediaId = MediaName.fromPlaintextHashAndRemoteKeyForThumbnail(plaintextHash, remoteKey).toMediaId(SignalStore.backup.mediaRootBackupKey).encode()
|
||||
//
|
||||
// coEvery { archiveService.listRemoteMediaObjects(any(), any()) } returns ArchiveApiV2.MediaItemsPage(
|
||||
// storedMediaObjects = listOf(ArchiveApiV2.StoredMediaObject(cdn = cdn, mediaId = mediaId, objectLength = attachment.size)),
|
||||
// cursor = null
|
||||
// ).right()
|
||||
// }
|
||||
|
||||
private fun oldSnapshotMediaIds(): List<String> {
|
||||
return SignalDatabase.backupMediaSnapshots.getPageOfOldMediaEntries(pageSize = 1_000).map { it.mediaId }
|
||||
}
|
||||
|
||||
private fun mediaIdFor(attachmentId: AttachmentId): String {
|
||||
val attachment = SignalDatabase.attachments.getAttachment(attachmentId)!!
|
||||
val plaintextHash = attachment.dataHash!!.decodeBase64OrThrow()
|
||||
val remoteKey = attachment.remoteKey!!.decodeBase64OrThrow()
|
||||
|
||||
return MediaName.fromPlaintextHashAndRemoteKey(plaintextHash, remoteKey).toMediaId(SignalStore.backup.mediaRootBackupKey).encode()
|
||||
}
|
||||
|
||||
/**
|
||||
* Committing a second entry is what pushes any previously committed row below MAX(snapshot_version), which is the "fell out of the latest snapshot" state.
|
||||
*/
|
||||
private fun commitRandomSnapshotEntry(): String {
|
||||
val plaintextHash = Random.nextBytes(32)
|
||||
val remoteKey = Random.nextBytes(32)
|
||||
val mediaId = MediaName.fromPlaintextHashAndRemoteKey(plaintextHash, remoteKey).toMediaId(SignalStore.backup.mediaRootBackupKey).encode()
|
||||
|
||||
SignalDatabase.backupMediaSnapshots.writePendingMediaEntries(
|
||||
listOf(MediaEntry(mediaId = mediaId, cdn = 3, plaintextHash = plaintextHash, remoteKey = remoteKey, isThumbnail = false))
|
||||
)
|
||||
SignalDatabase.backupMediaSnapshots.commitPendingRows()
|
||||
|
||||
return mediaId
|
||||
}
|
||||
|
||||
private fun seedArchivedAttachment(): AttachmentId {
|
||||
val attachment = ArchivedAttachment(
|
||||
contentType = MediaUtil.IMAGE_JPEG,
|
||||
size = 1024,
|
||||
cdn = 3,
|
||||
uploadTimestamp = 0,
|
||||
key = Random.nextBytes(8),
|
||||
cdnKey = "password",
|
||||
archiveCdn = 3,
|
||||
plaintextHash = Random.nextBytes(8),
|
||||
incrementalMac = Random.nextBytes(8),
|
||||
incrementalMacChunkSize = 8,
|
||||
width = 100,
|
||||
height = 100,
|
||||
caption = null,
|
||||
blurHash = null,
|
||||
voiceNote = false,
|
||||
borderless = false,
|
||||
stickerLocator = null,
|
||||
gif = false,
|
||||
quote = false,
|
||||
quoteTargetContentType = null,
|
||||
uuid = UUID.randomUUID(),
|
||||
fileName = null,
|
||||
localBackupKey = null
|
||||
)
|
||||
|
||||
val messageId = SignalDatabase.messages.insertMessageInbox(createIncomingMessage(serverTime = 0.days, attachment = attachment)).get().messageId
|
||||
return SignalDatabase.attachments.getAttachmentsForMessage(messageId).first().attachmentId
|
||||
}
|
||||
|
||||
private fun seedFinalizedAttachment(remoteKey: ByteArray, data: ByteArray, receivedAt: Duration = 0.days): AttachmentId {
|
||||
val attachment = createAttachmentPointer(remoteKey, data.size)
|
||||
val messageResult = SignalDatabase.messages.insertMessageInbox(createIncomingMessage(serverTime = 0.days, attachment = attachment)).get()
|
||||
val messageResult = SignalDatabase.messages.insertMessageInbox(createIncomingMessage(serverTime = receivedAt, attachment = attachment)).get()
|
||||
val attachmentId = messageResult.insertedAttachments!![attachment]!!
|
||||
SignalDatabase.attachments.setTransferState(messageResult.messageId, attachmentId, AttachmentTable.TRANSFER_PROGRESS_STARTED)
|
||||
SignalDatabase.attachments.finalizeAttachmentAfterDownload(messageResult.messageId, attachmentId, ByteArrayInputStream(data))
|
||||
|
||||
+411
@@ -0,0 +1,411 @@
|
||||
/*
|
||||
* Copyright 2026 Signal Messenger, LLC
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
package org.thoughtcrime.securesms.jobs
|
||||
|
||||
import androidx.test.ext.junit.runners.AndroidJUnit4
|
||||
import assertk.assertThat
|
||||
import assertk.assertions.contains
|
||||
import assertk.assertions.doesNotContain
|
||||
import assertk.assertions.hasSize
|
||||
import assertk.assertions.isEmpty
|
||||
import assertk.assertions.isEqualTo
|
||||
import io.mockk.coEvery
|
||||
import io.mockk.coVerify
|
||||
import io.mockk.mockkObject
|
||||
import io.mockk.slot
|
||||
import io.mockk.unmockkAll
|
||||
import org.junit.After
|
||||
import org.junit.Before
|
||||
import org.junit.Rule
|
||||
import org.junit.Test
|
||||
import org.junit.runner.RunWith
|
||||
import org.signal.core.models.backup.MediaName
|
||||
import org.signal.core.models.database.AttachmentId
|
||||
import org.signal.core.util.Base64.decodeBase64OrThrow
|
||||
import org.thoughtcrime.securesms.attachments.Attachment
|
||||
import org.thoughtcrime.securesms.attachments.PointerAttachment
|
||||
import org.thoughtcrime.securesms.backup.v2.ArchivedMediaObject
|
||||
import org.thoughtcrime.securesms.backup.v2.MessageBackupTier
|
||||
import org.thoughtcrime.securesms.database.AttachmentTable
|
||||
import org.thoughtcrime.securesms.database.BackupMediaSnapshotTable.MediaEntry
|
||||
import org.thoughtcrime.securesms.database.MessageType
|
||||
import org.thoughtcrime.securesms.database.SignalDatabase
|
||||
import org.thoughtcrime.securesms.keyvalue.SignalStore
|
||||
import org.thoughtcrime.securesms.mms.IncomingMessage
|
||||
import org.thoughtcrime.securesms.testing.SignalActivityRule
|
||||
import org.thoughtcrime.securesms.util.MediaUtil
|
||||
import org.whispersystems.signalservice.api.messages.SignalServiceAttachmentPointer
|
||||
import org.whispersystems.signalservice.api.messages.SignalServiceAttachmentRemoteId
|
||||
import java.io.ByteArrayInputStream
|
||||
import java.util.Optional
|
||||
import kotlin.random.Random
|
||||
import kotlin.time.Duration
|
||||
import kotlin.time.Duration.Companion.days
|
||||
|
||||
@RunWith(AndroidJUnit4::class)
|
||||
class ArchiveCommitAttachmentDeletesJobTest {
|
||||
|
||||
@get:Rule
|
||||
val harness = SignalActivityRule()
|
||||
|
||||
private val deletedFromCdn = slot<Set<ArchivedMediaObject>>()
|
||||
|
||||
private val jobPageSize = ArchiveCommitAttachmentDeletesJob.REMOTE_DELETE_BATCH_SIZE
|
||||
|
||||
@Before
|
||||
fun setUp() {
|
||||
SignalStore.backup.backupTier = MessageBackupTier.PAID
|
||||
SignalStore.backup.lastUsedMessageCutoffTime = 0
|
||||
|
||||
mockkObject(ArchiveCommitAttachmentDeletesJob)
|
||||
coEvery { ArchiveCommitAttachmentDeletesJob.deleteMediaObjectsFromCdn(any(), capture(deletedFromCdn), any(), any()) } returns null
|
||||
}
|
||||
|
||||
@After
|
||||
fun tearDown() {
|
||||
unmockkAll()
|
||||
}
|
||||
|
||||
/**
|
||||
* Media only leaves the snapshot because the backup stopped referencing it, which a bookkeeping bug can cause while the attachment is still very much present.
|
||||
* Deleting on that basis alone is how the only remaining copy of media gets destroyed, so the pass has to discriminate within a page rather than trusting the
|
||||
* snapshot.
|
||||
*/
|
||||
@Test
|
||||
fun givenOldSnapshotMediaBothReferencedAndOrphaned_whenIRun_thenIExpectOnlyTheOrphanDeletedFromTheCdn() {
|
||||
val attachmentId = seedFinalizedAttachment(byteArrayOf(1, 2, 3, 4, 5))
|
||||
val referenced = entryFor(attachmentId)
|
||||
val orphan = randomEntry()
|
||||
|
||||
commit(referenced, orphan)
|
||||
commit(randomEntry())
|
||||
|
||||
ArchiveCommitAttachmentDeletesJob().run()
|
||||
|
||||
val deletedMediaIds = deletedFromCdn.captured.map { it.mediaId }
|
||||
assertThat(deletedMediaIds).contains(orphan.mediaId)
|
||||
assertThat(deletedMediaIds).doesNotContain(referenced.mediaId)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun givenOnlyStillReferencedOldSnapshotMedia_whenIRun_thenIExpectNothingDeletedFromTheCdn() {
|
||||
val attachmentId = seedFinalizedAttachment(byteArrayOf(9, 8, 7, 6, 5))
|
||||
|
||||
commit(entryFor(attachmentId))
|
||||
commit(randomEntry())
|
||||
|
||||
ArchiveCommitAttachmentDeletesJob().run()
|
||||
|
||||
coVerify(exactly = 0) { ArchiveCommitAttachmentDeletesJob.deleteMediaObjectsFromCdn(any(), any(), any(), any()) }
|
||||
}
|
||||
|
||||
/**
|
||||
* A retained row is the only record that its CDN object exists, so pruning it would orphan the object permanently. The run still has to terminate, which it
|
||||
* does by paging past retained rows by id.
|
||||
*/
|
||||
@Test
|
||||
fun givenOnlyStillReferencedOldSnapshotMedia_whenIRun_thenIExpectTheSnapshotRowsRetained() {
|
||||
val attachmentId = seedFinalizedAttachment(byteArrayOf(4, 4, 4, 4, 4))
|
||||
val referenced = entryFor(attachmentId)
|
||||
|
||||
commit(referenced)
|
||||
commit(randomEntry())
|
||||
|
||||
ArchiveCommitAttachmentDeletesJob().run()
|
||||
|
||||
assertThat(remainingOldMediaIds()).contains(referenced.mediaId)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun givenOldSnapshotMediaWithNoAttachment_whenIRun_thenIExpectTheSnapshotRowPruned() {
|
||||
val orphan = randomEntry()
|
||||
|
||||
commit(orphan)
|
||||
commit(randomEntry())
|
||||
|
||||
ArchiveCommitAttachmentDeletesJob().run()
|
||||
|
||||
assertThat(remainingOldMediaIds()).doesNotContain(orphan.mediaId)
|
||||
}
|
||||
|
||||
/**
|
||||
* Coverage across pages: if the loop only ever handled the first page, the overflow rows would stay on the CDN forever with nothing left to reconsider them.
|
||||
*/
|
||||
@Test
|
||||
fun givenMoreOrphansThanOnePage_whenIRun_thenIExpectEveryPageProcessed() {
|
||||
val orphans = List(jobPageSize + 1) { randomEntry() }
|
||||
|
||||
commit(orphans)
|
||||
commit(randomEntry())
|
||||
|
||||
ArchiveCommitAttachmentDeletesJob().run()
|
||||
|
||||
assertThat(remainingOldMediaIds()).isEmpty()
|
||||
}
|
||||
|
||||
/**
|
||||
* Termination when a whole page is retained. Nothing is removed from the table here, so the loop can only end by advancing its cursor past rows it declined to
|
||||
* delete. If it paged from the start each time it would spin forever and this test would hang rather than fail.
|
||||
*/
|
||||
@Test
|
||||
fun givenMoreUndeletableRowsThanOnePage_whenIRun_thenIExpectItToTerminateAndRetainThem() {
|
||||
val unknownCdn = List(jobPageSize + 1) { randomEntry(cdn = null) }
|
||||
|
||||
commit(unknownCdn)
|
||||
commit(randomEntry())
|
||||
|
||||
ArchiveCommitAttachmentDeletesJob().run()
|
||||
|
||||
assertThat(remainingOldMediaIds()).hasSize(unknownCdn.size)
|
||||
coVerify(exactly = 0) { ArchiveCommitAttachmentDeletesJob.deleteMediaObjectsFromCdn(any(), any(), any(), any()) }
|
||||
}
|
||||
|
||||
/**
|
||||
* Users over the backup size limit get old messages left out of the export entirely, so nothing in the backup references their media any more and it should
|
||||
* stop occupying paid archive quota. Before this, an attachment row existing was enough to retain it forever.
|
||||
*/
|
||||
@Test
|
||||
fun givenMediaWhoseMessagePredatesTheMessageCutoff_whenIRun_thenIExpectItDeletedFromTheCdn() {
|
||||
val attachmentId = seedFinalizedAttachment(byteArrayOf(1, 1, 2, 3, 5), receivedAt = 10.days)
|
||||
val agedOut = entryFor(attachmentId)
|
||||
SignalStore.backup.lastUsedMessageCutoffTime = 30.days.inWholeMilliseconds
|
||||
|
||||
commit(agedOut)
|
||||
commit(randomEntry())
|
||||
|
||||
ArchiveCommitAttachmentDeletesJob().run()
|
||||
|
||||
assertThat(deletedFromCdn.captured.map { it.mediaId }).contains(agedOut.mediaId)
|
||||
}
|
||||
|
||||
/**
|
||||
* The exporter includes messages on `date_received >= cutoff`, so one landing exactly on the threshold is still in the backup. Deleting it would take media
|
||||
* the backup still references.
|
||||
*/
|
||||
@Test
|
||||
fun givenMediaWhoseMessageIsExactlyAtTheMessageCutoff_whenIRun_thenIExpectItRetained() {
|
||||
val attachmentId = seedFinalizedAttachment(byteArrayOf(2, 2, 2, 2, 2), receivedAt = 30.days)
|
||||
val boundary = entryFor(attachmentId)
|
||||
SignalStore.backup.lastUsedMessageCutoffTime = 30.days.inWholeMilliseconds
|
||||
|
||||
commit(boundary)
|
||||
commit(randomEntry())
|
||||
|
||||
ArchiveCommitAttachmentDeletesJob().run()
|
||||
|
||||
assertThat(remainingOldMediaIds()).contains(boundary.mediaId)
|
||||
coVerify(exactly = 0) { ArchiveCommitAttachmentDeletesJob.deleteMediaObjectsFromCdn(any(), any(), any(), any()) }
|
||||
}
|
||||
|
||||
/**
|
||||
* A wallpaper has no message row at all. Phrasing the cutoff as "its message is recent enough" would drop wallpapers out of the referent set and delete media
|
||||
* the user still has applied, so the check only disqualifies an attachment whose message exists and is too old.
|
||||
*/
|
||||
@Test
|
||||
fun givenAWallpaperAndAMessageCutoff_whenIRun_thenIExpectItRetained() {
|
||||
val attachmentId = SignalDatabase.attachments.insertWallpaper(ByteArrayInputStream(byteArrayOf(7, 7, 7, 7, 7)))
|
||||
val wallpaper = entryFor(attachmentId)
|
||||
SignalStore.backup.lastUsedMessageCutoffTime = 30.days.inWholeMilliseconds
|
||||
|
||||
commit(wallpaper)
|
||||
commit(randomEntry())
|
||||
|
||||
ArchiveCommitAttachmentDeletesJob().run()
|
||||
|
||||
assertThat(remainingOldMediaIds()).contains(wallpaper.mediaId)
|
||||
coVerify(exactly = 0) { ArchiveCommitAttachmentDeletesJob.deleteMediaObjectsFromCdn(any(), any(), any(), any()) }
|
||||
}
|
||||
|
||||
/**
|
||||
* Cutting off messages only happens for backups over the size limit, which is vanishingly rare, so the ordinary case has to behave exactly as it did before.
|
||||
*/
|
||||
@Test
|
||||
fun givenNoMessageCutoff_whenIRun_thenIExpectAncientMediaRetained() {
|
||||
val attachmentId = seedFinalizedAttachment(byteArrayOf(3, 3, 3, 3, 3), receivedAt = 0.days)
|
||||
val ancient = entryFor(attachmentId)
|
||||
|
||||
commit(ancient)
|
||||
commit(randomEntry())
|
||||
|
||||
ArchiveCommitAttachmentDeletesJob().run()
|
||||
|
||||
assertThat(remainingOldMediaIds()).contains(ancient.mediaId)
|
||||
coVerify(exactly = 0) { ArchiveCommitAttachmentDeletesJob.deleteMediaObjectsFromCdn(any(), any(), any(), any()) }
|
||||
}
|
||||
|
||||
/**
|
||||
* Media is keyed by hash and remote key, so the same bytes can hang off several messages. One of them surviving the cutoff means the backup still references
|
||||
* the media, and it has to be judged referenced rather than per-row.
|
||||
*/
|
||||
@Test
|
||||
fun givenMediaReferencedByBothAnAgedOutAndACurrentMessage_whenIRun_thenIExpectItRetained() {
|
||||
val data = byteArrayOf(4, 5, 6, 7, 8)
|
||||
val agedOutId = seedFinalizedAttachment(data, receivedAt = 10.days)
|
||||
val currentId = seedFinalizedAttachment(data, receivedAt = 40.days)
|
||||
|
||||
val shared = entryFor(agedOutId)
|
||||
assertThat(entryFor(currentId).mediaId).isEqualTo(shared.mediaId)
|
||||
|
||||
SignalStore.backup.lastUsedMessageCutoffTime = 30.days.inWholeMilliseconds
|
||||
|
||||
commit(shared)
|
||||
commit(randomEntry())
|
||||
|
||||
ArchiveCommitAttachmentDeletesJob().run()
|
||||
|
||||
assertThat(remainingOldMediaIds()).contains(shared.mediaId)
|
||||
coVerify(exactly = 0) { ArchiveCommitAttachmentDeletesJob.deleteMediaObjectsFromCdn(any(), any(), any(), any()) }
|
||||
}
|
||||
|
||||
/**
|
||||
* Full-size and thumbnail media share a hash and key but are judged by different rules, so a thumbnail can be reclaimable while its full-size copy is not.
|
||||
*/
|
||||
@Test
|
||||
fun givenAThumbnailWhoseAttachmentStillWantsIt_whenIRun_thenIExpectItRetained() {
|
||||
val attachmentId = seedFinalizedAttachment(byteArrayOf(5, 1, 5, 1, 5))
|
||||
val thumbnail = entryFor(attachmentId, isThumbnail = true)
|
||||
|
||||
commit(thumbnail)
|
||||
commit(randomEntry())
|
||||
|
||||
ArchiveCommitAttachmentDeletesJob().run()
|
||||
|
||||
assertThat(remainingOldMediaIds()).contains(thumbnail.mediaId)
|
||||
coVerify(exactly = 0) { ArchiveCommitAttachmentDeletesJob.deleteMediaObjectsFromCdn(any(), any(), any(), any()) }
|
||||
}
|
||||
|
||||
/**
|
||||
* Wallpapers never get a thumbnail written into the snapshot, so one already on the CDN is unreachable bookkeeping. Its full-size object still has a referent
|
||||
* and has to survive, which is what makes this a thumbnail-only reclaim rather than a blanket delete.
|
||||
*/
|
||||
@Test
|
||||
fun givenAWallpaperThumbnail_whenIRun_thenIExpectOnlyTheThumbnailDeletedFromTheCdn() {
|
||||
val attachmentId = SignalDatabase.attachments.insertWallpaper(ByteArrayInputStream(byteArrayOf(6, 2, 6, 2, 6)))
|
||||
val thumbnail = entryFor(attachmentId, isThumbnail = true)
|
||||
val fullSize = entryFor(attachmentId)
|
||||
|
||||
commit(thumbnail, fullSize)
|
||||
commit(randomEntry())
|
||||
|
||||
ArchiveCommitAttachmentDeletesJob().run()
|
||||
|
||||
val deletedMediaIds = deletedFromCdn.captured.map { it.mediaId }
|
||||
assertThat(deletedMediaIds).contains(thumbnail.mediaId)
|
||||
assertThat(deletedMediaIds).doesNotContain(fullSize.mediaId)
|
||||
}
|
||||
|
||||
/**
|
||||
* The message cutoff has to reach the thumbnail path too, otherwise media dropped from the backup keeps half of its CDN footprint.
|
||||
*/
|
||||
@Test
|
||||
fun givenAThumbnailWhoseMessagePredatesTheMessageCutoff_whenIRun_thenIExpectItDeletedFromTheCdn() {
|
||||
val attachmentId = seedFinalizedAttachment(byteArrayOf(7, 3, 7, 3, 7), receivedAt = 10.days)
|
||||
val thumbnail = entryFor(attachmentId, isThumbnail = true)
|
||||
SignalStore.backup.lastUsedMessageCutoffTime = 30.days.inWholeMilliseconds
|
||||
|
||||
commit(thumbnail)
|
||||
commit(randomEntry())
|
||||
|
||||
ArchiveCommitAttachmentDeletesJob().run()
|
||||
|
||||
assertThat(deletedFromCdn.captured.map { it.mediaId }).contains(thumbnail.mediaId)
|
||||
}
|
||||
|
||||
private fun entryFor(attachmentId: AttachmentId, isThumbnail: Boolean = false): MediaEntry {
|
||||
val attachment = SignalDatabase.attachments.getAttachment(attachmentId)!!
|
||||
val plaintextHash = attachment.dataHash!!.decodeBase64OrThrow()
|
||||
val remoteKey = attachment.remoteKey!!.decodeBase64OrThrow()
|
||||
|
||||
val mediaName = if (isThumbnail) {
|
||||
MediaName.fromPlaintextHashAndRemoteKeyForThumbnail(plaintextHash, remoteKey)
|
||||
} else {
|
||||
MediaName.fromPlaintextHashAndRemoteKey(plaintextHash, remoteKey)
|
||||
}
|
||||
|
||||
return MediaEntry(
|
||||
mediaId = mediaName.toMediaId(SignalStore.backup.mediaRootBackupKey).encode(),
|
||||
cdn = 3,
|
||||
plaintextHash = plaintextHash,
|
||||
remoteKey = remoteKey,
|
||||
isThumbnail = isThumbnail
|
||||
)
|
||||
}
|
||||
|
||||
private fun randomEntry(cdn: Int? = 3): MediaEntry {
|
||||
val plaintextHash = Random.nextBytes(32)
|
||||
val remoteKey = Random.nextBytes(32)
|
||||
|
||||
return MediaEntry(
|
||||
mediaId = MediaName.fromPlaintextHashAndRemoteKey(plaintextHash, remoteKey).toMediaId(SignalStore.backup.mediaRootBackupKey).encode(),
|
||||
cdn = cdn,
|
||||
plaintextHash = plaintextHash,
|
||||
remoteKey = remoteKey,
|
||||
isThumbnail = false
|
||||
)
|
||||
}
|
||||
|
||||
private fun commit(vararg entries: MediaEntry) {
|
||||
commit(entries.toList())
|
||||
}
|
||||
|
||||
private fun commit(entries: List<MediaEntry>) {
|
||||
SignalDatabase.backupMediaSnapshots.writePendingMediaEntries(entries)
|
||||
SignalDatabase.backupMediaSnapshots.commitPendingRows()
|
||||
}
|
||||
|
||||
private fun remainingOldMediaIds(): List<String> {
|
||||
return SignalDatabase.backupMediaSnapshots.getPageOfOldMediaEntries(pageSize = (jobPageSize * 3)).map { it.mediaId }
|
||||
}
|
||||
|
||||
private fun seedFinalizedAttachment(data: ByteArray, receivedAt: Duration = 0.days): AttachmentId {
|
||||
val attachment = createAttachmentPointer(Random.nextBytes(32), data.size)
|
||||
val messageResult = SignalDatabase.messages.insertMessageInbox(createIncomingMessage(serverTime = receivedAt, attachment = attachment)).get()
|
||||
val attachmentId = messageResult.insertedAttachments!![attachment]!!
|
||||
SignalDatabase.attachments.setTransferState(messageResult.messageId, attachmentId, AttachmentTable.TRANSFER_PROGRESS_STARTED)
|
||||
SignalDatabase.attachments.finalizeAttachmentAfterDownload(messageResult.messageId, attachmentId, ByteArrayInputStream(data))
|
||||
return attachmentId
|
||||
}
|
||||
|
||||
private fun createIncomingMessage(serverTime: Duration, attachment: Attachment): IncomingMessage {
|
||||
return IncomingMessage(
|
||||
type = MessageType.NORMAL,
|
||||
from = harness.others[0],
|
||||
body = null,
|
||||
sentTimeMillis = serverTime.inWholeMilliseconds,
|
||||
serverTimeMillis = serverTime.inWholeMilliseconds,
|
||||
receivedTimeMillis = serverTime.inWholeMilliseconds,
|
||||
attachments = listOf(attachment)
|
||||
)
|
||||
}
|
||||
|
||||
private fun createAttachmentPointer(key: ByteArray, size: Int): Attachment {
|
||||
return PointerAttachment.forPointer(
|
||||
pointer = Optional.of(
|
||||
SignalServiceAttachmentPointer(
|
||||
cdnNumber = 3,
|
||||
remoteId = SignalServiceAttachmentRemoteId.V4("asdf"),
|
||||
contentType = MediaUtil.IMAGE_JPEG,
|
||||
key = key,
|
||||
size = Optional.of(size),
|
||||
preview = Optional.empty(),
|
||||
width = 2,
|
||||
height = 2,
|
||||
digest = Optional.of(byteArrayOf()),
|
||||
incrementalDigest = Optional.empty(),
|
||||
incrementalMacChunkSize = 0,
|
||||
fileName = Optional.of("file.jpg"),
|
||||
voiceNote = false,
|
||||
isBorderless = false,
|
||||
isGif = false,
|
||||
caption = Optional.empty(),
|
||||
blurHash = Optional.empty(),
|
||||
uploadTimestamp = 0,
|
||||
uuid = null
|
||||
)
|
||||
)
|
||||
).get()
|
||||
}
|
||||
}
|
||||
+18
@@ -83,9 +83,11 @@ import org.thoughtcrime.securesms.components.settings.app.internal.backup.Intern
|
||||
import org.thoughtcrime.securesms.dependencies.AppDependencies
|
||||
import org.thoughtcrime.securesms.jobs.ArchiveAttachmentBackfillJob
|
||||
import org.thoughtcrime.securesms.jobs.ArchiveAttachmentReconciliationJob
|
||||
import org.thoughtcrime.securesms.jobs.ArchiveCommitAttachmentDeletesJob
|
||||
import org.thoughtcrime.securesms.jobs.ArchiveThumbnailBackfillJob
|
||||
import org.thoughtcrime.securesms.jobs.BackupRestoreMediaJob
|
||||
import org.thoughtcrime.securesms.jobs.LocalBackupJob
|
||||
import org.thoughtcrime.securesms.jobs.OptimizeMediaJob
|
||||
import org.thoughtcrime.securesms.keyvalue.BackupValues
|
||||
import org.thoughtcrime.securesms.keyvalue.SignalStore
|
||||
import org.thoughtcrime.securesms.registration.ui.restore.local.RestoreLocalBackupActivity
|
||||
@@ -161,6 +163,8 @@ class InternalBackupPlaygroundFragment : ComposeFragment() {
|
||||
onEnqueueAttachmentBackfillJob = { AppDependencies.jobManager.add(ArchiveAttachmentBackfillJob()) },
|
||||
onEnqueueThumbnailBackfillJob = { AppDependencies.jobManager.add(ArchiveThumbnailBackfillJob()) },
|
||||
onEnqueueMediaRestoreClicked = { AppDependencies.jobManager.add(BackupRestoreMediaJob()) },
|
||||
onEnqueueOptimizeMediaClicked = { AppDependencies.jobManager.add(OptimizeMediaJob()) },
|
||||
onEnqueueCommitDeletesClicked = { AppDependencies.jobManager.add(ArchiveCommitAttachmentDeletesJob()) },
|
||||
onHaltAllBackupJobsClicked = { viewModel.haltAllJobs() },
|
||||
onValidateBackupClicked = { viewModel.validateBackup() },
|
||||
onSaveEncryptedBackupToDiskClicked = {
|
||||
@@ -357,6 +361,8 @@ fun Screen(
|
||||
onEnqueueMediaRestoreClicked: () -> Unit = {},
|
||||
onEnqueueAttachmentBackfillJob: () -> Unit = {},
|
||||
onEnqueueThumbnailBackfillJob: () -> Unit = {},
|
||||
onEnqueueOptimizeMediaClicked: () -> Unit = {},
|
||||
onEnqueueCommitDeletesClicked: () -> Unit = {},
|
||||
onWipeDataAndRestoreFromRemoteClicked: () -> Unit = {},
|
||||
onHaltAllBackupJobsClicked: () -> Unit = {},
|
||||
onSavePlaintextCopyOfRemoteBackupClicked: () -> Unit = {},
|
||||
@@ -456,6 +462,18 @@ fun Screen(
|
||||
onClick = onEnqueueMediaRestoreClicked
|
||||
)
|
||||
|
||||
Rows.TextRow(
|
||||
text = "Enqueue optimize media job",
|
||||
label = "Schedules a job that will offload local copies of media the archive CDN has confirmed. Normally only runs after a backup.",
|
||||
onClick = onEnqueueOptimizeMediaClicked
|
||||
)
|
||||
|
||||
Rows.TextRow(
|
||||
text = "Enqueue commit deletes job",
|
||||
label = "Schedules a job that will delete unreferenced media from the archive CDN. Normally only runs after a backup.",
|
||||
onClick = onEnqueueCommitDeletesClicked
|
||||
)
|
||||
|
||||
Rows.TextRow(
|
||||
text = "Halt all backup jobs",
|
||||
label = "Stops all backup-related jobs to the best of our ability.",
|
||||
|
||||
+1
@@ -290,6 +290,7 @@ class InternalBackupPlaygroundViewModel : ViewModel() {
|
||||
fun wipeAllDataAndRestoreFromRemote(afterDbRestoreCallback: () -> Unit) {
|
||||
SignalExecutors.BOUNDED_IO.execute {
|
||||
SignalStore.backup.restoreWithCellular = false
|
||||
SignalStore.backup.clearArchiveVerificationState()
|
||||
restoreFromRemote(afterDbRestoreCallback)
|
||||
}
|
||||
}
|
||||
|
||||
+10
-5
@@ -24,7 +24,7 @@ import org.thoughtcrime.securesms.database.SignalDatabase
|
||||
import org.thoughtcrime.securesms.database.SignalDatabase.Companion.media
|
||||
import org.thoughtcrime.securesms.database.ThreadTable
|
||||
import org.thoughtcrime.securesms.dependencies.AppDependencies
|
||||
import org.thoughtcrime.securesms.jobs.OptimizeMediaJob
|
||||
import org.thoughtcrime.securesms.jobs.BackupMessagesJob
|
||||
import org.thoughtcrime.securesms.jobs.RestoreOptimizedMediaJob
|
||||
import org.thoughtcrime.securesms.keyvalue.KeepMessagesDuration
|
||||
import org.thoughtcrime.securesms.keyvalue.SignalStore
|
||||
@@ -37,7 +37,8 @@ class ManageStorageSettingsViewModel : ViewModel() {
|
||||
lengthLimit = if (SignalStore.settings.isTrimByLengthEnabled) SignalStore.settings.threadTrimLength else ManageStorageState.NO_LIMIT,
|
||||
syncTrimDeletes = SignalStore.settings.shouldSyncThreadTrimDeletes(),
|
||||
localBackupsEnabled = SignalStore.backup.newLocalBackupsEnabled,
|
||||
isPrimary = SignalStore.account.isPrimaryDevice
|
||||
isPrimary = SignalStore.account.isPrimaryDevice,
|
||||
initialOptimizeStorage = SignalStore.backup.optimizeStorage
|
||||
)
|
||||
)
|
||||
val state = store.asStateFlow()
|
||||
@@ -121,7 +122,7 @@ class ManageStorageSettingsViewModel : ViewModel() {
|
||||
store.update {
|
||||
it.copy(
|
||||
onDeviceStorageOptimizationState = if (enabled) OnDeviceStorageOptimizationState.ENABLED else OnDeviceStorageOptimizationState.DISABLED,
|
||||
storageOptimizationStateChanged = true
|
||||
storageOptimizationStateChanged = enabled != it.initialOptimizeStorage
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -146,7 +147,10 @@ class ManageStorageSettingsViewModel : ViewModel() {
|
||||
if (state.value.storageOptimizationStateChanged) {
|
||||
when (state.value.onDeviceStorageOptimizationState) {
|
||||
OnDeviceStorageOptimizationState.DISABLED -> RestoreOptimizedMediaJob.enqueue()
|
||||
OnDeviceStorageOptimizationState.ENABLED -> OptimizeMediaJob.enqueue()
|
||||
|
||||
// Backing up first guarantees a media snapshot exists to reconcile against, and the backup enqueues the offload for us once it succeeds
|
||||
OnDeviceStorageOptimizationState.ENABLED -> BackupMessagesJob.enqueue()
|
||||
|
||||
else -> Unit
|
||||
}
|
||||
}
|
||||
@@ -184,7 +188,8 @@ class ManageStorageSettingsViewModel : ViewModel() {
|
||||
val storageOptimizationStateChanged: Boolean = false,
|
||||
val isPaidTierPending: Boolean = false,
|
||||
val localBackupsEnabled: Boolean = false,
|
||||
val isPrimary: Boolean = true
|
||||
val isPrimary: Boolean = true,
|
||||
val initialOptimizeStorage: Boolean = false
|
||||
) {
|
||||
companion object {
|
||||
const val NO_LIMIT = 0
|
||||
|
||||
@@ -426,7 +426,9 @@ class AttachmentTable(
|
||||
}
|
||||
|
||||
/**
|
||||
* Filters thumbnail snapshot entries down to only those that have at least one eligible attachment capable of thumbnail upload.
|
||||
* Already-archived thumbnails are kept even with no local data file to upload from, because offloaded media still legitimately references its thumbnail.
|
||||
* Dropping those entries would take them out of the snapshot, which marks them for deletion off of the CDN. A restored thumbnail counts as archived too: we
|
||||
* downloaded it from the CDN, so it is there regardless of what the archive state says.
|
||||
*/
|
||||
fun filterThumbnailsWithoutEligibleAttachment(entries: Set<BackupMediaSnapshotTable.MediaEntry>): Set<BackupMediaSnapshotTable.MediaEntry> {
|
||||
if (entries.isEmpty()) {
|
||||
@@ -442,16 +444,7 @@ class AttachmentTable(
|
||||
readableDatabase
|
||||
.select(DATA_HASH_END, REMOTE_KEY)
|
||||
.from(TABLE_NAME)
|
||||
.where(
|
||||
"""
|
||||
$DATA_HASH_END NOT NULL AND
|
||||
$REMOTE_KEY NOT NULL AND
|
||||
$DATA_FILE NOT NULL AND
|
||||
$TRANSFER_STATE = $TRANSFER_PROGRESS_DONE AND
|
||||
$QUOTE = 0 AND
|
||||
$ARCHIVE_THUMBNAIL_TRANSFER_STATE != ${ArchiveTransferState.PERMANENT_FAILURE.value}
|
||||
"""
|
||||
)
|
||||
.where(buildThumbnailEligibilityClause())
|
||||
.run()
|
||||
.forEach { cursor ->
|
||||
val hashEnd = cursor.requireNonNullString(DATA_HASH_END)
|
||||
@@ -666,7 +659,7 @@ class AttachmentTable(
|
||||
fun getLast30DaysOfRestorableAttachments(batchSize: Int): List<RestorableAttachment> {
|
||||
val thirtyDaysAgo = System.currentTimeMillis().milliseconds - 30.days
|
||||
return readableDatabase
|
||||
.select("$TABLE_NAME.$ID", MESSAGE_ID, DATA_SIZE, DATA_HASH_END, REMOTE_KEY, STICKER_PACK_ID)
|
||||
.select("$TABLE_NAME.$ID", MESSAGE_ID, DATA_SIZE, DATA_HASH_END, REMOTE_KEY, STICKER_PACK_ID, CONTENT_TYPE, QUOTE, STICKER_ID)
|
||||
.from("$TABLE_NAME LEFT JOIN ${MessageTable.TABLE_NAME} ON ${MessageTable.TABLE_NAME}.${MessageTable.ID} = $TABLE_NAME.$MESSAGE_ID")
|
||||
.where("$TRANSFER_STATE = ? AND (${MessageTable.TABLE_NAME}.${MessageTable.DATE_RECEIVED} >= ? OR $MESSAGE_ID = ?)", TRANSFER_NEEDS_RESTORE, thirtyDaysAgo.inWholeMilliseconds, WALLPAPER_MESSAGE_ID)
|
||||
.limit(batchSize)
|
||||
@@ -679,7 +672,10 @@ class AttachmentTable(
|
||||
size = it.requireLong(DATA_SIZE),
|
||||
plaintextHash = it.requireString(DATA_HASH_END)?.let { hash -> Base64.decode(hash) },
|
||||
remoteKey = it.requireString(REMOTE_KEY)?.let { key -> Base64.decode(key) },
|
||||
stickerPackId = it.requireString(STICKER_PACK_ID)
|
||||
stickerPackId = it.requireString(STICKER_PACK_ID),
|
||||
contentType = it.requireString(CONTENT_TYPE),
|
||||
quote = it.requireBoolean(QUOTE),
|
||||
stickerId = it.requireInt(STICKER_ID)
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -691,7 +687,7 @@ class AttachmentTable(
|
||||
fun getOlderRestorableAttachments(batchSize: Int): List<RestorableAttachment> {
|
||||
val thirtyDaysAgo = System.currentTimeMillis().milliseconds - 30.days
|
||||
return readableDatabase
|
||||
.select("$TABLE_NAME.$ID", MESSAGE_ID, DATA_SIZE, DATA_HASH_END, REMOTE_KEY, STICKER_PACK_ID)
|
||||
.select("$TABLE_NAME.$ID", MESSAGE_ID, DATA_SIZE, DATA_HASH_END, REMOTE_KEY, STICKER_PACK_ID, CONTENT_TYPE, QUOTE, STICKER_ID)
|
||||
.from("$TABLE_NAME LEFT JOIN ${MessageTable.TABLE_NAME} ON ${MessageTable.TABLE_NAME}.${MessageTable.ID} = $TABLE_NAME.$MESSAGE_ID")
|
||||
.where("$TRANSFER_STATE = ? AND (${MessageTable.TABLE_NAME}.${MessageTable.DATE_RECEIVED} < ? OR $MESSAGE_ID = ?)", TRANSFER_NEEDS_RESTORE, thirtyDaysAgo.inWholeMilliseconds, WALLPAPER_MESSAGE_ID)
|
||||
.limit(batchSize)
|
||||
@@ -704,14 +700,17 @@ class AttachmentTable(
|
||||
size = it.requireLong(DATA_SIZE),
|
||||
plaintextHash = it.requireString(DATA_HASH_END)?.let { hash -> Base64.decode(hash) },
|
||||
remoteKey = it.requireString(REMOTE_KEY)?.let { key -> Base64.decode(key) },
|
||||
stickerPackId = it.requireString(STICKER_PACK_ID)
|
||||
stickerPackId = it.requireString(STICKER_PACK_ID),
|
||||
contentType = it.requireString(CONTENT_TYPE),
|
||||
quote = it.requireBoolean(QUOTE),
|
||||
stickerId = it.requireInt(STICKER_ID)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
fun getRestorableOptimizedAttachments(): List<RestorableAttachment> {
|
||||
return readableDatabase
|
||||
.select(ID, MESSAGE_ID, DATA_SIZE, DATA_HASH_END, REMOTE_KEY, STICKER_PACK_ID)
|
||||
.select(ID, MESSAGE_ID, DATA_SIZE, DATA_HASH_END, REMOTE_KEY, STICKER_PACK_ID, CONTENT_TYPE, QUOTE, STICKER_ID)
|
||||
.from(TABLE_NAME)
|
||||
.where("$TRANSFER_STATE = ? AND $DATA_HASH_END NOT NULL AND $REMOTE_KEY NOT NULL", TRANSFER_RESTORE_OFFLOADED)
|
||||
.orderBy("$ID DESC")
|
||||
@@ -723,7 +722,10 @@ class AttachmentTable(
|
||||
size = it.requireLong(DATA_SIZE),
|
||||
plaintextHash = it.requireString(DATA_HASH_END)?.let { hash -> Base64.decode(hash) },
|
||||
remoteKey = it.requireString(REMOTE_KEY)?.let { key -> Base64.decode(key) },
|
||||
stickerPackId = it.requireString(STICKER_PACK_ID)
|
||||
stickerPackId = it.requireString(STICKER_PACK_ID),
|
||||
contentType = it.requireString(CONTENT_TYPE),
|
||||
quote = it.requireBoolean(QUOTE),
|
||||
stickerId = it.requireInt(STICKER_ID)
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -948,16 +950,7 @@ class AttachmentTable(
|
||||
fun doAnyThumbnailsNeedArchiveUpload(): Boolean {
|
||||
return readableDatabase
|
||||
.exists("$TABLE_NAME INNER JOIN ${MessageTable.TABLE_NAME} ON $TABLE_NAME.$MESSAGE_ID = ${MessageTable.TABLE_NAME}.${MessageTable.ID}")
|
||||
.where(
|
||||
"""
|
||||
${buildAttachmentsThatCanArchiveQuery("$ARCHIVE_THUMBNAIL_TRANSFER_STATE IN (${ArchiveTransferState.NONE.value}, ${ArchiveTransferState.TEMPORARY_FAILURE.value})")} AND
|
||||
$QUOTE = 0 AND
|
||||
$STICKER_ID = -1 AND
|
||||
($CONTENT_TYPE LIKE 'image/%' OR $CONTENT_TYPE LIKE 'video/%') AND
|
||||
$CONTENT_TYPE != 'image/svg+xml' AND
|
||||
$MESSAGE_ID != $WALLPAPER_MESSAGE_ID
|
||||
"""
|
||||
)
|
||||
.where(buildThumbnailsThatNeedArchiveWorkQuery())
|
||||
.run()
|
||||
}
|
||||
|
||||
@@ -996,16 +989,7 @@ class AttachmentTable(
|
||||
return readableDatabase
|
||||
.select("$TABLE_NAME.$ID")
|
||||
.from("$TABLE_NAME INNER JOIN ${MessageTable.TABLE_NAME} ON $TABLE_NAME.$MESSAGE_ID = ${MessageTable.TABLE_NAME}.${MessageTable.ID}")
|
||||
.where(
|
||||
"""
|
||||
${buildAttachmentsThatCanArchiveQuery("$ARCHIVE_THUMBNAIL_TRANSFER_STATE IN (${ArchiveTransferState.NONE.value}, ${ArchiveTransferState.TEMPORARY_FAILURE.value})")} AND
|
||||
$QUOTE = 0 AND
|
||||
$STICKER_ID = -1 AND
|
||||
($CONTENT_TYPE LIKE 'image/%' OR $CONTENT_TYPE LIKE 'video/%') AND
|
||||
$CONTENT_TYPE != 'image/svg+xml' AND
|
||||
$MESSAGE_ID != $WALLPAPER_MESSAGE_ID
|
||||
"""
|
||||
)
|
||||
.where(buildThumbnailsThatNeedArchiveWorkQuery())
|
||||
.run()
|
||||
.readToList { AttachmentId(it.requireLong(ID)) }
|
||||
}
|
||||
@@ -1034,6 +1018,13 @@ class AttachmentTable(
|
||||
.readToSingleObject { ArchiveTransferState.deserialize(it.requireInt(ARCHIVE_THUMBNAIL_TRANSFER_STATE)) }
|
||||
}
|
||||
|
||||
fun hasThumbnailFile(id: AttachmentId): Boolean {
|
||||
return readableDatabase
|
||||
.exists(TABLE_NAME)
|
||||
.where("$ID = ? AND $THUMBNAIL_FILE NOT NULL", id.id)
|
||||
.run()
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets the archive transfer state for the given attachment and all other attachments that share the same data file.
|
||||
*/
|
||||
@@ -1161,28 +1152,63 @@ class AttachmentTable(
|
||||
/**
|
||||
* Resets the archive upload state by hash/key if we believe the attachment should have been uploaded already.
|
||||
*/
|
||||
fun resetArchiveTransferStateByPlaintextHashAndRemoteKeyIfNecessary(plaintextHash: ByteArray, remoteKey: ByteArray): Boolean {
|
||||
return writableDatabase
|
||||
.update(TABLE_NAME)
|
||||
.values(
|
||||
fun resetArchiveTransferStateByPlaintextHashAndRemoteKeyIfNecessary(plaintextHash: ByteArray, remoteKey: ByteArray): ArchiveTransferStateResetResult {
|
||||
return resetArchiveTransferStateIfNecessary(
|
||||
plaintextHash = plaintextHash,
|
||||
remoteKey = remoteKey,
|
||||
stateColumn = ARCHIVE_TRANSFER_STATE,
|
||||
values = contentValuesOf(
|
||||
ARCHIVE_TRANSFER_STATE to ArchiveTransferState.NONE.value,
|
||||
ARCHIVE_CDN to null
|
||||
)
|
||||
.where("$DATA_HASH_END = ? AND $REMOTE_KEY = ? AND $ARCHIVE_TRANSFER_STATE = ${ArchiveTransferState.FINISHED.value}", Base64.encodeWithPadding(plaintextHash), Base64.encodeWithPadding(remoteKey))
|
||||
.run() > 0
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Resets the archive thumbnail upload state by hash/key if we believe the thumbnail should have been uploaded already.
|
||||
*/
|
||||
fun resetArchiveThumbnailTransferStateByPlaintextHashAndRemoteKeyIfNecessary(plaintextHash: ByteArray, remoteKey: ByteArray): Boolean {
|
||||
return writableDatabase
|
||||
.update(TABLE_NAME)
|
||||
.values(
|
||||
ARCHIVE_THUMBNAIL_TRANSFER_STATE to ArchiveTransferState.NONE.value
|
||||
)
|
||||
.where("$DATA_HASH_END = ? AND $REMOTE_KEY = ? AND $ARCHIVE_THUMBNAIL_TRANSFER_STATE = ${ArchiveTransferState.FINISHED.value}", Base64.encodeWithPadding(plaintextHash), Base64.encodeWithPadding(remoteKey))
|
||||
.run() > 0
|
||||
fun resetArchiveThumbnailTransferStateByPlaintextHashAndRemoteKeyIfNecessary(plaintextHash: ByteArray, remoteKey: ByteArray): ArchiveTransferStateResetResult {
|
||||
return resetArchiveTransferStateIfNecessary(
|
||||
plaintextHash = plaintextHash,
|
||||
remoteKey = remoteKey,
|
||||
stateColumn = ARCHIVE_THUMBNAIL_TRANSFER_STATE,
|
||||
values = contentValuesOf(ARCHIVE_THUMBNAIL_TRANSFER_STATE to ArchiveTransferState.NONE.value)
|
||||
)
|
||||
}
|
||||
|
||||
private fun resetArchiveTransferStateIfNecessary(plaintextHash: ByteArray, remoteKey: ByteArray, stateColumn: String, values: ContentValues): ArchiveTransferStateResetResult {
|
||||
val encodedHash = Base64.encodeWithPadding(plaintextHash)
|
||||
val encodedKey = Base64.encodeWithPadding(remoteKey)
|
||||
val finishedClause = "$DATA_HASH_END = ? AND $REMOTE_KEY = ? AND $stateColumn = ${ArchiveTransferState.FINISHED.value}"
|
||||
|
||||
return writableDatabase.withinTransaction { db ->
|
||||
val isFinished = db
|
||||
.exists(TABLE_NAME)
|
||||
.where(finishedClause, encodedHash, encodedKey)
|
||||
.run()
|
||||
|
||||
if (!isFinished) {
|
||||
return@withinTransaction ArchiveTransferStateResetResult.NOT_NEEDED
|
||||
}
|
||||
|
||||
val hasLocalData = db
|
||||
.exists(TABLE_NAME)
|
||||
.where("$DATA_HASH_END = ? AND $REMOTE_KEY = ? AND $DATA_FILE NOT NULL", encodedHash, encodedKey)
|
||||
.run()
|
||||
|
||||
// With no local data file there is nothing to re-upload, and clearing the state makes the export emit a tombstone instead of a locator, which in turn marks
|
||||
// the media for deletion off of the CDN.
|
||||
if (!hasLocalData) {
|
||||
return@withinTransaction ArchiveTransferStateResetResult.SKIPPED_NO_LOCAL_DATA
|
||||
}
|
||||
|
||||
db.update(TABLE_NAME)
|
||||
.values(values)
|
||||
.where(finishedClause, encodedHash, encodedKey)
|
||||
.run()
|
||||
|
||||
ArchiveTransferStateResetResult.RESET
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -1233,49 +1259,128 @@ class AttachmentTable(
|
||||
*
|
||||
* Marking offloaded only clears the strong references to the on disk file and clears other local file data like hashes.
|
||||
* Another operation must run to actually delete the data from disk. See [deleteAbandonedAttachmentFiles].
|
||||
*
|
||||
* @param lastCompletedCrawlVersion The snapshot version of the most recent reconciliation that finished crawling the archive CDN, or negative if none ever has.
|
||||
* @param pageSize How many candidates to derive mediaIds for at a time.
|
||||
*/
|
||||
fun markEligibleAttachmentsAsOptimized(minimumAge: Duration = 30.days) {
|
||||
val now = System.currentTimeMillis()
|
||||
fun markEligibleAttachmentsAsOptimized(lastCompletedCrawlVersion: Long, minimumAge: Duration = 30.days, now: Long = System.currentTimeMillis(), pageSize: Int = 500) {
|
||||
if (lastCompletedCrawlVersion < 0) {
|
||||
Log.w(TAG, "No reconciliation has ever completed on this device. Refusing to offload anything.", true)
|
||||
return
|
||||
}
|
||||
|
||||
val subSelect = """
|
||||
SELECT $TABLE_NAME.$ID
|
||||
FROM $TABLE_NAME
|
||||
INNER JOIN ${MessageTable.TABLE_NAME} ON ${MessageTable.TABLE_NAME}.${MessageTable.ID} = $TABLE_NAME.$MESSAGE_ID
|
||||
WHERE
|
||||
// Everything that looks offloadable locally, saying nothing about whether the archive CDN actually has the bytes
|
||||
fun eligibilityClause(prefix: String): String = """
|
||||
$prefix$OFFLOAD_RESTORED_AT < ${now - 7.days.inWholeMilliseconds} AND
|
||||
$prefix$TRANSFER_STATE = $TRANSFER_PROGRESS_DONE AND
|
||||
$prefix$ARCHIVE_TRANSFER_STATE = ${ArchiveTransferState.FINISHED.value} AND
|
||||
$prefix$DATA_FILE IS NOT NULL AND
|
||||
$prefix$STICKER_ID = -1 AND
|
||||
$prefix$REMOTE_KEY IS NOT NULL AND
|
||||
$prefix$DATA_HASH_END IS NOT NULL AND
|
||||
(
|
||||
$TABLE_NAME.$OFFLOAD_RESTORED_AT < ${now - 7.days.inWholeMilliseconds} AND
|
||||
$TABLE_NAME.$TRANSFER_STATE = $TRANSFER_PROGRESS_DONE AND
|
||||
$TABLE_NAME.$ARCHIVE_TRANSFER_STATE = ${ArchiveTransferState.FINISHED.value} AND
|
||||
$TABLE_NAME.$DATA_FILE IS NOT NULL AND
|
||||
$TABLE_NAME.$STICKER_ID = -1 AND
|
||||
$TABLE_NAME.$REMOTE_KEY IS NOT NULL AND
|
||||
$TABLE_NAME.$DATA_HASH_END IS NOT NULL AND
|
||||
(
|
||||
$TABLE_NAME.$THUMBNAIL_FILE IS NOT NULL OR
|
||||
NOT ($TABLE_NAME.$CONTENT_TYPE LIKE 'image/%' OR $TABLE_NAME.$CONTENT_TYPE LIKE 'video/%') OR
|
||||
$TABLE_NAME.$CONTENT_TYPE = 'image/svg+xml'
|
||||
)
|
||||
)
|
||||
AND
|
||||
(
|
||||
${MessageTable.TABLE_NAME}.${MessageTable.DATE_RECEIVED} < ${now - minimumAge.inWholeMilliseconds}
|
||||
$prefix$THUMBNAIL_FILE IS NOT NULL OR
|
||||
NOT ($prefix$CONTENT_TYPE LIKE 'image/%' OR $prefix$CONTENT_TYPE LIKE 'video/%') OR
|
||||
$prefix$CONTENT_TYPE = 'image/svg+xml'
|
||||
)
|
||||
"""
|
||||
|
||||
val count = writableDatabase
|
||||
.update(TABLE_NAME)
|
||||
.values(
|
||||
TRANSFER_STATE to TRANSFER_RESTORE_OFFLOADED,
|
||||
DATA_FILE to null,
|
||||
DATA_RANDOM to null,
|
||||
TRANSFORM_PROPERTIES to null,
|
||||
DATA_HASH_START to null,
|
||||
OFFLOAD_RESTORED_AT to 0
|
||||
)
|
||||
.where("$ID in ($subSelect)")
|
||||
.run()
|
||||
val oldestAllowedDateReceived = now - minimumAge.inWholeMilliseconds
|
||||
val mediaRootBackupKey = SignalStore.backup.mediaRootBackupKey
|
||||
|
||||
Log.i(TAG, "Marked $count attachments as optimized")
|
||||
var lastId = 0L
|
||||
var offloadedCount = 0
|
||||
var candidateCount = 0
|
||||
var unconfirmedCount = 0
|
||||
|
||||
while (true) {
|
||||
// Paged by id rather than looping until the candidate query runs dry: candidates the CDN never confirms stay eligible forever, so only a strictly advancing
|
||||
// cursor terminates.
|
||||
val candidates: List<Pair<Long, MediaNameParts>> = readableDatabase
|
||||
.rawQuery(
|
||||
"""
|
||||
SELECT $TABLE_NAME.$ID, $TABLE_NAME.$DATA_HASH_END, $TABLE_NAME.$REMOTE_KEY
|
||||
FROM $TABLE_NAME
|
||||
INNER JOIN ${MessageTable.TABLE_NAME} ON ${MessageTable.TABLE_NAME}.${MessageTable.ID} = $TABLE_NAME.$MESSAGE_ID
|
||||
WHERE
|
||||
$TABLE_NAME.$ID > $lastId AND
|
||||
${eligibilityClause("$TABLE_NAME.")} AND
|
||||
${MessageTable.TABLE_NAME}.${MessageTable.DATE_RECEIVED} < $oldestAllowedDateReceived
|
||||
ORDER BY $TABLE_NAME.$ID ASC
|
||||
LIMIT $pageSize
|
||||
"""
|
||||
)
|
||||
.readToList { cursor ->
|
||||
cursor.requireLong(ID) to MediaNameParts(
|
||||
plaintextHash = cursor.requireNonNullString(DATA_HASH_END),
|
||||
remoteKey = cursor.requireNonNullString(REMOTE_KEY)
|
||||
)
|
||||
}
|
||||
|
||||
if (candidates.isEmpty()) {
|
||||
break
|
||||
}
|
||||
|
||||
lastId = candidates.last().first
|
||||
|
||||
// Group by mediaId, which is the only thing the snapshot table can be joined on. Scoped to the page so the derivation cache can't grow without bound.
|
||||
val mediaIdsByMediaName: MutableMap<MediaNameParts, String> = mutableMapOf()
|
||||
val candidateIdsByMediaId: MutableMap<String, MutableList<Long>> = mutableMapOf()
|
||||
|
||||
candidates.forEach { (id, mediaName) ->
|
||||
val mediaId = mediaIdsByMediaName.getOrPut(mediaName) {
|
||||
MediaName.fromPlaintextHashAndRemoteKey(Base64.decode(mediaName.plaintextHash), Base64.decode(mediaName.remoteKey))
|
||||
.toMediaId(mediaRootBackupKey)
|
||||
.encode()
|
||||
}
|
||||
|
||||
candidateIdsByMediaId.getOrPut(mediaId) { mutableListOf() } += id
|
||||
}
|
||||
|
||||
// Narrow to media the server itself told us about. This is the only step backed by evidence that didn't originate on this device.
|
||||
val confirmedOnCdn = SignalDatabase.backupMediaSnapshots.getMediaIdsConfirmedOnCdn(candidateIdsByMediaId.keys, lastCompletedCrawlVersion)
|
||||
val idsToOffload: List<Long> = confirmedOnCdn.flatMap { candidateIdsByMediaId[it] ?: emptyList() }
|
||||
|
||||
candidateCount += candidateIdsByMediaId.size
|
||||
unconfirmedCount += candidateIdsByMediaId.size - confirmedOnCdn.size
|
||||
|
||||
if (idsToOffload.isNotEmpty()) {
|
||||
// Eligibility is re-checked at write time, since anything could have changed while we were deriving mediaIds outside of a transaction.
|
||||
val idQuery = SqlUtil.buildFastCollectionQuery(ID, idsToOffload)
|
||||
|
||||
offloadedCount += writableDatabase
|
||||
.update(TABLE_NAME)
|
||||
.values(
|
||||
TRANSFER_STATE to TRANSFER_RESTORE_OFFLOADED,
|
||||
DATA_FILE to null,
|
||||
DATA_RANDOM to null,
|
||||
TRANSFORM_PROPERTIES to null,
|
||||
DATA_HASH_START to null,
|
||||
OFFLOAD_RESTORED_AT to 0
|
||||
)
|
||||
.where(
|
||||
"""
|
||||
${idQuery.where} AND
|
||||
${eligibilityClause("")} AND
|
||||
EXISTS (
|
||||
SELECT 1
|
||||
FROM ${MessageTable.TABLE_NAME}
|
||||
WHERE
|
||||
${MessageTable.TABLE_NAME}.${MessageTable.ID} = $TABLE_NAME.$MESSAGE_ID AND
|
||||
${MessageTable.TABLE_NAME}.${MessageTable.DATE_RECEIVED} < $oldestAllowedDateReceived
|
||||
)
|
||||
""",
|
||||
idQuery.whereArgs
|
||||
)
|
||||
.run()
|
||||
}
|
||||
}
|
||||
|
||||
if (unconfirmedCount > 0) {
|
||||
Log.w(TAG, "Skipping $unconfirmedCount/$candidateCount candidate media objects that the archive CDN has not confirmed since snapshot version $lastCompletedCrawlVersion.", true)
|
||||
}
|
||||
|
||||
Log.i(TAG, "Marked $offloadedCount attachments as optimized")
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -1659,12 +1764,23 @@ class AttachmentTable(
|
||||
writableDatabase
|
||||
.update(TABLE_NAME)
|
||||
.values(THUMBNAIL_RESTORE_STATE to ThumbnailRestoreState.PERMANENT_FAILURE.value)
|
||||
.where("$ID = ? AND $THUMBNAIL_RESTORE_STATE != ?", attachmentId.id, ThumbnailRestoreState.FINISHED)
|
||||
.where("$ID = ? AND $THUMBNAIL_RESTORE_STATE != ?", attachmentId.id, ThumbnailRestoreState.FINISHED.value)
|
||||
.run()
|
||||
|
||||
notifyConversationListeners(messages.getThreadIdForMessage(mmsId))
|
||||
}
|
||||
|
||||
/**
|
||||
* Records that a thumbnail can't be built from the local data file. Skips rows with a restore pending or completed.
|
||||
*/
|
||||
fun markThumbnailPermanentlyFailedIfUnrestorable(attachmentId: AttachmentId) {
|
||||
writableDatabase
|
||||
.update(TABLE_NAME)
|
||||
.values(THUMBNAIL_RESTORE_STATE to ThumbnailRestoreState.PERMANENT_FAILURE.value)
|
||||
.where("$ID = ? AND $THUMBNAIL_RESTORE_STATE = ?", attachmentId.id, ThumbnailRestoreState.NONE.value)
|
||||
.run()
|
||||
}
|
||||
|
||||
fun setTransferProgressPermanentFailure(attachmentId: AttachmentId, mmsId: Long) {
|
||||
writableDatabase
|
||||
.update(TABLE_NAME)
|
||||
@@ -2571,6 +2687,7 @@ class AttachmentTable(
|
||||
}
|
||||
|
||||
val objectsByMediaId: Map<String, ArchivedMediaObject> = objects.associateBy { it.mediaId }
|
||||
val mediaRootBackupKey = SignalStore.backup.mediaRootBackupKey
|
||||
|
||||
// Collect updates grouped by CDN: Map<cdn, List<Pair<plaintextHash, remoteKey>>>
|
||||
val updatesByCdn: MutableMap<Int, MutableList<Pair<String, String>>> = mutableMapOf()
|
||||
@@ -2588,7 +2705,7 @@ class AttachmentTable(
|
||||
val plaintextHash = Base64.decode(plaintextHashStr)
|
||||
|
||||
val mediaId = MediaName.fromPlaintextHashAndRemoteKey(plaintextHash, remoteKey)
|
||||
.toMediaId(SignalStore.backup.mediaRootBackupKey)
|
||||
.toMediaId(mediaRootBackupKey)
|
||||
.encode()
|
||||
|
||||
val matchingObject = objectsByMediaId[mediaId]
|
||||
@@ -2631,6 +2748,83 @@ class AttachmentTable(
|
||||
return updatedCount
|
||||
}
|
||||
|
||||
/**
|
||||
* A restore only carries a claim about the full-size object, so a CDN listing is the only proof we ever get that a *thumbnail* object exists. Without this a
|
||||
* restored device both drops the thumbnail from its snapshot and re-uploads a thumbnail the CDN already has.
|
||||
*
|
||||
* @return the number of unique (plaintextHash, remoteKey) pairs that were updated
|
||||
*/
|
||||
fun setArchiveThumbnailFinishedForMatchingMediaObjects(objects: Set<ArchivedMediaObject>): Int {
|
||||
if (objects.isEmpty()) {
|
||||
return 0
|
||||
}
|
||||
|
||||
val candidateClause = """
|
||||
$DATA_HASH_END NOT NULL AND
|
||||
$REMOTE_KEY NOT NULL AND
|
||||
$ARCHIVE_TRANSFER_STATE = ${ArchiveTransferState.FINISHED.value} AND
|
||||
$ARCHIVE_THUMBNAIL_TRANSFER_STATE != ${ArchiveTransferState.FINISHED.value}
|
||||
"""
|
||||
|
||||
// Skips the group-by scan below in the common case, where every thumbnail state is already correct and there is nothing to promote.
|
||||
if (!readableDatabase.exists(TABLE_NAME).where(candidateClause).run()) {
|
||||
return 0
|
||||
}
|
||||
|
||||
val mediaIds: Set<String> = objects.map { it.mediaId }.toSet()
|
||||
val mediaRootBackupKey = SignalStore.backup.mediaRootBackupKey
|
||||
val matches: MutableList<MediaNameParts> = mutableListOf()
|
||||
|
||||
readableDatabase
|
||||
.select(DATA_HASH_END, REMOTE_KEY)
|
||||
.from(TABLE_NAME)
|
||||
.where(candidateClause)
|
||||
.groupBy("$DATA_HASH_END, $REMOTE_KEY")
|
||||
.run()
|
||||
.readToList { cursor ->
|
||||
MediaNameParts(
|
||||
plaintextHash = cursor.requireNonNullString(DATA_HASH_END),
|
||||
remoteKey = cursor.requireNonNullString(REMOTE_KEY)
|
||||
)
|
||||
}
|
||||
.forEach { parts ->
|
||||
val thumbnailMediaId = MediaName
|
||||
.fromPlaintextHashAndRemoteKeyForThumbnail(Base64.decode(parts.plaintextHash), Base64.decode(parts.remoteKey))
|
||||
.toMediaId(mediaRootBackupKey)
|
||||
.encode()
|
||||
|
||||
if (thumbnailMediaId in mediaIds) {
|
||||
matches += parts
|
||||
}
|
||||
}
|
||||
|
||||
if (matches.isEmpty()) {
|
||||
return 0
|
||||
}
|
||||
|
||||
var updatedCount = 0
|
||||
|
||||
writableDatabase.withinTransaction { db ->
|
||||
for (batch in matches.chunked(250)) {
|
||||
val whereClause = batch.joinToString(" OR ") { "($DATA_HASH_END = ? AND $REMOTE_KEY = ?)" }
|
||||
val whereArgs = batch.flatMap { listOf(it.plaintextHash, it.remoteKey) }.toTypedArray()
|
||||
|
||||
db.update(TABLE_NAME)
|
||||
.values(ARCHIVE_THUMBNAIL_TRANSFER_STATE to ArchiveTransferState.FINISHED.value)
|
||||
.where(whereClause, *whereArgs)
|
||||
.run()
|
||||
|
||||
updatedCount += batch.size
|
||||
}
|
||||
}
|
||||
|
||||
if (updatedCount > 0) {
|
||||
AppDependencies.databaseObserver.notifyAttachmentUpdatedObservers()
|
||||
}
|
||||
|
||||
return updatedCount
|
||||
}
|
||||
|
||||
fun clearArchiveData(attachmentId: AttachmentId) {
|
||||
writableDatabase
|
||||
.update(TABLE_NAME)
|
||||
@@ -3453,6 +3647,31 @@ class AttachmentTable(
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Thumbnails that need work, which is either "the archive has no copy" or "we have no local copy".
|
||||
*/
|
||||
private fun buildThumbnailsThatNeedArchiveWorkQuery(): String {
|
||||
val stateFilter = """
|
||||
(
|
||||
$ARCHIVE_THUMBNAIL_TRANSFER_STATE IN (${ArchiveTransferState.NONE.value}, ${ArchiveTransferState.TEMPORARY_FAILURE.value}) OR
|
||||
(
|
||||
$ARCHIVE_THUMBNAIL_TRANSFER_STATE = ${ArchiveTransferState.FINISHED.value} AND
|
||||
$THUMBNAIL_FILE IS NULL AND
|
||||
$THUMBNAIL_RESTORE_STATE != ${ThumbnailRestoreState.PERMANENT_FAILURE.value}
|
||||
)
|
||||
)
|
||||
"""
|
||||
|
||||
return """
|
||||
${buildAttachmentsThatCanArchiveQuery(stateFilter)} AND
|
||||
$QUOTE = 0 AND
|
||||
$STICKER_ID = -1 AND
|
||||
($CONTENT_TYPE LIKE 'image/%' OR $CONTENT_TYPE LIKE 'video/%') AND
|
||||
$CONTENT_TYPE != 'image/svg+xml' AND
|
||||
$MESSAGE_ID != $WALLPAPER_MESSAGE_ID
|
||||
"""
|
||||
}
|
||||
|
||||
/**
|
||||
* IMPORTANT: This query may match against rows that have no associated message (like a wallpaper attachment).
|
||||
* This needs to be accounted by allowing nulls when reading any message table row.
|
||||
@@ -3593,6 +3812,7 @@ class AttachmentTable(
|
||||
fun getAttachmentDataForMediaIds(mediaIds: Collection<MediaId>): List<ArchiveAttachmentMatch> {
|
||||
if (mediaIds.isEmpty()) return emptyList()
|
||||
val mediaIdByteStrings = mediaIds.map { it.value.toByteString() }.toSet()
|
||||
val mediaRootBackupKey = SignalStore.backup.mediaRootBackupKey
|
||||
|
||||
val found: MutableList<ArchiveAttachmentMatch> = mutableListOf()
|
||||
|
||||
@@ -3609,13 +3829,13 @@ class AttachmentTable(
|
||||
|
||||
val mediaId = MediaName
|
||||
.fromPlaintextHashAndRemoteKey(plaintextHash, remoteKey)
|
||||
.toMediaId(SignalStore.backup.mediaRootBackupKey)
|
||||
.toMediaId(mediaRootBackupKey)
|
||||
.value
|
||||
.toByteString()
|
||||
|
||||
val mediaIdThumbnail = MediaName
|
||||
.fromPlaintextHashAndRemoteKeyForThumbnail(plaintextHash, remoteKey)
|
||||
.toMediaId(SignalStore.backup.mediaRootBackupKey)
|
||||
.toMediaId(mediaRootBackupKey)
|
||||
.value
|
||||
.toByteString()
|
||||
|
||||
@@ -3635,27 +3855,124 @@ class AttachmentTable(
|
||||
return found
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether an attachment still makes its thumbnail worth having on the archive CDN: we can either upload one from local data, or one is already up there.
|
||||
* Mirrors the snapshot write side, except stickers, which it doesn't exclude either, so filtering them here would delete thumbnails it still wants.
|
||||
*/
|
||||
private fun buildThumbnailEligibilityClause(): String {
|
||||
return """
|
||||
$DATA_HASH_END NOT NULL AND
|
||||
$REMOTE_KEY NOT NULL AND
|
||||
$QUOTE = 0 AND
|
||||
$MESSAGE_ID != $WALLPAPER_MESSAGE_ID AND
|
||||
($CONTENT_TYPE LIKE 'image/%' OR $CONTENT_TYPE LIKE 'video/%') AND
|
||||
$CONTENT_TYPE != 'image/svg+xml' AND
|
||||
$ARCHIVE_THUMBNAIL_TRANSFER_STATE != ${ArchiveTransferState.PERMANENT_FAILURE.value} AND
|
||||
(
|
||||
($DATA_FILE NOT NULL AND $TRANSFER_STATE = $TRANSFER_PROGRESS_DONE) OR
|
||||
$ARCHIVE_THUMBNAIL_TRANSFER_STATE = ${ArchiveTransferState.FINISHED.value} OR
|
||||
$THUMBNAIL_RESTORE_STATE = ${ThumbnailRestoreState.FINISHED.value}
|
||||
)
|
||||
"""
|
||||
}
|
||||
|
||||
/**
|
||||
* Stops an attachment from counting as a referent when the over-size-limit cutoff left its message out of the backup, matching the exporter's
|
||||
* `$DATE_RECEIVED >= cutoff`. Attachments with no message row at all, like wallpapers, are deliberately still referents.
|
||||
*
|
||||
* @param messageInclusionCutoffTime Zero when every message made it into the backup, which is the overwhelmingly common case.
|
||||
*/
|
||||
private fun buildMessageIncludedInBackupClause(messageInclusionCutoffTime: Long): String {
|
||||
if (messageInclusionCutoffTime <= 0) {
|
||||
return ""
|
||||
}
|
||||
|
||||
return """
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM ${MessageTable.TABLE_NAME}
|
||||
WHERE
|
||||
${MessageTable.TABLE_NAME}.${MessageTable.ID} = $TABLE_NAME.$MESSAGE_ID AND
|
||||
${MessageTable.TABLE_NAME}.${MessageTable.DATE_RECEIVED} < $messageInclusionCutoffTime
|
||||
)
|
||||
"""
|
||||
}
|
||||
|
||||
/**
|
||||
* The thumbnail counterpart to [getMediaNamesWithNoAttachment]: of the given media names, the ones no attachment still wants a thumbnail for.
|
||||
*/
|
||||
fun getMediaNamesWithNoEligibleThumbnail(mediaNames: Set<MediaNameParts>, messageInclusionCutoffTime: Long = 0): Set<MediaNameParts> {
|
||||
if (mediaNames.isEmpty()) {
|
||||
return emptySet()
|
||||
}
|
||||
|
||||
val gone: MutableSet<MediaNameParts> = mediaNames.toMutableSet()
|
||||
|
||||
for (batch in mediaNames.chunked(250)) {
|
||||
val whereClause = batch.joinToString(" OR ") { "($DATA_HASH_END = ? AND $REMOTE_KEY = ?)" }
|
||||
val whereArgs = batch.flatMap { listOf(it.plaintextHash, it.remoteKey) }.toTypedArray()
|
||||
|
||||
readableDatabase
|
||||
.select(DATA_HASH_END, REMOTE_KEY)
|
||||
.from(TABLE_NAME)
|
||||
.where("($whereClause) AND ${buildThumbnailEligibilityClause()}${buildMessageIncludedInBackupClause(messageInclusionCutoffTime)}", *whereArgs)
|
||||
.groupBy("$DATA_HASH_END, $REMOTE_KEY")
|
||||
.run()
|
||||
.forEach { cursor ->
|
||||
gone -= MediaNameParts(plaintextHash = cursor.requireNonNullString(DATA_HASH_END), remoteKey = cursor.requireNonNullString(REMOTE_KEY))
|
||||
}
|
||||
}
|
||||
|
||||
return gone
|
||||
}
|
||||
|
||||
fun getMediaNamesWithNoAttachment(mediaNames: Set<MediaNameParts>, messageInclusionCutoffTime: Long = 0): Set<MediaNameParts> {
|
||||
if (mediaNames.isEmpty()) {
|
||||
return emptySet()
|
||||
}
|
||||
|
||||
val gone: MutableSet<MediaNameParts> = mediaNames.toMutableSet()
|
||||
|
||||
for (batch in mediaNames.chunked(250)) {
|
||||
val whereClause = batch.joinToString(" OR ") { "($DATA_HASH_END = ? AND $REMOTE_KEY = ?)" }
|
||||
val whereArgs = batch.flatMap { listOf(it.plaintextHash, it.remoteKey) }.toTypedArray()
|
||||
|
||||
readableDatabase
|
||||
.select(DATA_HASH_END, REMOTE_KEY)
|
||||
.from(TABLE_NAME)
|
||||
.where("($whereClause)${buildMessageIncludedInBackupClause(messageInclusionCutoffTime)}", *whereArgs)
|
||||
.groupBy("$DATA_HASH_END, $REMOTE_KEY")
|
||||
.run()
|
||||
.forEach { cursor ->
|
||||
gone -= MediaNameParts(plaintextHash = cursor.requireNonNullString(DATA_HASH_END), remoteKey = cursor.requireNonNullString(REMOTE_KEY))
|
||||
}
|
||||
}
|
||||
|
||||
return gone
|
||||
}
|
||||
|
||||
/**
|
||||
* Given a set of media objects, this will return all of the items in the set that could not be found locally.
|
||||
*/
|
||||
fun getMediaObjectsThatCantBeFound(objects: Set<ArchivedMediaObject>): Set<ArchivedMediaObject> {
|
||||
fun getMediaObjectsThatCantBeFound(objects: Set<ArchivedMediaObject>, messageInclusionCutoffTime: Long = 0): Set<ArchivedMediaObject> {
|
||||
if (objects.isEmpty()) {
|
||||
return emptySet()
|
||||
}
|
||||
|
||||
val objectsByMediaId: MutableMap<String, ArchivedMediaObject> = objects.associateBy { it.mediaId }.toMutableMap()
|
||||
val mediaRootBackupKey = SignalStore.backup.mediaRootBackupKey
|
||||
|
||||
readableDatabase
|
||||
.select(*PROJECTION)
|
||||
.from(TABLE_NAME)
|
||||
.where("$REMOTE_KEY NOT NULL AND $DATA_HASH_END NOT NULL")
|
||||
.where("$REMOTE_KEY NOT NULL AND $DATA_HASH_END NOT NULL${buildMessageIncludedInBackupClause(messageInclusionCutoffTime)}")
|
||||
.groupBy("$DATA_HASH_END, $REMOTE_KEY")
|
||||
.run()
|
||||
.forEach { cursor ->
|
||||
val remoteKey = Base64.decode(cursor.requireNonNullString(REMOTE_KEY))
|
||||
val plaintextHash = Base64.decode(cursor.requireNonNullString(DATA_HASH_END))
|
||||
val mediaId = MediaName.fromPlaintextHashAndRemoteKey(plaintextHash, remoteKey).toMediaId(SignalStore.backup.mediaRootBackupKey).encode()
|
||||
val mediaIdThumbnail = MediaName.fromPlaintextHashAndRemoteKeyForThumbnail(plaintextHash, remoteKey).toMediaId(SignalStore.backup.mediaRootBackupKey).encode()
|
||||
val mediaId = MediaName.fromPlaintextHashAndRemoteKey(plaintextHash, remoteKey).toMediaId(mediaRootBackupKey).encode()
|
||||
val mediaIdThumbnail = MediaName.fromPlaintextHashAndRemoteKeyForThumbnail(plaintextHash, remoteKey).toMediaId(mediaRootBackupKey).encode()
|
||||
|
||||
objectsByMediaId.remove(mediaId)
|
||||
objectsByMediaId.remove(mediaIdThumbnail)
|
||||
@@ -4039,8 +4356,24 @@ class AttachmentTable(
|
||||
val size: Long,
|
||||
val plaintextHash: ByteArray?,
|
||||
val remoteKey: ByteArray?,
|
||||
val stickerPackId: String?
|
||||
val stickerPackId: String?,
|
||||
val contentType: String?,
|
||||
val quote: Boolean,
|
||||
val stickerId: Int
|
||||
) {
|
||||
/**
|
||||
* Whether a thumbnail for this attachment could ever have been uploaded, matches [buildThumbnailsThatNeedArchiveWorkQuery] and
|
||||
* `BackupMessagesJob.toThumbnailMediaEntries`.
|
||||
*/
|
||||
val couldHaveArchivedThumbnail: Boolean
|
||||
get() {
|
||||
return mmsId != WALLPAPER_MESSAGE_ID &&
|
||||
!quote &&
|
||||
stickerId == -1 &&
|
||||
contentType != "image/svg+xml" &&
|
||||
MediaUtil.isImageOrVideoType(contentType)
|
||||
}
|
||||
|
||||
override fun equals(other: Any?): Boolean {
|
||||
return this === other || attachmentId == (other as? RestorableAttachment)?.attachmentId
|
||||
}
|
||||
@@ -4165,4 +4498,20 @@ class AttachmentTable(
|
||||
return "attachmentId=${attachment.attachmentId}, mediaId=$mediaId, messageId=${attachment.mmsId}, isThumbnail=$isThumbnail, contentType=${attachment.contentType}, quote=${attachment.quote}, wallpaper=$isWallpaper, transferState=${attachment.transferState}, archiveTransferState=${attachment.archiveTransferState}, hasData=${attachment.hasData}, dateSent=${messageRecord?.dateSent}, messageType=${messageRecord?.type}, messageFrom=${messageRecord?.fromRecipient?.id}, messageTo=${messageRecord?.toRecipient?.id}, expiresIn=${messageRecord?.expiresIn}, expireStarted=${messageRecord?.expireStarted}"
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The base64 [DATA_HASH_END] and [REMOTE_KEY] that a [MediaName] is derived from.
|
||||
*/
|
||||
data class MediaNameParts(val plaintextHash: String, val remoteKey: String)
|
||||
|
||||
enum class ArchiveTransferStateResetResult {
|
||||
/** We cleared the state, so the media will be re-uploaded. */
|
||||
RESET,
|
||||
|
||||
/** We left the state alone because there are no local bytes to re-upload, so the media is unrecoverable rather than merely out of date. */
|
||||
SKIPPED_NO_LOCAL_DATA,
|
||||
|
||||
/** There was nothing to clear. The attachment is gone, or an upload is already underway. */
|
||||
NOT_NEEDED
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,6 +20,7 @@ import org.signal.core.util.readToSingleLong
|
||||
import org.signal.core.util.requireBoolean
|
||||
import org.signal.core.util.requireInt
|
||||
import org.signal.core.util.requireIntOrNull
|
||||
import org.signal.core.util.requireLong
|
||||
import org.signal.core.util.requireNonNullBlob
|
||||
import org.signal.core.util.requireNonNullString
|
||||
import org.signal.core.util.select
|
||||
@@ -195,22 +196,44 @@ class BackupMediaSnapshotTable(context: Context, database: SignalDatabase) : Dat
|
||||
.readToSingleLong(0)
|
||||
}
|
||||
|
||||
fun getPageOfOldMediaObjects(pageSize: Int): Set<ArchivedMediaObject> {
|
||||
return readableDatabase.select(MEDIA_ID, CDN)
|
||||
/**
|
||||
* Media objects that have dropped out of the most recent snapshot. Carries the hash/key so callers can check whether anything in the [AttachmentTable] still
|
||||
* refers to them before deleting them off of the CDN.
|
||||
*/
|
||||
fun getPageOfOldMediaEntries(pageSize: Int, afterId: Long = 0): List<ExistingMediaEntry> {
|
||||
return readableDatabase.select(ID, MEDIA_ID, CDN, PLAINTEXT_HASH, REMOTE_KEY, IS_THUMBNAIL)
|
||||
.from(TABLE_NAME)
|
||||
.where("$SNAPSHOT_VERSION < $MAX_VERSION AND $IS_PENDING = 0")
|
||||
.where("$SNAPSHOT_VERSION < $MAX_VERSION AND $IS_PENDING = 0 AND $ID > $afterId")
|
||||
.orderBy("$ID ASC")
|
||||
.limit(pageSize)
|
||||
.run()
|
||||
.readToSet {
|
||||
ArchivedMediaObject(mediaId = it.requireNonNullString(MEDIA_ID), cdn = it.requireInt(CDN))
|
||||
}
|
||||
.readToList { ExistingMediaEntry.fromCursor(it) }
|
||||
}
|
||||
|
||||
/**
|
||||
* Of the given mediaIds, the ones present in the most recent snapshot that a reconciliation crawl has since seen on the archive CDN. Being in the snapshot
|
||||
* only reflects our own bookkeeping, so [LAST_SEEN_ON_REMOTE_SNAPSHOT_VERSION] > 0 is the only part of this that the server told us.
|
||||
*/
|
||||
fun getMediaIdsConfirmedOnCdn(mediaIds: Collection<String>, minimumLastSeenSnapshotVersion: Long): Set<String> {
|
||||
if (mediaIds.isEmpty()) {
|
||||
return emptySet()
|
||||
}
|
||||
|
||||
val query = SqlUtil.buildFastCollectionQuery(MEDIA_ID, mediaIds)
|
||||
|
||||
return readableDatabase
|
||||
.select(MEDIA_ID)
|
||||
.from(TABLE_NAME)
|
||||
.where("${query.where} AND $SNAPSHOT_VERSION = $MAX_VERSION AND $LAST_SEEN_ON_REMOTE_SNAPSHOT_VERSION > 0 AND $LAST_SEEN_ON_REMOTE_SNAPSHOT_VERSION >= $minimumLastSeenSnapshotVersion", query.whereArgs)
|
||||
.run()
|
||||
.readToSet { it.requireNonNullString(MEDIA_ID) }
|
||||
}
|
||||
|
||||
/**
|
||||
* This will remove any old snapshot entries with matching mediaId's. No pending entries or entries in the latest snapshot will be affected.
|
||||
*/
|
||||
fun deleteOldMediaObjects(mediaObjects: Collection<ArchivedMediaObject>) {
|
||||
val query = SqlUtil.buildFastCollectionQuery(MEDIA_ID, mediaObjects.map { it.mediaId })
|
||||
fun deleteOldMediaObjects(mediaIds: Collection<String>) {
|
||||
val query = SqlUtil.buildFastCollectionQuery(MEDIA_ID, mediaIds)
|
||||
|
||||
writableDatabase.delete(TABLE_NAME)
|
||||
.where("$SNAPSHOT_VERSION < $MAX_VERSION AND $IS_PENDING = 0 AND " + query.where, query.whereArgs)
|
||||
@@ -251,32 +274,6 @@ class BackupMediaSnapshotTable(context: Context, database: SignalDatabase) : Dat
|
||||
return objectsByMediaId.values.toSet()
|
||||
}
|
||||
|
||||
fun getMediaEntriesForObjects(objects: List<ArchivedMediaObject>): Set<MediaEntry> {
|
||||
if (objects.isEmpty()) {
|
||||
return emptySet()
|
||||
}
|
||||
|
||||
val queries: List<SqlUtil.Query> = SqlUtil.buildCollectionQuery(
|
||||
column = MEDIA_ID,
|
||||
values = objects.map { it.mediaId },
|
||||
collectionOperator = SqlUtil.CollectionOperator.IN,
|
||||
prefix = "$SNAPSHOT_VERSION = $MAX_VERSION AND "
|
||||
)
|
||||
|
||||
val entries: MutableSet<MediaEntry> = mutableSetOf()
|
||||
|
||||
for (query in queries) {
|
||||
entries += readableDatabase
|
||||
.select(MEDIA_ID, CDN, PLAINTEXT_HASH, REMOTE_KEY, IS_THUMBNAIL)
|
||||
.from("$TABLE_NAME JOIN ${AttachmentTable.TABLE_NAME}")
|
||||
.where(query.where, query.whereArgs)
|
||||
.run()
|
||||
.readToList { MediaEntry.fromCursor(it) }
|
||||
}
|
||||
|
||||
return entries.toSet()
|
||||
}
|
||||
|
||||
/**
|
||||
* Given a list of media objects, find the ones that are present in the most recent snapshot, but have a different CDN than the one passed in.
|
||||
* This will ignore thumbnails, as the results are intended to be used to update CDNs, which we do not track for thumbnails.
|
||||
@@ -308,8 +305,25 @@ class BackupMediaSnapshotTable(context: Context, database: SignalDatabase) : Dat
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Limited to rows *no* crawl has ever confirmed, so we never contradict our own earlier evidence that an object existed. Only valid after a *completed* crawl,
|
||||
* since on a partial one an unstamped row may simply not have been paged yet, and deleting it would orphan the object it tracks.
|
||||
*
|
||||
* @param crawlSnapshotVersion The version the crawl pinned itself to. Rows above it were committed after the crawl fixed its view, so an upload that landed
|
||||
* behind the cursor still looks unconfirmed. Dropping those would leave an object on the CDN with nothing left tracking it.
|
||||
*/
|
||||
fun deleteOldMediaObjectsNeverSeenOnCdn(crawlSnapshotVersion: Long): Int {
|
||||
return writableDatabase
|
||||
.delete(TABLE_NAME)
|
||||
.where("$SNAPSHOT_VERSION < $MAX_VERSION AND $SNAPSHOT_VERSION <= $crawlSnapshotVersion AND $IS_PENDING = 0 AND $LAST_SEEN_ON_REMOTE_SNAPSHOT_VERSION = 0")
|
||||
.run()
|
||||
}
|
||||
|
||||
/**
|
||||
* Indicate the time that the set of media objects were seen on the archive CDN. Can be used to reconcile our local state with the server state.
|
||||
*
|
||||
* The write only moves forward: a resumed crawl can carry a version older than what later backups have committed, and 0 doubles as the "never confirmed"
|
||||
* default, so a stale write would make confirmed rows look like they had never been seen.
|
||||
*/
|
||||
fun markSeenOnRemote(mediaIdBatch: Collection<String>, snapshotVersion: Long) {
|
||||
if (mediaIdBatch.isEmpty()) {
|
||||
@@ -320,7 +334,7 @@ class BackupMediaSnapshotTable(context: Context, database: SignalDatabase) : Dat
|
||||
writableDatabase
|
||||
.update(TABLE_NAME)
|
||||
.values(LAST_SEEN_ON_REMOTE_SNAPSHOT_VERSION to snapshotVersion)
|
||||
.where(query.where, query.whereArgs)
|
||||
.where("${query.where} AND $LAST_SEEN_ON_REMOTE_SNAPSHOT_VERSION < $snapshotVersion", query.whereArgs)
|
||||
.run()
|
||||
}
|
||||
|
||||
@@ -374,6 +388,31 @@ class BackupMediaSnapshotTable(context: Context, database: SignalDatabase) : Dat
|
||||
val cdn: Int
|
||||
)
|
||||
|
||||
class ExistingMediaEntry(
|
||||
val id: Long,
|
||||
val mediaEntry: MediaEntry
|
||||
) {
|
||||
val mediaId: String
|
||||
get() = mediaEntry.mediaId
|
||||
val cdn: Int?
|
||||
get() = mediaEntry.cdn
|
||||
val plaintextHash: ByteArray
|
||||
get() = mediaEntry.plaintextHash
|
||||
val remoteKey: ByteArray
|
||||
get() = mediaEntry.remoteKey
|
||||
val isThumbnail: Boolean
|
||||
get() = mediaEntry.isThumbnail
|
||||
|
||||
companion object {
|
||||
fun fromCursor(cursor: Cursor): ExistingMediaEntry {
|
||||
return ExistingMediaEntry(
|
||||
id = cursor.requireLong(ID),
|
||||
mediaEntry = MediaEntry.fromCursor(cursor)
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
class MediaEntry(
|
||||
val mediaId: String,
|
||||
val cdn: Int?,
|
||||
|
||||
+85
-19
@@ -27,6 +27,7 @@ import org.signal.network.service.ArchiveError
|
||||
import org.thoughtcrime.securesms.R
|
||||
import org.thoughtcrime.securesms.backup.v2.ArchivedMediaObject
|
||||
import org.thoughtcrime.securesms.database.AttachmentTable
|
||||
import org.thoughtcrime.securesms.database.AttachmentTable.ArchiveTransferStateResetResult
|
||||
import org.thoughtcrime.securesms.database.BackupMediaSnapshotTable
|
||||
import org.thoughtcrime.securesms.database.SignalDatabase
|
||||
import org.thoughtcrime.securesms.dependencies.AppDependencies
|
||||
@@ -40,8 +41,10 @@ import org.thoughtcrime.securesms.notifications.NotificationChannels
|
||||
import org.thoughtcrime.securesms.notifications.NotificationIds
|
||||
import org.thoughtcrime.securesms.util.RemoteConfig
|
||||
import org.thoughtcrime.securesms.wallpaper.WallpaperStorage
|
||||
import kotlin.time.Duration
|
||||
import kotlin.time.Duration.Companion.days
|
||||
import kotlin.time.Duration.Companion.hours
|
||||
import kotlin.time.Duration.Companion.milliseconds
|
||||
|
||||
/**
|
||||
* We do our best to keep our local attachments in sync with the archive CDN, but we still want to have a backstop that periodically
|
||||
@@ -84,6 +87,21 @@ class ArchiveAttachmentReconciliationJob private constructor(
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Rate-limited because the caller runs after every backup.
|
||||
*/
|
||||
fun enqueueToVerifyMediaBeforeOffloading(minTimeBetweenAttempts: Duration) {
|
||||
val sinceLastAttempt = (System.currentTimeMillis() - SignalStore.backup.lastForcedReconciliationAttemptTime).milliseconds
|
||||
|
||||
if (sinceLastAttempt > Duration.ZERO && sinceLastAttempt < minTimeBetweenAttempts) {
|
||||
Log.i(TAG, "Already forced a reconciliation $sinceLastAttempt ago. Waiting before forcing another.")
|
||||
return
|
||||
}
|
||||
|
||||
Log.i(TAG, "Forcing a reconciliation so we can verify our media against the archive CDN.", true)
|
||||
AppDependencies.jobManager.add(ArchiveAttachmentReconciliationJob(forced = true))
|
||||
}
|
||||
|
||||
/**
|
||||
* Reconcile-first entry point for after a local restore. Runs a [BackupMessagesJob] (to capture a snapshot) chained into a forced reconciliation.
|
||||
* Sets [BackupValues.localRestoreReconcilePending] so the backup holds off on the bulk attachment backfill, letting reconciliation run first. Reconciliation
|
||||
@@ -152,6 +170,12 @@ class ArchiveAttachmentReconciliationJob private constructor(
|
||||
// we use to determine which attachments need to be re-uploaded will possibly result in us unnecessarily re-uploading attachments.
|
||||
snapshotVersion = snapshotVersion ?: SignalDatabase.backupMediaSnapshots.getCurrentSnapshotVersion()
|
||||
|
||||
// Only spend the forced-attempt budget on a crawl that could actually open the offload gate. Without a snapshot it can't, and the rate limit would then keep
|
||||
// us from retrying once a backup has built one.
|
||||
if (forced && snapshotVersion!! > 0) {
|
||||
SignalStore.backup.lastForcedReconciliationAttemptTime = System.currentTimeMillis()
|
||||
}
|
||||
|
||||
syncDataFromCdn(snapshotVersion!!)?.let { return it }
|
||||
|
||||
clearPendingLocalRestoreReconcile()
|
||||
@@ -239,6 +263,7 @@ class ArchiveAttachmentReconciliationJob private constructor(
|
||||
|
||||
var newBackupJobRequired = false
|
||||
var bookkeepingErrorCount = 0
|
||||
var unrecoverableCount = 0
|
||||
|
||||
var fullSizeMismatchFound = false
|
||||
var thumbnailMismatchFound = false
|
||||
@@ -251,26 +276,37 @@ class ArchiveAttachmentReconciliationJob private constructor(
|
||||
}
|
||||
|
||||
val mediaIdLog = if (internalUser) "[${MediaId(entry.mediaId)}]" else ""
|
||||
val logPrefix = if (entry.isThumbnail) "[Thumbnail]$mediaIdLog" else "[Fullsize]$mediaIdLog"
|
||||
|
||||
if (entry.isThumbnail) {
|
||||
thumbnailMismatchFound = true
|
||||
val wasReset = SignalDatabase.attachments.resetArchiveThumbnailTransferStateByPlaintextHashAndRemoteKeyIfNecessary(entry.plaintextHash, entry.remoteKey)
|
||||
if (wasReset) {
|
||||
Log.w(TAG, "[Thumbnail]$mediaIdLog Reset transfer state by hash/key.", true)
|
||||
newBackupJobRequired = true
|
||||
bookkeepingErrorCount++
|
||||
} else {
|
||||
Log.i(TAG, "[Thumbnail]$mediaIdLog Did not need to reset the transfer state by hash/key because the thumbnail either no longer exists or the upload is already in-progress.", true)
|
||||
}
|
||||
val resetResult = if (entry.isThumbnail) {
|
||||
SignalDatabase.attachments.resetArchiveThumbnailTransferStateByPlaintextHashAndRemoteKeyIfNecessary(entry.plaintextHash, entry.remoteKey)
|
||||
} else {
|
||||
fullSizeMismatchFound = true
|
||||
val wasReset = SignalDatabase.attachments.resetArchiveTransferStateByPlaintextHashAndRemoteKeyIfNecessary(entry.plaintextHash, entry.remoteKey)
|
||||
if (wasReset) {
|
||||
Log.w(TAG, "[Fullsize]$mediaIdLog Reset transfer state by hash/key.", true)
|
||||
SignalDatabase.attachments.resetArchiveTransferStateByPlaintextHashAndRemoteKeyIfNecessary(entry.plaintextHash, entry.remoteKey)
|
||||
}
|
||||
|
||||
when (resetResult) {
|
||||
ArchiveTransferStateResetResult.RESET -> {
|
||||
Log.w(TAG, "$logPrefix Reset transfer state by hash/key.", true)
|
||||
newBackupJobRequired = true
|
||||
bookkeepingErrorCount++
|
||||
} else {
|
||||
Log.i(TAG, "[Fullsize]$mediaIdLog Did not need to reset the transfer state by hash/key because the attachment either no longer exists or the upload is already in-progress.", true)
|
||||
}
|
||||
|
||||
ArchiveTransferStateResetResult.SKIPPED_NO_LOCAL_DATA -> {
|
||||
if (internalUser) {
|
||||
Log.w(TAG, "$logPrefix Missing from the CDN with no local data to re-upload. Leaving the archive state alone so we keep the locator.", true)
|
||||
}
|
||||
unrecoverableCount++
|
||||
}
|
||||
|
||||
ArchiveTransferStateResetResult.NOT_NEEDED -> {
|
||||
Log.i(TAG, "$logPrefix Did not need to reset the transfer state by hash/key because it either no longer exists or the upload is already in-progress.", true)
|
||||
|
||||
// Deliberately not set for SKIPPED_NO_LOCAL_DATA, since the precautionary backfills these drive could never upload media that has no local bytes.
|
||||
if (entry.isThumbnail) {
|
||||
thumbnailMismatchFound = true
|
||||
} else {
|
||||
fullSizeMismatchFound = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -282,6 +318,10 @@ class ArchiveAttachmentReconciliationJob private constructor(
|
||||
Log.i(TAG, "None of the $mayNeedReUploadCount CDN mismatches were bookkeeping errors.", true)
|
||||
}
|
||||
|
||||
if (unrecoverableCount > 0) {
|
||||
Log.w(TAG, "Found that $unrecoverableCount/$mayNeedReUploadCount of the CDN mismatches have no local data to re-upload. That media is not recoverable from this device.", true)
|
||||
}
|
||||
|
||||
Log.d(TAG, "AFTER:\n" + SignalDatabase.attachments.debugGetAttachmentStats().shortPrettyString(), true)
|
||||
stopwatch.split("stats-after")
|
||||
|
||||
@@ -320,8 +360,24 @@ class ArchiveAttachmentReconciliationJob private constructor(
|
||||
Log.d(TAG, "No attachments need to be repaired.", true)
|
||||
}
|
||||
|
||||
if (snapshotVersion > 0) {
|
||||
val prunedCount = SignalDatabase.backupMediaSnapshots.deleteOldMediaObjectsNeverSeenOnCdn(snapshotVersion)
|
||||
if (prunedCount > 0) {
|
||||
Log.i(TAG, "Pruned $prunedCount snapshot entries that left the latest snapshot and have never been seen on the CDN.", true)
|
||||
}
|
||||
}
|
||||
stopwatch.split("prune-absent")
|
||||
|
||||
val completionTime = System.currentTimeMillis()
|
||||
|
||||
SignalStore.backup.remoteStorageGarbageCollectionPending = false
|
||||
SignalStore.backup.lastAttachmentReconciliationTime = System.currentTimeMillis()
|
||||
SignalStore.backup.lastAttachmentReconciliationTime = completionTime
|
||||
|
||||
// A crawl with no snapshot to compare against verified nothing, so it must not satisfy the gate that lets us delete local copies of media.
|
||||
if (snapshotVersion > 0) {
|
||||
SignalStore.backup.lastCompletedReconciliationSnapshotVersion = snapshotVersion
|
||||
SignalStore.backup.lastCompletedReconciliationTime = completionTime
|
||||
}
|
||||
|
||||
stopwatch.stop(TAG)
|
||||
|
||||
@@ -369,6 +425,13 @@ class ArchiveAttachmentReconciliationJob private constructor(
|
||||
}
|
||||
}
|
||||
|
||||
// TODO [cody] Fix perf problems of calling setArchiveThumbnailFinishedForMatchingMediaObjects
|
||||
// Takes the whole listing, since a restored device has no thumbnail snapshot rows yet
|
||||
// val thumbnailsMarkedFinished = SignalDatabase.attachments.setArchiveThumbnailFinishedForMatchingMediaObjects(mediaObjects.toSet())
|
||||
// if (thumbnailsMarkedFinished > 0) {
|
||||
// Log.i(TAG, "Marked $thumbnailsMarkedFinished thumbnail group(s) as finished after finding them on the CDN.", true)
|
||||
// }
|
||||
|
||||
return mediaOnRemoteButNotLocal
|
||||
}
|
||||
|
||||
@@ -422,8 +485,8 @@ class ArchiveAttachmentReconciliationJob private constructor(
|
||||
}
|
||||
|
||||
val stopwatch = Stopwatch("remote-delete")
|
||||
val validatedDeletes: MutableSet<ArchivedMediaObject> = SignalDatabase.attachments.getMediaObjectsThatCantBeFound(deletes).toMutableSet()
|
||||
Log.d(TAG, "Found that ${validatedDeletes.size}/${deletes.size} requested remote deletes have no data at all locally, and are therefore safe to delete.", true)
|
||||
val validatedDeletes: MutableSet<ArchivedMediaObject> = SignalDatabase.attachments.getMediaObjectsThatCantBeFound(deletes, SignalStore.backup.lastUsedMessageCutoffTime).toMutableSet()
|
||||
Log.d(TAG, "Found that ${validatedDeletes.size}/${deletes.size} requested remote deletes are no longer referenced by any attachment, and are therefore safe to delete.", true)
|
||||
stopwatch.split("validate")
|
||||
|
||||
// Fix archive state for attachments that are found locally but weren't in the latest snapshot.
|
||||
@@ -474,6 +537,9 @@ class ArchiveAttachmentReconciliationJob private constructor(
|
||||
}
|
||||
stopwatch.split("network")
|
||||
|
||||
// Any snapshot row left behind here would keep claiming an object we just removed, and reviving that row would present it as still CDN-confirmed.
|
||||
SignalDatabase.backupMediaSnapshots.deleteOldMediaObjects(validatedDeletes.map { it.mediaId })
|
||||
|
||||
stopwatch.stop(TAG)
|
||||
|
||||
return null
|
||||
|
||||
+59
-7
@@ -5,6 +5,7 @@
|
||||
|
||||
package org.thoughtcrime.securesms.jobs
|
||||
|
||||
import androidx.annotation.VisibleForTesting
|
||||
import arrow.core.Either
|
||||
import org.signal.core.models.backup.MediaId
|
||||
import org.signal.core.util.Base64
|
||||
@@ -12,6 +13,7 @@ import org.signal.core.util.logging.Log
|
||||
import org.signal.network.service.ArchiveError
|
||||
import org.thoughtcrime.securesms.attachments.Cdn
|
||||
import org.thoughtcrime.securesms.backup.v2.ArchivedMediaObject
|
||||
import org.thoughtcrime.securesms.database.AttachmentTable
|
||||
import org.thoughtcrime.securesms.database.BackupMediaSnapshotTable
|
||||
import org.thoughtcrime.securesms.database.SignalDatabase
|
||||
import org.thoughtcrime.securesms.dependencies.AppDependencies
|
||||
@@ -19,7 +21,6 @@ import org.thoughtcrime.securesms.jobmanager.CoroutineJob
|
||||
import org.thoughtcrime.securesms.jobmanager.Job
|
||||
import org.thoughtcrime.securesms.keyvalue.SignalStore
|
||||
import org.thoughtcrime.securesms.util.RemoteConfig
|
||||
import java.lang.RuntimeException
|
||||
import kotlin.time.Duration.Companion.days
|
||||
import kotlin.time.Duration.Companion.hours
|
||||
|
||||
@@ -36,7 +37,8 @@ class ArchiveCommitAttachmentDeletesJob private constructor(parameters: Paramete
|
||||
const val KEY = "ArchiveCommitAttachmentDeletesJob"
|
||||
const val ARCHIVE_ATTACHMENT_QUEUE = "ArchiveAttachmentQueue"
|
||||
|
||||
private const val REMOTE_DELETE_BATCH_SIZE = 1_000
|
||||
@VisibleForTesting
|
||||
internal const val REMOTE_DELETE_BATCH_SIZE = 1_000
|
||||
|
||||
/**
|
||||
* Deletes the provided attachments from the CDN.
|
||||
@@ -116,18 +118,61 @@ class ArchiveCommitAttachmentDeletesJob private constructor(parameters: Paramete
|
||||
return Result.success()
|
||||
}
|
||||
|
||||
var mediaObjects = SignalDatabase.backupMediaSnapshots.getPageOfOldMediaObjects(REMOTE_DELETE_BATCH_SIZE)
|
||||
// Read once so every page judges against the same threshold, even if a backup finishes while we're paging.
|
||||
val messageInclusionCutoffTime = SignalStore.backup.lastUsedMessageCutoffTime
|
||||
|
||||
while (mediaObjects.isNotEmpty()) {
|
||||
var retainedCount = 0
|
||||
var unknownCdnCount = 0
|
||||
var lastId = 0L
|
||||
var page = SignalDatabase.backupMediaSnapshots.getPageOfOldMediaEntries(pageSize = REMOTE_DELETE_BATCH_SIZE, afterId = lastId)
|
||||
|
||||
while (page.isNotEmpty()) {
|
||||
if (isCanceled) {
|
||||
Log.w(TAG, "Job cancelled while processing media objects for deletion.")
|
||||
return Result.failure()
|
||||
}
|
||||
|
||||
deleteMediaObjectsFromCdn(TAG, mediaObjects, this::defaultBackoff, this::isCanceled)?.let { result -> return result }
|
||||
SignalDatabase.backupMediaSnapshots.deleteOldMediaObjects(mediaObjects)
|
||||
lastId = page.last().id
|
||||
|
||||
mediaObjects = SignalDatabase.backupMediaSnapshots.getPageOfOldMediaObjects(REMOTE_DELETE_BATCH_SIZE)
|
||||
// Full-size and thumbnail rows share a mediaName, so they have to be judged separately. A surviving attachment keeps its full-size object alive but says
|
||||
// nothing about whether that attachment still wants a thumbnail archived.
|
||||
val (thumbnailPage, fullSizePage) = page.partition { it.isThumbnail }
|
||||
|
||||
val fullSizeByMediaName = fullSizePage.groupBy { it.mediaNameParts() }
|
||||
val unreferencedFullSizeNames = SignalDatabase.attachments.getMediaNamesWithNoAttachment(fullSizeByMediaName.keys, messageInclusionCutoffTime)
|
||||
val unreferencedFullSize = fullSizeByMediaName.filterKeys { it in unreferencedFullSizeNames }.values.flatten()
|
||||
|
||||
val thumbnailsByMediaName = thumbnailPage.groupBy { it.mediaNameParts() }
|
||||
val unreferencedThumbnailNames = SignalDatabase.attachments.getMediaNamesWithNoEligibleThumbnail(thumbnailsByMediaName.keys, messageInclusionCutoffTime)
|
||||
val unreferencedThumbnails = thumbnailsByMediaName.filterKeys { it in unreferencedThumbnailNames }.values.flatten()
|
||||
|
||||
val unreferencedEntries = unreferencedFullSize + unreferencedThumbnails
|
||||
|
||||
val safeToDelete = unreferencedEntries.mapNotNull { entry -> entry.cdn?.let { ArchivedMediaObject(mediaId = entry.mediaId, cdn = it) } }.toSet()
|
||||
|
||||
retainedCount += page.size - unreferencedEntries.size
|
||||
unknownCdnCount += unreferencedEntries.size - safeToDelete.size
|
||||
|
||||
if (safeToDelete.isNotEmpty()) {
|
||||
deleteMediaObjectsFromCdn(
|
||||
tag = TAG,
|
||||
attachmentsToDelete = safeToDelete,
|
||||
backoffGenerator = this::defaultBackoff,
|
||||
cancellationSignal = this::isCanceled
|
||||
)?.let { result -> return result }
|
||||
|
||||
SignalDatabase.backupMediaSnapshots.deleteOldMediaObjects(safeToDelete.map { it.mediaId })
|
||||
}
|
||||
|
||||
page = SignalDatabase.backupMediaSnapshots.getPageOfOldMediaEntries(pageSize = REMOTE_DELETE_BATCH_SIZE, afterId = lastId)
|
||||
}
|
||||
|
||||
if (retainedCount > 0) {
|
||||
Log.w(TAG, "Retained $retainedCount media objects that dropped out of the latest snapshot but are still referenced by an attachment. They stay tracked and will be reconsidered after the next backup.", true)
|
||||
}
|
||||
|
||||
if (unknownCdnCount > 0) {
|
||||
Log.w(TAG, "Retained $unknownCdnCount unreferenced media objects that have no recorded CDN, will get resolved during reconciliation rather than here.", true)
|
||||
}
|
||||
|
||||
return Result.success()
|
||||
@@ -135,6 +180,13 @@ class ArchiveCommitAttachmentDeletesJob private constructor(parameters: Paramete
|
||||
|
||||
override fun onFailure() = Unit
|
||||
|
||||
private fun BackupMediaSnapshotTable.ExistingMediaEntry.mediaNameParts(): AttachmentTable.MediaNameParts {
|
||||
return AttachmentTable.MediaNameParts(
|
||||
plaintextHash = Base64.encodeWithPadding(plaintextHash),
|
||||
remoteKey = Base64.encodeWithPadding(remoteKey)
|
||||
)
|
||||
}
|
||||
|
||||
class Factory : Job.Factory<ArchiveCommitAttachmentDeletesJob> {
|
||||
override fun create(parameters: Parameters, serializedData: ByteArray?): ArchiveCommitAttachmentDeletesJob {
|
||||
return ArchiveCommitAttachmentDeletesJob(parameters)
|
||||
|
||||
@@ -157,13 +157,36 @@ class ArchiveThumbnailUploadJob private constructor(
|
||||
return Result.success()
|
||||
}
|
||||
|
||||
// TODO [backups] Determine if we actually need to upload or are reusing a thumbnail from another attachment
|
||||
val alreadyArchived = SignalDatabase.attachments.getArchiveThumbnailTransferState(attachmentId) == AttachmentTable.ArchiveTransferState.FINISHED
|
||||
|
||||
// Attachments sharing a mediaName are enqueued individually, but finalizing fills in the whole group, so by now a sibling may have done the work for us.
|
||||
if (alreadyArchived && SignalDatabase.attachments.hasThumbnailFile(attachmentId)) {
|
||||
Log.i(TAG, "[$attachmentId] Thumbnail is already archived and present locally. Nothing to do.")
|
||||
return Result.success()
|
||||
}
|
||||
|
||||
val thumbnailResult = generateThumbnailIfPossible(attachment)
|
||||
if (thumbnailResult == null) {
|
||||
Log.w(TAG, "Unable to generate a thumbnail result for $attachmentId")
|
||||
ArchiveDatabaseExecutor.runBlocking {
|
||||
SignalDatabase.attachments.setArchiveThumbnailTransferState(attachmentId, AttachmentTable.ArchiveTransferState.PERMANENT_FAILURE)
|
||||
if (alreadyArchived) {
|
||||
SignalDatabase.attachments.markThumbnailPermanentlyFailedIfUnrestorable(attachmentId)
|
||||
} else {
|
||||
SignalDatabase.attachments.setArchiveThumbnailTransferState(attachmentId, AttachmentTable.ArchiveTransferState.PERMANENT_FAILURE)
|
||||
}
|
||||
}
|
||||
return Result.success()
|
||||
}
|
||||
|
||||
if (alreadyArchived) {
|
||||
Log.i(TAG, "[$attachmentId] Thumbnail is already on the archive CDN. Saving the generated copy locally rather than uploading it again.")
|
||||
ArchiveDatabaseExecutor.runBlocking {
|
||||
SignalDatabase.attachments.finalizeAttachmentThumbnailAfterUpload(
|
||||
attachmentId = attachmentId,
|
||||
attachmentPlaintextHash = attachment.dataHash,
|
||||
attachmentRemoteKey = attachment.remoteKey,
|
||||
data = thumbnailResult.data
|
||||
)
|
||||
}
|
||||
return Result.success()
|
||||
}
|
||||
@@ -306,6 +329,11 @@ class ArchiveThumbnailUploadJob private constructor(
|
||||
}
|
||||
|
||||
override fun onFailure() {
|
||||
if (SignalDatabase.attachments.getArchiveThumbnailTransferState(attachmentId) == AttachmentTable.ArchiveTransferState.FINISHED) {
|
||||
Log.w(TAG, "[$attachmentId] Job didn't finish, but the thumbnail is already archived. Leaving the transfer state alone so we don't re-upload it.")
|
||||
return
|
||||
}
|
||||
|
||||
if (this.isCanceled) {
|
||||
Log.w(TAG, "[$attachmentId] Job was canceled, updating archive thumbnail transfer state to ${AttachmentTable.ArchiveTransferState.NONE}.")
|
||||
ArchiveDatabaseExecutor.runBlocking {
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
|
||||
package org.thoughtcrime.securesms.jobs
|
||||
|
||||
import androidx.annotation.VisibleForTesting
|
||||
import org.signal.core.models.database.AttachmentId
|
||||
import org.signal.core.util.logging.Log
|
||||
import org.signal.core.util.withinTransaction
|
||||
@@ -57,9 +58,23 @@ class BackupRestoreMediaJob private constructor(parameters: Parameters) : BaseJo
|
||||
throw NotPushRegisteredException()
|
||||
}
|
||||
|
||||
enqueueRestoreJobs(restoreTime = System.currentTimeMillis())
|
||||
|
||||
BackupMediaRestoreService.start(context, context.getString(R.string.BackupStatus__restoring_media))
|
||||
ArchiveRestoreProgress.onRestoringMedia()
|
||||
|
||||
RestoreAttachmentJob.Queues.INITIAL_RESTORE.forEach { queue ->
|
||||
AppDependencies.jobManager.add(CheckRestoreMediaLeftJob(queue))
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Walks every restorable attachment, enqueuing a full-size or thumbnail restore for each and moving it out of [AttachmentTable.TRANSFER_NEEDS_RESTORE] so the
|
||||
* next batch makes progress.
|
||||
*/
|
||||
@VisibleForTesting
|
||||
internal fun enqueueRestoreJobs(restoreTime: Long, batchSize: Int = 500) {
|
||||
val jobManager = AppDependencies.jobManager
|
||||
val batchSize = 500
|
||||
val restoreTime = System.currentTimeMillis()
|
||||
|
||||
val orphanedCount = SignalDatabase.attachments.markRestorableAttachmentsWithoutMessageAsFailed()
|
||||
if (orphanedCount > 0) {
|
||||
@@ -71,6 +86,8 @@ class BackupRestoreMediaJob private constructor(parameters: Parameters) : BaseJo
|
||||
Log.w(TAG, "$stalledCount attachments were stuck mid-restore; reset to needs-restore so they can be re-enqueued")
|
||||
}
|
||||
|
||||
var batchWasEmpty: Boolean
|
||||
|
||||
do {
|
||||
val restoreThumbnailJobs: MutableList<RestoreAttachmentThumbnailJob> = mutableListOf()
|
||||
val restoreFullAttachmentJobs: MutableList<RestoreAttachmentJob> = mutableListOf()
|
||||
@@ -88,6 +105,7 @@ class BackupRestoreMediaJob private constructor(parameters: Parameters) : BaseJo
|
||||
}
|
||||
|
||||
val attachmentBatch = last30DaysAttachments + remaining
|
||||
batchWasEmpty = attachmentBatch.isEmpty()
|
||||
val messageIds = attachmentBatch.map { it.mmsId }.toSet()
|
||||
val messageMap = SignalDatabase.messages.getMessages(messageIds).associate { it.id to (it as MmsMessageRecord) }
|
||||
|
||||
@@ -109,11 +127,13 @@ class BackupRestoreMediaJob private constructor(parameters: Parameters) : BaseJo
|
||||
queueHash = attachment.plaintextHash?.contentHashCode() ?: attachment.remoteKey?.contentHashCode()
|
||||
)
|
||||
} else {
|
||||
restoreThumbnailJobs += RestoreAttachmentThumbnailJob(
|
||||
messageId = attachment.mmsId,
|
||||
attachmentId = attachment.attachmentId,
|
||||
highPriority = false
|
||||
)
|
||||
if (attachment.couldHaveArchivedThumbnail) {
|
||||
restoreThumbnailJobs += RestoreAttachmentThumbnailJob(
|
||||
messageId = attachment.mmsId,
|
||||
attachmentId = attachment.attachmentId,
|
||||
highPriority = false
|
||||
)
|
||||
}
|
||||
|
||||
restoreThumbnailOnlyAttachmentsIds += attachment.attachmentId
|
||||
}
|
||||
@@ -133,14 +153,7 @@ class BackupRestoreMediaJob private constructor(parameters: Parameters) : BaseJo
|
||||
// Intentionally enqueues one at a time for safer attachment transfer state management
|
||||
restoreThumbnailJobs.forEach { jobManager.add(it) }
|
||||
restoreFullAttachmentJobs.forEach { jobManager.add(it) }
|
||||
} while (restoreThumbnailJobs.isNotEmpty() || restoreFullAttachmentJobs.isNotEmpty() || notRestorable.isNotEmpty())
|
||||
|
||||
BackupMediaRestoreService.start(context, context.getString(R.string.BackupStatus__restoring_media))
|
||||
ArchiveRestoreProgress.onRestoringMedia()
|
||||
|
||||
RestoreAttachmentJob.Queues.INITIAL_RESTORE.forEach { queue ->
|
||||
jobManager.add(CheckRestoreMediaLeftJob(queue))
|
||||
}
|
||||
} while (!batchWasEmpty)
|
||||
}
|
||||
|
||||
private fun shouldRestoreFullSize(message: MmsMessageRecord, restoreTime: Long, optimizeStorage: Boolean): Boolean {
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
|
||||
package org.thoughtcrime.securesms.jobs
|
||||
|
||||
import androidx.annotation.VisibleForTesting
|
||||
import org.signal.core.util.DiskUtil
|
||||
import org.signal.core.util.logging.Log
|
||||
import org.thoughtcrime.securesms.backup.v2.ArchiveRestoreProgress
|
||||
@@ -12,7 +13,10 @@ import org.thoughtcrime.securesms.database.SignalDatabase
|
||||
import org.thoughtcrime.securesms.dependencies.AppDependencies
|
||||
import org.thoughtcrime.securesms.jobmanager.Job
|
||||
import org.thoughtcrime.securesms.keyvalue.SignalStore
|
||||
import org.thoughtcrime.securesms.util.RemoteConfig
|
||||
import kotlin.time.Duration
|
||||
import kotlin.time.Duration.Companion.days
|
||||
import kotlin.time.Duration.Companion.milliseconds
|
||||
|
||||
/**
|
||||
* Optimizes media storage by relying on backups for full copies of files and only keeping thumbnails locally.
|
||||
@@ -23,6 +27,11 @@ class OptimizeMediaJob private constructor(parameters: Parameters) : Job(paramet
|
||||
private val TAG = Log.tag(OptimizeMediaJob::class)
|
||||
const val KEY = "OptimizeMediaJob"
|
||||
|
||||
private const val LOW_STORAGE_THRESHOLD_PERCENT = 5f
|
||||
|
||||
/** How many reconciliation intervals a completed crawl stays trustworthy for. */
|
||||
private const val EVIDENCE_AGE_INTERVAL_MULTIPLIER = 4
|
||||
|
||||
fun enqueue() {
|
||||
if (!SignalStore.backup.optimizeStorage || !SignalStore.backup.backsUpMedia) {
|
||||
Log.i(TAG, "Optimize media is not enabled, skipping. backsUpMedia: ${SignalStore.backup.backsUpMedia} optimizeStorage: ${SignalStore.backup.optimizeStorage}")
|
||||
@@ -61,18 +70,16 @@ class OptimizeMediaJob private constructor(parameters: Parameters) : Job(paramet
|
||||
RestoreAttachmentJob.Queues.OFFLOAD_RESTORE.forEach { queue -> AppDependencies.jobManager.add(CheckRestoreMediaLeftJob(queue)) }
|
||||
}
|
||||
|
||||
Log.i(TAG, "Optimizing media in the db")
|
||||
|
||||
val available = DiskUtil.getAvailableSpace(context).bytes.toFloat()
|
||||
val total = DiskUtil.getTotalDiskSize(context).bytes.toFloat()
|
||||
val remaining = (total - available) / total * 100
|
||||
val percentAvailable = if (total > 0f) available / total * 100 else 100f
|
||||
val minimumAge = if (percentAvailable > LOW_STORAGE_THRESHOLD_PERCENT) 30.days else 15.days
|
||||
|
||||
val minimumAge = if (remaining > 5f) 30.days else 15.days
|
||||
Log.i(TAG, "${"%.1f".format(percentAvailable)}% storage available")
|
||||
|
||||
Log.i(TAG, "${"%.1f".format(remaining)}% storage available, optimizing attachments older than $minimumAge")
|
||||
|
||||
SignalDatabase.attachments.markEligibleAttachmentsAsOptimized(minimumAge)
|
||||
offloadVerifiedMedia(minimumAge, RemoteConfig.archiveReconciliationSyncInterval)
|
||||
|
||||
// Reclaims files nothing references anymore, which doesn't depend on CDN confirmation, so it has to run even when offloading is gated off.
|
||||
Log.i(TAG, "Deleting abandoned attachment files")
|
||||
val count = SignalDatabase.attachments.deleteAbandonedAttachmentFiles()
|
||||
Log.i(TAG, "Deleted $count attachments")
|
||||
@@ -80,6 +87,39 @@ class OptimizeMediaJob private constructor(parameters: Parameters) : Job(paramet
|
||||
return Result.success()
|
||||
}
|
||||
|
||||
/** Offloads the local copy of media the archive CDN confirmed during a completed crawl, and nothing else. */
|
||||
@VisibleForTesting
|
||||
internal fun offloadVerifiedMedia(minimumAge: Duration, reconciliationInterval: Duration) {
|
||||
val crawlInterval = reconciliationInterval.coerceAtLeast(1.days)
|
||||
val lastCompletedCrawlVersion = SignalStore.backup.lastCompletedReconciliationSnapshotVersion
|
||||
|
||||
if (lastCompletedCrawlVersion < 0) {
|
||||
Log.w(TAG, "No archive reconciliation has completed on this device yet. Not offloading anything until our media has been verified against the archive CDN.", true)
|
||||
ArchiveAttachmentReconciliationJob.enqueueToVerifyMediaBeforeOffloading(crawlInterval)
|
||||
return
|
||||
}
|
||||
|
||||
val maxEvidenceAge = crawlInterval * EVIDENCE_AGE_INTERVAL_MULTIPLIER
|
||||
val evidenceAge = (System.currentTimeMillis() - SignalStore.backup.lastCompletedReconciliationTime).milliseconds
|
||||
|
||||
// Refusing alone isn't enough here: a far-future timestamp keeps reading as untrustworthy until the clock catches up to it, which could be years.
|
||||
if (evidenceAge.isNegative()) {
|
||||
Log.w(TAG, "The last completed archive reconciliation is timestamped ${-evidenceAge} in the future, likely from a clock change. Discarding our verification state so a fresh crawl has to confirm our media again.", true)
|
||||
SignalStore.backup.clearArchiveVerificationState()
|
||||
ArchiveAttachmentReconciliationJob.enqueueToVerifyMediaBeforeOffloading(crawlInterval)
|
||||
return
|
||||
}
|
||||
|
||||
if (evidenceAge > maxEvidenceAge) {
|
||||
Log.w(TAG, "The last completed archive reconciliation is $evidenceAge old, past the $maxEvidenceAge we trust. Not offloading anything until our media has been verified again.", true)
|
||||
ArchiveAttachmentReconciliationJob.enqueueToVerifyMediaBeforeOffloading(crawlInterval)
|
||||
return
|
||||
}
|
||||
|
||||
Log.i(TAG, "Optimizing attachments older than $minimumAge")
|
||||
SignalDatabase.attachments.markEligibleAttachmentsAsOptimized(lastCompletedCrawlVersion, minimumAge)
|
||||
}
|
||||
|
||||
override fun serialize(): ByteArray? = null
|
||||
override fun getFactoryKey(): String = KEY
|
||||
override fun onFailure() = Unit
|
||||
|
||||
@@ -52,6 +52,9 @@ class BackupValues(store: KeyValueStore) : SignalStoreValues(store) {
|
||||
private const val KEY_NEXT_BACKUP_TIME = "backup.nextBackupTime"
|
||||
private const val KEY_LAST_BACKUP_TIME = "backup.lastBackupTime"
|
||||
private const val KEY_LAST_ATTACHMENT_RECONCILIATION_TIME = "backup.lastBackupMediaSyncTime"
|
||||
private const val KEY_LAST_COMPLETED_RECONCILIATION_SNAPSHOT_VERSION = "backup.lastCompletedReconciliationSnapshotVersion"
|
||||
private const val KEY_LAST_COMPLETED_RECONCILIATION_TIME = "backup.lastCompletedReconciliationTime"
|
||||
private const val KEY_LAST_FORCED_RECONCILIATION_ATTEMPT_TIME = "backup.lastForcedReconciliationAttemptTime"
|
||||
private const val KEY_TOTAL_RESTORABLE_ATTACHMENT_SIZE = "backup.totalRestorableAttachmentSize"
|
||||
private const val KEY_LAST_BACKUP_PROTO_VERSION = "backup.lastBackupProtoVersion"
|
||||
|
||||
@@ -193,6 +196,33 @@ class BackupValues(store: KeyValueStore) : SignalStoreValues(store) {
|
||||
|
||||
var lastAttachmentReconciliationTime: Long by longValue(KEY_LAST_ATTACHMENT_RECONCILIATION_TIME, -1)
|
||||
|
||||
/**
|
||||
* Negative if no crawl has ever completed on this device. Must be cleared by anything that invalidates our view of the CDN (new media root backup key, tier
|
||||
* change), because it gates deleting local copies of media.
|
||||
*/
|
||||
var lastCompletedReconciliationSnapshotVersion: Long by longValue(KEY_LAST_COMPLETED_RECONCILIATION_SNAPSHOT_VERSION, -1)
|
||||
|
||||
/**
|
||||
* When the crawl behind [lastCompletedReconciliationSnapshotVersion] finished, or zero if none ever has. Written alongside it.
|
||||
*/
|
||||
var lastCompletedReconciliationTime: Long by longValue(KEY_LAST_COMPLETED_RECONCILIATION_TIME, 0)
|
||||
|
||||
/**
|
||||
* Advances when a forced crawl *starts*, unlike [lastAttachmentReconciliationTime] which only advances on completion, so that a crawl which can never finish
|
||||
* doesn't get re-forced daily while one that never started still can be.
|
||||
*/
|
||||
var lastForcedReconciliationAttemptTime: Long by longValue(KEY_LAST_FORCED_RECONCILIATION_ATTEMPT_TIME, 0)
|
||||
|
||||
/**
|
||||
* Discards everything we know about having verified our media against the archive CDN, so offloading stays gated until a fresh crawl confirms it again. Call
|
||||
* this from anything that invalidates that view.
|
||||
*/
|
||||
fun clearArchiveVerificationState() {
|
||||
lastCompletedReconciliationSnapshotVersion = -1
|
||||
lastCompletedReconciliationTime = 0
|
||||
lastForcedReconciliationAttemptTime = 0
|
||||
}
|
||||
|
||||
var userManuallySkippedMediaRestore: Boolean by booleanValue(KEY_USER_MANUALLY_SKIPPED_MEDIA_RESTORE, false)
|
||||
|
||||
/**
|
||||
@@ -258,6 +288,8 @@ class BackupValues(store: KeyValueStore) : SignalStoreValues(store) {
|
||||
store.beginWrite().putBlob(KEY_MEDIA_ROOT_BACKUP_KEY, value.value).commit()
|
||||
mediaCredentials.clearAll()
|
||||
cachedMediaCdnPath = null
|
||||
|
||||
clearArchiveVerificationState()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -299,6 +331,8 @@ class BackupValues(store: KeyValueStore) : SignalStoreValues(store) {
|
||||
clearMessageBackupFailureSheetWatermark()
|
||||
backupCreationError = null
|
||||
|
||||
clearArchiveVerificationState()
|
||||
|
||||
if (storedValue == null) {
|
||||
Log.i(TAG, "Enabling backups. Resetting 'finished initial backup' state.")
|
||||
} else if (value == MessageBackupTier.PAID) {
|
||||
|
||||
@@ -46,6 +46,9 @@ class LogSectionRemoteBackups : LogSection {
|
||||
output.append("Detected subscription state mismatch: ${SignalStore.backup.subscriptionStateMismatchDetected}\n")
|
||||
output.append("Last verified key time : ${SignalStore.backup.lastVerifyKeyTime}\n")
|
||||
output.append("Local restore reconcile pending : ${SignalStore.backup.localRestoreReconcilePending}\n")
|
||||
output.append("Last CDN-verified snapshot version : ${SignalStore.backup.lastCompletedReconciliationSnapshotVersion}\n")
|
||||
output.append("Last CDN-verified time : ${SignalStore.backup.lastCompletedReconciliationTime}\n")
|
||||
output.append("Last forced reconciliation attempt : ${SignalStore.backup.lastForcedReconciliationAttemptTime}\n")
|
||||
output.append("Restore state : ${ArchiveRestoreProgress.state}\n")
|
||||
output.append("\n -- Subscription State\n")
|
||||
|
||||
|
||||
+31
@@ -18,6 +18,7 @@ import org.junit.runner.RunWith
|
||||
import org.robolectric.RobolectricTestRunner
|
||||
import org.robolectric.annotation.Config
|
||||
import org.signal.core.models.database.AttachmentId
|
||||
import org.signal.core.util.Base64
|
||||
import org.signal.core.util.logging.Log
|
||||
import org.thoughtcrime.securesms.attachments.ArchivedAttachment
|
||||
import org.thoughtcrime.securesms.attachments.Attachment
|
||||
@@ -90,6 +91,36 @@ class AttachmentTableTest_localRestoreArchiveState {
|
||||
assertThat(SignalDatabase.attachments.hasArchiveFinishedLocalBackupMedia()).isFalse()
|
||||
}
|
||||
|
||||
@Test
|
||||
fun resetArchiveTransferState_skippedWhenThereIsNoLocalDataFileToReUploadFrom() {
|
||||
val attachmentId = insertArchivedAttachment(archiveCdn = 3, localBackupKey = Random.nextBytes(32))
|
||||
val attachment = SignalDatabase.attachments.getAttachment(attachmentId)!!
|
||||
|
||||
val result = SignalDatabase.attachments.resetArchiveTransferStateByPlaintextHashAndRemoteKeyIfNecessary(
|
||||
plaintextHash = Base64.decode(attachment.dataHash!!),
|
||||
remoteKey = Base64.decode(attachment.remoteKey!!)
|
||||
)
|
||||
|
||||
assertThat(result).isEqualTo(AttachmentTable.ArchiveTransferStateResetResult.SKIPPED_NO_LOCAL_DATA)
|
||||
|
||||
val after = SignalDatabase.attachments.getAttachment(attachmentId)!!
|
||||
assertThat(after.archiveTransferState).isEqualTo(AttachmentTable.ArchiveTransferState.FINISHED)
|
||||
assertThat(after.archiveCdn).isEqualTo(3)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun resetArchiveTransferState_notNeededWhenNotFinished() {
|
||||
val attachmentId = insertArchivedAttachment(archiveCdn = null, localBackupKey = Random.nextBytes(32))
|
||||
val attachment = SignalDatabase.attachments.getAttachment(attachmentId)!!
|
||||
|
||||
val result = SignalDatabase.attachments.resetArchiveTransferStateByPlaintextHashAndRemoteKeyIfNecessary(
|
||||
plaintextHash = Base64.decode(attachment.dataHash!!),
|
||||
remoteKey = Base64.decode(attachment.remoteKey!!)
|
||||
)
|
||||
|
||||
assertThat(result).isEqualTo(AttachmentTable.ArchiveTransferStateResetResult.NOT_NEEDED)
|
||||
}
|
||||
|
||||
private fun insertArchivedAttachment(archiveCdn: Int?, localBackupKey: ByteArray?): AttachmentId {
|
||||
val from = recipients.createRecipient("Some Contact")
|
||||
val threadId = SignalDatabase.threads.getOrCreateThreadIdFor(Recipient.resolved(from))
|
||||
|
||||
+267
@@ -0,0 +1,267 @@
|
||||
/*
|
||||
* Copyright 2026 Signal Messenger, LLC
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
package org.thoughtcrime.securesms.database
|
||||
|
||||
import android.app.Application
|
||||
import assertk.assertThat
|
||||
import assertk.assertions.hasSize
|
||||
import assertk.assertions.isEmpty
|
||||
import assertk.assertions.isEqualTo
|
||||
import org.junit.BeforeClass
|
||||
import org.junit.Rule
|
||||
import org.junit.Test
|
||||
import org.junit.runner.RunWith
|
||||
import org.robolectric.RobolectricTestRunner
|
||||
import org.robolectric.annotation.Config
|
||||
import org.signal.core.models.database.AttachmentId
|
||||
import org.signal.core.util.Base64
|
||||
import org.signal.core.util.logging.Log
|
||||
import org.signal.core.util.update
|
||||
import org.thoughtcrime.securesms.attachments.ArchivedAttachment
|
||||
import org.thoughtcrime.securesms.attachments.Attachment
|
||||
import org.thoughtcrime.securesms.mms.IncomingMessage
|
||||
import org.thoughtcrime.securesms.recipients.Recipient
|
||||
import org.thoughtcrime.securesms.testutil.RecipientTestRule
|
||||
import org.thoughtcrime.securesms.testutil.SystemOutLogger
|
||||
import java.util.UUID
|
||||
import kotlin.random.Random
|
||||
|
||||
/**
|
||||
* Coverage for which thumbnail entries survive into the backup media snapshot. An entry that does not survive falls out of the snapshot and is
|
||||
* therefore marked for deletion off of the archive CDN, so media with no local data file but an archived thumbnail has to be kept.
|
||||
*/
|
||||
@Suppress("ClassName")
|
||||
@RunWith(RobolectricTestRunner::class)
|
||||
@Config(manifest = Config.NONE, application = Application::class)
|
||||
class AttachmentTableTest_thumbnailSnapshotEntries {
|
||||
|
||||
@get:Rule val recipients = RecipientTestRule()
|
||||
|
||||
companion object {
|
||||
@BeforeClass
|
||||
@JvmStatic
|
||||
fun setUpClass() {
|
||||
Log.initialize(SystemOutLogger())
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
fun givenNoLocalDataFileButAnArchivedThumbnail_whenIFilter_thenIExpectTheEntryKept() {
|
||||
val attachmentId = insertArchivedAttachment()
|
||||
SignalDatabase.attachments.setArchiveThumbnailTransferState(attachmentId, AttachmentTable.ArchiveTransferState.FINISHED)
|
||||
|
||||
val entry = thumbnailEntryFor(attachmentId)
|
||||
val kept = SignalDatabase.attachments.filterThumbnailsWithoutEligibleAttachment(setOf(entry))
|
||||
|
||||
assertThat(kept.size).isEqualTo(1)
|
||||
assertThat(kept.first().mediaId).isEqualTo(entry.mediaId)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun givenNoLocalDataFileAndNoArchivedThumbnail_whenIFilter_thenIExpectTheEntryDropped() {
|
||||
val attachmentId = insertArchivedAttachment()
|
||||
|
||||
val kept = SignalDatabase.attachments.filterThumbnailsWithoutEligibleAttachment(setOf(thumbnailEntryFor(attachmentId)))
|
||||
|
||||
assertThat(kept).isEmpty()
|
||||
}
|
||||
|
||||
@Test
|
||||
fun givenAThumbnailRestoredFromTheCdn_whenIFilter_thenIExpectTheEntryKept() {
|
||||
val attachmentId = insertArchivedAttachment()
|
||||
SignalDatabase.attachments.setThumbnailRestoreState(listOf(attachmentId), AttachmentTable.ThumbnailRestoreState.FINISHED)
|
||||
|
||||
val entry = thumbnailEntryFor(attachmentId)
|
||||
val kept = SignalDatabase.attachments.filterThumbnailsWithoutEligibleAttachment(setOf(entry))
|
||||
|
||||
assertThat(kept.size).isEqualTo(1)
|
||||
assertThat(kept.first().mediaId).isEqualTo(entry.mediaId)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun givenAPermanentlyFailedThumbnail_whenIFilter_thenIExpectTheEntryDropped() {
|
||||
val attachmentId = insertArchivedAttachment()
|
||||
SignalDatabase.attachments.setArchiveThumbnailTransferState(attachmentId, AttachmentTable.ArchiveTransferState.PERMANENT_FAILURE)
|
||||
|
||||
val kept = SignalDatabase.attachments.filterThumbnailsWithoutEligibleAttachment(setOf(thumbnailEntryFor(attachmentId)))
|
||||
|
||||
assertThat(kept).isEmpty()
|
||||
}
|
||||
|
||||
/**
|
||||
* These four mirror the cases above against [AttachmentTable.getMediaNamesWithNoEligibleThumbnail], which answers the same question with an indexed lookup
|
||||
* instead of a full scan. The two share a predicate, so a divergence here means the CDN-delete side and the snapshot-write side have drifted apart.
|
||||
*/
|
||||
@Test
|
||||
fun givenNoLocalDataFileButAnArchivedThumbnail_whenIQueryByName_thenIExpectItStillWanted() {
|
||||
val attachmentId = insertArchivedAttachment()
|
||||
SignalDatabase.attachments.setArchiveThumbnailTransferState(attachmentId, AttachmentTable.ArchiveTransferState.FINISHED)
|
||||
|
||||
assertThat(SignalDatabase.attachments.getMediaNamesWithNoEligibleThumbnail(setOf(mediaNameFor(attachmentId)))).isEmpty()
|
||||
}
|
||||
|
||||
@Test
|
||||
fun givenNoLocalDataFileAndNoArchivedThumbnail_whenIQueryByName_thenIExpectItUnwanted() {
|
||||
val attachmentId = insertArchivedAttachment()
|
||||
|
||||
assertThat(SignalDatabase.attachments.getMediaNamesWithNoEligibleThumbnail(setOf(mediaNameFor(attachmentId)))).hasSize(1)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun givenAThumbnailRestoredFromTheCdn_whenIQueryByName_thenIExpectItStillWanted() {
|
||||
val attachmentId = insertArchivedAttachment()
|
||||
SignalDatabase.attachments.setThumbnailRestoreState(listOf(attachmentId), AttachmentTable.ThumbnailRestoreState.FINISHED)
|
||||
|
||||
assertThat(SignalDatabase.attachments.getMediaNamesWithNoEligibleThumbnail(setOf(mediaNameFor(attachmentId)))).isEmpty()
|
||||
}
|
||||
|
||||
@Test
|
||||
fun givenAPermanentlyFailedThumbnail_whenIQueryByName_thenIExpectItUnwanted() {
|
||||
val attachmentId = insertArchivedAttachment()
|
||||
SignalDatabase.attachments.setArchiveThumbnailTransferState(attachmentId, AttachmentTable.ArchiveTransferState.PERMANENT_FAILURE)
|
||||
|
||||
assertThat(SignalDatabase.attachments.getMediaNamesWithNoEligibleThumbnail(setOf(mediaNameFor(attachmentId)))).hasSize(1)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun givenAMediaNameWithNoAttachmentAtAll_whenIQueryByName_thenIExpectItUnwanted() {
|
||||
val orphan = AttachmentTable.MediaNameParts(
|
||||
plaintextHash = Base64.encodeWithPadding(Random.nextBytes(32)),
|
||||
remoteKey = Base64.encodeWithPadding(Random.nextBytes(32))
|
||||
)
|
||||
|
||||
assertThat(SignalDatabase.attachments.getMediaNamesWithNoEligibleThumbnail(setOf(orphan))).hasSize(1)
|
||||
}
|
||||
|
||||
/**
|
||||
* These pin the clause against [org.thoughtcrime.securesms.jobs.BackupMessagesJob]'s thumbnail filters. Anything the write side refuses to put in a snapshot
|
||||
* has to read as unwanted here, or we keep paying to store an object nothing will ever reference again.
|
||||
*/
|
||||
@Test
|
||||
fun givenAWallpaper_whenIFilter_thenIExpectTheEntryDropped() {
|
||||
val attachmentId = insertArchivedAttachment()
|
||||
SignalDatabase.attachments.setArchiveThumbnailTransferState(attachmentId, AttachmentTable.ArchiveTransferState.FINISHED)
|
||||
setColumn(attachmentId, AttachmentTable.MESSAGE_ID, AttachmentTable.WALLPAPER_MESSAGE_ID)
|
||||
|
||||
assertThat(SignalDatabase.attachments.filterThumbnailsWithoutEligibleAttachment(setOf(thumbnailEntryFor(attachmentId)))).isEmpty()
|
||||
}
|
||||
|
||||
@Test
|
||||
fun givenAWallpaper_whenIQueryByName_thenIExpectItUnwanted() {
|
||||
val attachmentId = insertArchivedAttachment()
|
||||
SignalDatabase.attachments.setArchiveThumbnailTransferState(attachmentId, AttachmentTable.ArchiveTransferState.FINISHED)
|
||||
setColumn(attachmentId, AttachmentTable.MESSAGE_ID, AttachmentTable.WALLPAPER_MESSAGE_ID)
|
||||
|
||||
assertThat(SignalDatabase.attachments.getMediaNamesWithNoEligibleThumbnail(setOf(mediaNameFor(attachmentId)))).hasSize(1)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun givenNonVisualMedia_whenIQueryByName_thenIExpectItUnwanted() {
|
||||
val attachmentId = insertArchivedAttachment(contentType = "application/pdf")
|
||||
SignalDatabase.attachments.setArchiveThumbnailTransferState(attachmentId, AttachmentTable.ArchiveTransferState.FINISHED)
|
||||
|
||||
assertThat(SignalDatabase.attachments.getMediaNamesWithNoEligibleThumbnail(setOf(mediaNameFor(attachmentId)))).hasSize(1)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun givenAnSvg_whenIQueryByName_thenIExpectItUnwanted() {
|
||||
val attachmentId = insertArchivedAttachment(contentType = "image/svg+xml")
|
||||
SignalDatabase.attachments.setArchiveThumbnailTransferState(attachmentId, AttachmentTable.ArchiveTransferState.FINISHED)
|
||||
|
||||
assertThat(SignalDatabase.attachments.getMediaNamesWithNoEligibleThumbnail(setOf(mediaNameFor(attachmentId)))).hasSize(1)
|
||||
}
|
||||
|
||||
/**
|
||||
* Stickers are excluded from thumbnail generation upstream, so in practice they never reach the CDN and this branch is unreachable. It stays unfiltered
|
||||
* deliberately: this clause exists to mirror the snapshot write side, and the write side doesn't special-case stickers either.
|
||||
*/
|
||||
@Test
|
||||
fun givenASticker_whenIQueryByName_thenIExpectItStillWanted() {
|
||||
val attachmentId = insertArchivedAttachment()
|
||||
SignalDatabase.attachments.setArchiveThumbnailTransferState(attachmentId, AttachmentTable.ArchiveTransferState.FINISHED)
|
||||
|
||||
// Read before writing the sticker id, since setting it alone leaves a sticker locator the attachment reader can't parse.
|
||||
val mediaName = mediaNameFor(attachmentId)
|
||||
setColumn(attachmentId, AttachmentTable.STICKER_ID, 7L)
|
||||
|
||||
assertThat(SignalDatabase.attachments.getMediaNamesWithNoEligibleThumbnail(setOf(mediaName))).isEmpty()
|
||||
}
|
||||
|
||||
private fun setColumn(attachmentId: AttachmentId, column: String, value: Long) {
|
||||
SignalDatabase.attachments.writableDatabase
|
||||
.update(AttachmentTable.TABLE_NAME)
|
||||
.values(column to value)
|
||||
.where("${AttachmentTable.ID} = ?", attachmentId.id)
|
||||
.run()
|
||||
}
|
||||
|
||||
private fun mediaNameFor(attachmentId: AttachmentId): AttachmentTable.MediaNameParts {
|
||||
val attachment = SignalDatabase.attachments.getAttachment(attachmentId)!!
|
||||
|
||||
return AttachmentTable.MediaNameParts(
|
||||
plaintextHash = attachment.dataHash!!,
|
||||
remoteKey = attachment.remoteKey!!
|
||||
)
|
||||
}
|
||||
|
||||
private fun thumbnailEntryFor(attachmentId: AttachmentId): BackupMediaSnapshotTable.MediaEntry {
|
||||
val attachment = SignalDatabase.attachments.getAttachment(attachmentId)!!
|
||||
|
||||
return BackupMediaSnapshotTable.MediaEntry(
|
||||
mediaId = "media-id-${attachment.attachmentId.id}",
|
||||
cdn = 3,
|
||||
plaintextHash = Base64.decode(attachment.dataHash!!),
|
||||
remoteKey = Base64.decode(attachment.remoteKey!!),
|
||||
isThumbnail = true
|
||||
)
|
||||
}
|
||||
|
||||
private fun insertArchivedAttachment(contentType: String = "image/jpeg"): AttachmentId {
|
||||
val from = recipients.createRecipient("Some Contact")
|
||||
val threadId = SignalDatabase.threads.getOrCreateThreadIdFor(Recipient.resolved(from))
|
||||
|
||||
val message = IncomingMessage(
|
||||
type = MessageType.NORMAL,
|
||||
from = from,
|
||||
body = null,
|
||||
sentTimeMillis = 100L,
|
||||
serverTimeMillis = 100L,
|
||||
receivedTimeMillis = 200L,
|
||||
attachments = listOf(createArchivedAttachment(contentType))
|
||||
)
|
||||
|
||||
val messageId = SignalDatabase.messages.insertMessageInbox(message, threadId).get().messageId
|
||||
return SignalDatabase.attachments.getAttachmentsForMessage(messageId).first().attachmentId
|
||||
}
|
||||
|
||||
private fun createArchivedAttachment(contentType: String): Attachment {
|
||||
return ArchivedAttachment(
|
||||
contentType = contentType,
|
||||
size = 1024,
|
||||
cdn = 3,
|
||||
uploadTimestamp = 0,
|
||||
key = Random.nextBytes(32),
|
||||
cdnKey = "password",
|
||||
archiveCdn = 3,
|
||||
plaintextHash = Random.nextBytes(32),
|
||||
incrementalMac = Random.nextBytes(8),
|
||||
incrementalMacChunkSize = 8,
|
||||
width = 100,
|
||||
height = 100,
|
||||
caption = null,
|
||||
blurHash = null,
|
||||
voiceNote = false,
|
||||
borderless = false,
|
||||
stickerLocator = null,
|
||||
gif = false,
|
||||
quote = false,
|
||||
quoteTargetContentType = null,
|
||||
uuid = UUID.randomUUID(),
|
||||
fileName = null,
|
||||
localBackupKey = null
|
||||
)
|
||||
}
|
||||
}
|
||||
+275
@@ -0,0 +1,275 @@
|
||||
/*
|
||||
* Copyright 2026 Signal Messenger, LLC
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
package org.thoughtcrime.securesms.database
|
||||
|
||||
import android.app.Application
|
||||
import assertk.assertThat
|
||||
import assertk.assertions.containsExactly
|
||||
import assertk.assertions.isEmpty
|
||||
import assertk.assertions.isEqualTo
|
||||
import assertk.assertions.isFalse
|
||||
import assertk.assertions.isTrue
|
||||
import org.junit.BeforeClass
|
||||
import org.junit.Rule
|
||||
import org.junit.Test
|
||||
import org.junit.runner.RunWith
|
||||
import org.robolectric.RobolectricTestRunner
|
||||
import org.robolectric.annotation.Config
|
||||
import org.signal.core.models.database.AttachmentId
|
||||
import org.signal.core.util.logging.Log
|
||||
import org.signal.core.util.update
|
||||
import org.thoughtcrime.securesms.attachments.ArchivedAttachment
|
||||
import org.thoughtcrime.securesms.attachments.Attachment
|
||||
import org.thoughtcrime.securesms.database.AttachmentTable.ArchiveTransferState
|
||||
import org.thoughtcrime.securesms.mms.IncomingMessage
|
||||
import org.thoughtcrime.securesms.recipients.Recipient
|
||||
import org.thoughtcrime.securesms.testutil.RecipientTestRule
|
||||
import org.thoughtcrime.securesms.testutil.SystemOutLogger
|
||||
import java.util.UUID
|
||||
import kotlin.random.Random
|
||||
|
||||
/**
|
||||
* Coverage for which thumbnails are selected for archive work. A thumbnail qualifies when the archive has no copy, and also when we have no
|
||||
* local copy: without a local thumbnail file the attachment can never satisfy the offload eligibility check, so a restored device would be stuck.
|
||||
*/
|
||||
@Suppress("ClassName")
|
||||
@RunWith(RobolectricTestRunner::class)
|
||||
@Config(manifest = Config.NONE, application = Application::class)
|
||||
class AttachmentTableTest_thumbnailsThatNeedArchiveWork {
|
||||
|
||||
@get:Rule val recipients = RecipientTestRule()
|
||||
|
||||
companion object {
|
||||
@BeforeClass
|
||||
@JvmStatic
|
||||
fun setUpClass() {
|
||||
Log.initialize(SystemOutLogger())
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
fun givenAnArchivedThumbnailWithNoLocalFile_whenIQuery_thenIExpectItSelected() {
|
||||
val attachmentId = insertAttachment(thumbnailState = ArchiveTransferState.FINISHED, hasLocalThumbnail = false)
|
||||
|
||||
assertThat(SignalDatabase.attachments.getThumbnailsThatNeedArchiveUpload()).containsExactly(attachmentId)
|
||||
assertThat(SignalDatabase.attachments.doAnyThumbnailsNeedArchiveUpload()).isTrue()
|
||||
}
|
||||
|
||||
@Test
|
||||
fun givenAnArchivedThumbnailWithALocalFile_whenIQuery_thenIExpectItSkipped() {
|
||||
insertAttachment(thumbnailState = ArchiveTransferState.FINISHED, hasLocalThumbnail = true)
|
||||
|
||||
assertThat(SignalDatabase.attachments.getThumbnailsThatNeedArchiveUpload()).isEmpty()
|
||||
assertThat(SignalDatabase.attachments.doAnyThumbnailsNeedArchiveUpload()).isFalse()
|
||||
}
|
||||
|
||||
@Test
|
||||
fun givenAnUnarchivedThumbnailWithALocalFile_whenIQuery_thenIExpectItSelected() {
|
||||
val attachmentId = insertAttachment(thumbnailState = ArchiveTransferState.NONE, hasLocalThumbnail = true)
|
||||
|
||||
assertThat(SignalDatabase.attachments.getThumbnailsThatNeedArchiveUpload()).containsExactly(attachmentId)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun givenAnUnarchivedThumbnailWithNoLocalFile_whenIQuery_thenIExpectItSelected() {
|
||||
val attachmentId = insertAttachment(thumbnailState = ArchiveTransferState.NONE, hasLocalThumbnail = false)
|
||||
|
||||
assertThat(SignalDatabase.attachments.getThumbnailsThatNeedArchiveUpload()).containsExactly(attachmentId)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun givenATemporarilyFailedThumbnail_whenIQuery_thenIExpectItSelected() {
|
||||
val attachmentId = insertAttachment(thumbnailState = ArchiveTransferState.TEMPORARY_FAILURE, hasLocalThumbnail = true)
|
||||
|
||||
assertThat(SignalDatabase.attachments.getThumbnailsThatNeedArchiveUpload()).containsExactly(attachmentId)
|
||||
}
|
||||
|
||||
/**
|
||||
* An upload writes the thumbnail file only once it finishes, so an in-flight row always looks like it has no local copy. Selecting it would enqueue a second
|
||||
* upload for work already underway.
|
||||
*/
|
||||
@Test
|
||||
fun givenAnInFlightUploadWithNoLocalFile_whenIQuery_thenIExpectItSkipped() {
|
||||
insertAttachment(thumbnailState = ArchiveTransferState.UPLOAD_IN_PROGRESS, hasLocalThumbnail = false)
|
||||
|
||||
assertThat(SignalDatabase.attachments.getThumbnailsThatNeedArchiveUpload()).isEmpty()
|
||||
assertThat(SignalDatabase.attachments.doAnyThumbnailsNeedArchiveUpload()).isFalse()
|
||||
}
|
||||
|
||||
@Test
|
||||
fun givenACopyPendingThumbnailWithNoLocalFile_whenIQuery_thenIExpectItSkipped() {
|
||||
insertAttachment(thumbnailState = ArchiveTransferState.COPY_PENDING, hasLocalThumbnail = false)
|
||||
|
||||
assertThat(SignalDatabase.attachments.getThumbnailsThatNeedArchiveUpload()).isEmpty()
|
||||
}
|
||||
|
||||
/**
|
||||
* If we already established we can't produce a local thumbnail for this media, selecting it again just fails again after every backup.
|
||||
*/
|
||||
@Test
|
||||
fun givenAnArchivedThumbnailWeCannotRestoreLocally_whenIQuery_thenIExpectItSkipped() {
|
||||
val attachmentId = insertAttachment(thumbnailState = ArchiveTransferState.FINISHED, hasLocalThumbnail = false)
|
||||
SignalDatabase.attachments.setThumbnailRestoreState(attachmentId, AttachmentTable.ThumbnailRestoreState.PERMANENT_FAILURE)
|
||||
|
||||
assertThat(SignalDatabase.attachments.getThumbnailsThatNeedArchiveUpload()).isEmpty()
|
||||
assertThat(SignalDatabase.attachments.doAnyThumbnailsNeedArchiveUpload()).isFalse()
|
||||
}
|
||||
|
||||
@Test
|
||||
fun givenAPermanentlyFailedThumbnailWithNoLocalFile_whenIQuery_thenIExpectItSkipped() {
|
||||
insertAttachment(thumbnailState = ArchiveTransferState.PERMANENT_FAILURE, hasLocalThumbnail = false)
|
||||
|
||||
assertThat(SignalDatabase.attachments.getThumbnailsThatNeedArchiveUpload()).isEmpty()
|
||||
assertThat(SignalDatabase.attachments.doAnyThumbnailsNeedArchiveUpload()).isFalse()
|
||||
}
|
||||
|
||||
@Test
|
||||
fun givenNoLocalDataFile_whenIQuery_thenIExpectItSkipped() {
|
||||
insertAttachment(thumbnailState = ArchiveTransferState.FINISHED, hasLocalThumbnail = false, hasLocalData = false)
|
||||
|
||||
assertThat(SignalDatabase.attachments.getThumbnailsThatNeedArchiveUpload()).isEmpty()
|
||||
}
|
||||
|
||||
@Test
|
||||
fun givenNonVisualMedia_whenIQuery_thenIExpectItSkipped() {
|
||||
insertAttachment(thumbnailState = ArchiveTransferState.FINISHED, hasLocalThumbnail = false, contentType = "application/pdf")
|
||||
|
||||
assertThat(SignalDatabase.attachments.getThumbnailsThatNeedArchiveUpload()).isEmpty()
|
||||
}
|
||||
|
||||
@Test
|
||||
fun givenAMixOfStates_whenIQuery_thenIExpectOnlyTheOnesNeedingWork() {
|
||||
val archivedWithNoFile = insertAttachment(thumbnailState = ArchiveTransferState.FINISHED, hasLocalThumbnail = false)
|
||||
val unarchived = insertAttachment(thumbnailState = ArchiveTransferState.NONE, hasLocalThumbnail = true)
|
||||
insertAttachment(thumbnailState = ArchiveTransferState.FINISHED, hasLocalThumbnail = true)
|
||||
insertAttachment(thumbnailState = ArchiveTransferState.PERMANENT_FAILURE, hasLocalThumbnail = false)
|
||||
|
||||
assertThat(SignalDatabase.attachments.getThumbnailsThatNeedArchiveUpload()).containsExactly(archivedWithNoFile, unarchived)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun givenALocalThumbnailFile_whenIAskIfItHasOne_thenIExpectTrue() {
|
||||
val attachmentId = insertAttachment(thumbnailState = ArchiveTransferState.FINISHED, hasLocalThumbnail = true)
|
||||
|
||||
assertThat(SignalDatabase.attachments.hasThumbnailFile(attachmentId)).isTrue()
|
||||
}
|
||||
|
||||
@Test
|
||||
fun givenNoLocalThumbnailFile_whenIAskIfItHasOne_thenIExpectFalse() {
|
||||
val attachmentId = insertAttachment(thumbnailState = ArchiveTransferState.FINISHED, hasLocalThumbnail = false)
|
||||
|
||||
assertThat(SignalDatabase.attachments.hasThumbnailFile(attachmentId)).isFalse()
|
||||
}
|
||||
|
||||
/**
|
||||
* [AttachmentTable.ThumbnailRestoreState.PERMANENT_FAILURE] is also what stops the restore job from downloading, so tombstoning a row a restore could still
|
||||
* recover would cost the user a thumbnail the archive CDN is holding.
|
||||
*/
|
||||
@Test
|
||||
fun givenARestoreIsPending_whenIMarkItUnrestorable_thenIExpectTheRestoreStateUntouched() {
|
||||
val attachmentId = insertAttachment(thumbnailState = ArchiveTransferState.FINISHED, hasLocalThumbnail = false)
|
||||
SignalDatabase.attachments.setThumbnailRestoreState(attachmentId, AttachmentTable.ThumbnailRestoreState.NEEDS_RESTORE)
|
||||
|
||||
SignalDatabase.attachments.markThumbnailPermanentlyFailedIfUnrestorable(attachmentId)
|
||||
|
||||
assertThat(restoreStateOf(attachmentId)).isEqualTo(AttachmentTable.ThumbnailRestoreState.NEEDS_RESTORE)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun givenARestoreAlreadyFinished_whenIMarkItUnrestorable_thenIExpectTheRestoreStateUntouched() {
|
||||
val attachmentId = insertAttachment(thumbnailState = ArchiveTransferState.FINISHED, hasLocalThumbnail = false)
|
||||
SignalDatabase.attachments.setThumbnailRestoreState(attachmentId, AttachmentTable.ThumbnailRestoreState.FINISHED)
|
||||
|
||||
SignalDatabase.attachments.markThumbnailPermanentlyFailedIfUnrestorable(attachmentId)
|
||||
|
||||
assertThat(restoreStateOf(attachmentId)).isEqualTo(AttachmentTable.ThumbnailRestoreState.FINISHED)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun givenNoRestoreIsPossible_whenIMarkItUnrestorable_thenIExpectItTombstonedAndSkipped() {
|
||||
val attachmentId = insertAttachment(thumbnailState = ArchiveTransferState.FINISHED, hasLocalThumbnail = false)
|
||||
SignalDatabase.attachments.setThumbnailRestoreState(attachmentId, AttachmentTable.ThumbnailRestoreState.NONE)
|
||||
|
||||
SignalDatabase.attachments.markThumbnailPermanentlyFailedIfUnrestorable(attachmentId)
|
||||
|
||||
assertThat(restoreStateOf(attachmentId)).isEqualTo(AttachmentTable.ThumbnailRestoreState.PERMANENT_FAILURE)
|
||||
assertThat(SignalDatabase.attachments.getThumbnailsThatNeedArchiveUpload()).isEmpty()
|
||||
}
|
||||
|
||||
private fun restoreStateOf(attachmentId: AttachmentId): AttachmentTable.ThumbnailRestoreState {
|
||||
return SignalDatabase.attachments.getAttachment(attachmentId)!!.thumbnailRestoreState
|
||||
}
|
||||
|
||||
private fun insertAttachment(
|
||||
thumbnailState: ArchiveTransferState,
|
||||
hasLocalThumbnail: Boolean,
|
||||
hasLocalData: Boolean = true,
|
||||
contentType: String = "image/jpeg"
|
||||
): AttachmentId {
|
||||
val attachmentId = insertArchivedAttachment(contentType)
|
||||
|
||||
SignalDatabase.attachments.writableDatabase
|
||||
.update(AttachmentTable.TABLE_NAME)
|
||||
.values(
|
||||
AttachmentTable.DATA_FILE to if (hasLocalData) "/fake/path/data-${attachmentId.id}" else null,
|
||||
AttachmentTable.DATA_RANDOM to if (hasLocalData) Random.nextBytes(32) else null,
|
||||
AttachmentTable.TRANSFER_STATE to AttachmentTable.TRANSFER_PROGRESS_DONE,
|
||||
AttachmentTable.THUMBNAIL_FILE to if (hasLocalThumbnail) "/fake/path/thumb-${attachmentId.id}" else null,
|
||||
AttachmentTable.THUMBNAIL_RANDOM to if (hasLocalThumbnail) Random.nextBytes(32) else null,
|
||||
AttachmentTable.ARCHIVE_THUMBNAIL_TRANSFER_STATE to thumbnailState.value
|
||||
)
|
||||
.where("${AttachmentTable.ID} = ?", attachmentId.id)
|
||||
.run()
|
||||
|
||||
return attachmentId
|
||||
}
|
||||
|
||||
private fun insertArchivedAttachment(contentType: String): AttachmentId {
|
||||
val from = recipients.createRecipient("Some Contact")
|
||||
val threadId = SignalDatabase.threads.getOrCreateThreadIdFor(Recipient.resolved(from))
|
||||
|
||||
val message = IncomingMessage(
|
||||
type = MessageType.NORMAL,
|
||||
from = from,
|
||||
body = null,
|
||||
sentTimeMillis = 100L,
|
||||
serverTimeMillis = 100L,
|
||||
receivedTimeMillis = 200L,
|
||||
attachments = listOf(createArchivedAttachment(contentType))
|
||||
)
|
||||
|
||||
val messageId = SignalDatabase.messages.insertMessageInbox(message, threadId).get().messageId
|
||||
return SignalDatabase.attachments.getAttachmentsForMessage(messageId).first().attachmentId
|
||||
}
|
||||
|
||||
private fun createArchivedAttachment(contentType: String): Attachment {
|
||||
return ArchivedAttachment(
|
||||
contentType = contentType,
|
||||
size = 1024,
|
||||
cdn = 3,
|
||||
uploadTimestamp = 0,
|
||||
key = Random.nextBytes(32),
|
||||
cdnKey = "password",
|
||||
archiveCdn = 3,
|
||||
plaintextHash = Random.nextBytes(32),
|
||||
incrementalMac = Random.nextBytes(8),
|
||||
incrementalMacChunkSize = 8,
|
||||
width = 100,
|
||||
height = 100,
|
||||
caption = null,
|
||||
blurHash = null,
|
||||
voiceNote = false,
|
||||
borderless = false,
|
||||
stickerLocator = null,
|
||||
gif = false,
|
||||
quote = false,
|
||||
quoteTargetContentType = null,
|
||||
uuid = UUID.randomUUID(),
|
||||
fileName = null,
|
||||
localBackupKey = null
|
||||
)
|
||||
}
|
||||
}
|
||||
+68
-2
@@ -2,6 +2,7 @@ package org.thoughtcrime.securesms.database
|
||||
|
||||
import android.app.Application
|
||||
import assertk.assertThat
|
||||
import assertk.assertions.isEmpty
|
||||
import assertk.assertions.isEqualTo
|
||||
import org.junit.Rule
|
||||
import org.junit.Test
|
||||
@@ -108,8 +109,8 @@ class BackupMediaSnapshotTableTest {
|
||||
SignalDatabase.backupMediaSnapshots.writePendingMediaEntries(generateArchiveMediaItemSequence(count = additionalCount))
|
||||
SignalDatabase.backupMediaSnapshots.commitPendingRows()
|
||||
|
||||
val page = SignalDatabase.backupMediaSnapshots.getPageOfOldMediaObjects(pageSize = 1_000)
|
||||
SignalDatabase.backupMediaSnapshots.deleteOldMediaObjects(page)
|
||||
val page = SignalDatabase.backupMediaSnapshots.getPageOfOldMediaEntries(pageSize = 1_000)
|
||||
SignalDatabase.backupMediaSnapshots.deleteOldMediaObjects(page.map { it.mediaId })
|
||||
|
||||
val total = getTotalItemCount(includeThumbnails = false)
|
||||
|
||||
@@ -251,6 +252,71 @@ class BackupMediaSnapshotTableTest {
|
||||
assertThat(notSeenCount).isEqualTo(expectedOldCount)
|
||||
}
|
||||
|
||||
/**
|
||||
* A reconciliation that completes while the snapshot is empty records version 0, and 0 is also the never-confirmed default, so the minimum must not be
|
||||
* satisfiable by rows the CDN has never confirmed. Otherwise offloading would delete the only local copy of unconfirmed media.
|
||||
*/
|
||||
@Test
|
||||
fun getMediaIdsConfirmedOnCdn_neverSeenOnRemoteIsNotConfirmedEvenWhenMinimumIsZero() {
|
||||
val items = generateArchiveMediaItemSequence(count = 10)
|
||||
|
||||
SignalDatabase.backupMediaSnapshots.writePendingMediaEntries(items)
|
||||
SignalDatabase.backupMediaSnapshots.commitPendingRows()
|
||||
|
||||
val confirmed = SignalDatabase.backupMediaSnapshots.getMediaIdsConfirmedOnCdn(items.map { it.mediaId }, minimumLastSeenSnapshotVersion = 0)
|
||||
|
||||
assertThat(confirmed).isEmpty()
|
||||
}
|
||||
|
||||
@Test
|
||||
fun getMediaIdsConfirmedOnCdn_seenOnRemoteIsConfirmed() {
|
||||
val items = generateArchiveMediaItemSequence(count = 10)
|
||||
|
||||
SignalDatabase.backupMediaSnapshots.writePendingMediaEntries(items)
|
||||
SignalDatabase.backupMediaSnapshots.commitPendingRows()
|
||||
SignalDatabase.backupMediaSnapshots.markSeenOnRemote(items.map { it.mediaId }, 1)
|
||||
|
||||
val confirmed = SignalDatabase.backupMediaSnapshots.getMediaIdsConfirmedOnCdn(items.map { it.mediaId }, minimumLastSeenSnapshotVersion = 1)
|
||||
|
||||
assertThat(confirmed.size).isEqualTo(items.size)
|
||||
}
|
||||
|
||||
/**
|
||||
* A reconciliation captures its snapshot version once and can resume days later, after further backups have committed. Replaying the version it captured must
|
||||
* not walk a confirmation backwards, and least of all to 0, which the prune reads as "the CDN has never had this".
|
||||
*/
|
||||
@Test
|
||||
fun markSeenOnRemote_staleVersionDoesNotEraseNewerConfirmation() {
|
||||
val items = generateArchiveMediaItemSequence(count = 10)
|
||||
val mediaIds = items.map { it.mediaId }
|
||||
|
||||
SignalDatabase.backupMediaSnapshots.writePendingMediaEntries(items)
|
||||
SignalDatabase.backupMediaSnapshots.commitPendingRows()
|
||||
SignalDatabase.backupMediaSnapshots.markSeenOnRemote(mediaIds, 1)
|
||||
|
||||
SignalDatabase.backupMediaSnapshots.markSeenOnRemote(mediaIds, 0)
|
||||
|
||||
val confirmed = SignalDatabase.backupMediaSnapshots.getMediaIdsConfirmedOnCdn(mediaIds, minimumLastSeenSnapshotVersion = 1)
|
||||
|
||||
assertThat(confirmed.size).isEqualTo(items.size)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun markSeenOnRemote_newerVersionAdvancesConfirmation() {
|
||||
val items = generateArchiveMediaItemSequence(count = 10)
|
||||
val mediaIds = items.map { it.mediaId }
|
||||
|
||||
SignalDatabase.backupMediaSnapshots.writePendingMediaEntries(items)
|
||||
SignalDatabase.backupMediaSnapshots.commitPendingRows()
|
||||
SignalDatabase.backupMediaSnapshots.markSeenOnRemote(mediaIds, 1)
|
||||
|
||||
SignalDatabase.backupMediaSnapshots.markSeenOnRemote(mediaIds, 2)
|
||||
|
||||
val confirmed = SignalDatabase.backupMediaSnapshots.getMediaIdsConfirmedOnCdn(mediaIds, minimumLastSeenSnapshotVersion = 2)
|
||||
|
||||
assertThat(confirmed.size).isEqualTo(items.size)
|
||||
}
|
||||
|
||||
private fun getTotalItemCount(includeThumbnails: Boolean): Int {
|
||||
return if (includeThumbnails) {
|
||||
SignalDatabase.backupMediaSnapshots.readableDatabase
|
||||
|
||||
@@ -0,0 +1,289 @@
|
||||
/*
|
||||
* Copyright 2026 Signal Messenger, LLC
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
package org.thoughtcrime.securesms.jobs
|
||||
|
||||
import android.app.Application
|
||||
import assertk.assertThat
|
||||
import assertk.assertions.containsExactlyInAnyOrder
|
||||
import assertk.assertions.isEmpty
|
||||
import assertk.assertions.isEqualTo
|
||||
import io.mockk.Runs
|
||||
import io.mockk.every
|
||||
import io.mockk.just
|
||||
import io.mockk.mockkObject
|
||||
import io.mockk.unmockkObject
|
||||
import org.junit.After
|
||||
import org.junit.Before
|
||||
import org.junit.BeforeClass
|
||||
import org.junit.Rule
|
||||
import org.junit.Test
|
||||
import org.junit.runner.RunWith
|
||||
import org.robolectric.RobolectricTestRunner
|
||||
import org.robolectric.annotation.Config
|
||||
import org.signal.core.models.database.AttachmentId
|
||||
import org.signal.core.util.logging.Log
|
||||
import org.signal.core.util.readToList
|
||||
import org.signal.core.util.readToSingleInt
|
||||
import org.signal.core.util.requireLong
|
||||
import org.signal.core.util.select
|
||||
import org.signal.core.util.update
|
||||
import org.thoughtcrime.securesms.attachments.ArchivedAttachment
|
||||
import org.thoughtcrime.securesms.attachments.Attachment
|
||||
import org.thoughtcrime.securesms.backup.RestoreState
|
||||
import org.thoughtcrime.securesms.backup.v2.ArchiveRestoreProgress
|
||||
import org.thoughtcrime.securesms.database.AttachmentTable
|
||||
import org.thoughtcrime.securesms.database.MessageTable
|
||||
import org.thoughtcrime.securesms.database.MessageType
|
||||
import org.thoughtcrime.securesms.database.SignalDatabase
|
||||
import org.thoughtcrime.securesms.dependencies.AppDependencies
|
||||
import org.thoughtcrime.securesms.jobmanager.Job
|
||||
import org.thoughtcrime.securesms.mms.IncomingMessage
|
||||
import org.thoughtcrime.securesms.recipients.Recipient
|
||||
import org.thoughtcrime.securesms.testutil.RecipientTestRule
|
||||
import org.thoughtcrime.securesms.testutil.SystemOutLogger
|
||||
import java.util.UUID
|
||||
import kotlin.random.Random
|
||||
import kotlin.time.Duration.Companion.days
|
||||
|
||||
/**
|
||||
* Coverage for which attachments get a thumbnail restore enqueued. Asking the CDN for a thumbnail that was never generated is guaranteed to 404, so the
|
||||
* enqueue side has to agree with the upload side about what can have one.
|
||||
*/
|
||||
@RunWith(RobolectricTestRunner::class)
|
||||
@Config(manifest = Config.NONE, application = Application::class)
|
||||
class BackupRestoreMediaJobTest {
|
||||
|
||||
@get:Rule val recipients = RecipientTestRule()
|
||||
|
||||
companion object {
|
||||
@BeforeClass
|
||||
@JvmStatic
|
||||
fun setUpClass() {
|
||||
Log.initialize(SystemOutLogger())
|
||||
}
|
||||
}
|
||||
|
||||
private val enqueuedJobs = mutableListOf<Job>()
|
||||
|
||||
@Before
|
||||
fun setUp() {
|
||||
every { recipients.signalStore.backup.optimizeStorage } returns true
|
||||
|
||||
// ArchiveRestoreProgress reads these while initializing, and BackupValues is a strict mock, so they have to exist before the object is touched at all.
|
||||
every { recipients.signalStore.backup.restoreState } returns RestoreState.NONE
|
||||
every { recipients.signalStore.backup.totalRestorableAttachmentSize } returns 0L
|
||||
mockkObject(ArchiveRestoreProgress)
|
||||
every { ArchiveRestoreProgress.onProcessStart() } just Runs
|
||||
|
||||
every { AppDependencies.jobManager.add(capture(enqueuedJobs)) } returns Unit
|
||||
}
|
||||
|
||||
@After
|
||||
fun tearDown() {
|
||||
unmockkObject(ArchiveRestoreProgress)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun givenAnImage_whenIEnqueue_thenIExpectAThumbnailRestore() {
|
||||
val image = givenRestorableAttachment(contentType = "image/jpeg")
|
||||
|
||||
enqueue()
|
||||
|
||||
assertThat(thumbnailRestoreTargets()).containsExactlyInAnyOrder(image)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun givenAVideo_whenIEnqueue_thenIExpectAThumbnailRestore() {
|
||||
val video = givenRestorableAttachment(contentType = "video/mp4")
|
||||
|
||||
enqueue()
|
||||
|
||||
assertThat(thumbnailRestoreTargets()).containsExactlyInAnyOrder(video)
|
||||
}
|
||||
|
||||
/** A document never had a thumbnail generated, so the upload side never produced one to fetch. */
|
||||
@Test
|
||||
fun givenADocument_whenIEnqueue_thenIExpectNoThumbnailRestore() {
|
||||
givenRestorableAttachment(contentType = "application/pdf")
|
||||
|
||||
enqueue()
|
||||
|
||||
assertThat(thumbnailRestoreTargets()).isEmpty()
|
||||
}
|
||||
|
||||
/** Stickers are image types, so only the sticker id keeps them out of thumbnail work on the upload side. */
|
||||
@Test
|
||||
fun givenASticker_whenIEnqueue_thenIExpectNoThumbnailRestore() {
|
||||
val sticker = givenRestorableAttachment(contentType = "image/webp")
|
||||
markAsSticker(sticker)
|
||||
|
||||
enqueue()
|
||||
|
||||
assertThat(thumbnailRestoreTargets()).isEmpty()
|
||||
}
|
||||
|
||||
@Test
|
||||
fun givenAnSvg_whenIEnqueue_thenIExpectNoThumbnailRestore() {
|
||||
givenRestorableAttachment(contentType = "image/svg+xml")
|
||||
|
||||
enqueue()
|
||||
|
||||
assertThat(thumbnailRestoreTargets()).isEmpty()
|
||||
}
|
||||
|
||||
@Test
|
||||
fun givenAQuote_whenIEnqueue_thenIExpectNoThumbnailRestore() {
|
||||
val quote = givenRestorableAttachment(contentType = "image/jpeg")
|
||||
markAsQuote(quote)
|
||||
|
||||
enqueue()
|
||||
|
||||
assertThat(thumbnailRestoreTargets()).isEmpty()
|
||||
}
|
||||
|
||||
/** Skipping the job must not skip the state change, or the row would be re-selected by every later batch. */
|
||||
@Test
|
||||
fun givenAnIneligibleAttachment_whenIEnqueue_thenIExpectItStillOffloaded() {
|
||||
val document = givenRestorableAttachment(contentType = "application/pdf")
|
||||
|
||||
enqueue()
|
||||
|
||||
assertThat(transferStateOf(document)).isEqualTo(AttachmentTable.TRANSFER_RESTORE_OFFLOADED)
|
||||
}
|
||||
|
||||
/**
|
||||
* The batch that drains no jobs is the regression this guards. With a page size of 1 and three ineligible attachments, a loop keyed off the jobs it created
|
||||
* would stop after the first batch and silently leave the rest of the restore undone.
|
||||
*/
|
||||
@Test
|
||||
fun givenMoreIneligibleAttachmentsThanOnePage_whenIEnqueue_thenIExpectEveryRowDrained() {
|
||||
repeat(3) { givenRestorableAttachment(contentType = "application/pdf") }
|
||||
|
||||
enqueue(batchSize = 1)
|
||||
|
||||
assertThat(needsRestoreCount()).isEqualTo(0)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun givenAMixOfEligibleAndNot_whenIEnqueue_thenIExpectOnlyTheEligibleOnesRequested() {
|
||||
val image = givenRestorableAttachment(contentType = "image/jpeg")
|
||||
givenRestorableAttachment(contentType = "application/pdf")
|
||||
val video = givenRestorableAttachment(contentType = "video/mp4")
|
||||
givenRestorableAttachment(contentType = "audio/mpeg")
|
||||
|
||||
enqueue(batchSize = 2)
|
||||
|
||||
assertThat(thumbnailRestoreTargets()).containsExactlyInAnyOrder(image, video)
|
||||
assertThat(needsRestoreCount()).isEqualTo(0)
|
||||
}
|
||||
|
||||
private fun enqueue(batchSize: Int = 500) {
|
||||
BackupRestoreMediaJob().enqueueRestoreJobs(restoreTime = System.currentTimeMillis(), batchSize = batchSize)
|
||||
}
|
||||
|
||||
private fun thumbnailRestoreTargets(): List<AttachmentId> {
|
||||
return enqueuedJobs.filterIsInstance<RestoreAttachmentThumbnailJob>().map { it.attachmentId }
|
||||
}
|
||||
|
||||
/**
|
||||
* Inserts an attachment that needs restoring on a message old enough that optimize storage routes it down the thumbnail-only branch rather than a full-size
|
||||
* restore.
|
||||
*/
|
||||
private fun givenRestorableAttachment(contentType: String): AttachmentId {
|
||||
val from = recipients.createRecipient("Contact ${UUID.randomUUID()}")
|
||||
val threadId = SignalDatabase.threads.getOrCreateThreadIdFor(Recipient.resolved(from))
|
||||
|
||||
val message = IncomingMessage(
|
||||
type = MessageType.NORMAL,
|
||||
from = from,
|
||||
body = null,
|
||||
sentTimeMillis = 100L,
|
||||
serverTimeMillis = 100L,
|
||||
receivedTimeMillis = 200L,
|
||||
attachments = listOf(createAttachment(contentType))
|
||||
)
|
||||
|
||||
val messageId = SignalDatabase.messages.insertMessageInbox(message, threadId).get().messageId
|
||||
val attachmentId = SignalDatabase.attachments.getAttachmentsForMessage(messageId).first().attachmentId
|
||||
|
||||
SignalDatabase.messages.writableDatabase
|
||||
.update(MessageTable.TABLE_NAME)
|
||||
.values(MessageTable.DATE_RECEIVED to System.currentTimeMillis() - 60.days.inWholeMilliseconds)
|
||||
.where("${MessageTable.ID} = ?", messageId)
|
||||
.run()
|
||||
|
||||
SignalDatabase.attachments.writableDatabase
|
||||
.update(AttachmentTable.TABLE_NAME)
|
||||
.values(AttachmentTable.TRANSFER_STATE to AttachmentTable.TRANSFER_NEEDS_RESTORE)
|
||||
.where("${AttachmentTable.ID} = ?", attachmentId.id)
|
||||
.run()
|
||||
|
||||
return attachmentId
|
||||
}
|
||||
|
||||
private fun markAsSticker(attachmentId: AttachmentId) {
|
||||
SignalDatabase.attachments.writableDatabase
|
||||
.update(AttachmentTable.TABLE_NAME)
|
||||
.values(AttachmentTable.STICKER_ID to 0)
|
||||
.where("${AttachmentTable.ID} = ?", attachmentId.id)
|
||||
.run()
|
||||
}
|
||||
|
||||
private fun markAsQuote(attachmentId: AttachmentId) {
|
||||
SignalDatabase.attachments.writableDatabase
|
||||
.update(AttachmentTable.TABLE_NAME)
|
||||
.values(AttachmentTable.QUOTE to 1)
|
||||
.where("${AttachmentTable.ID} = ?", attachmentId.id)
|
||||
.run()
|
||||
}
|
||||
|
||||
private fun transferStateOf(attachmentId: AttachmentId): Int {
|
||||
return SignalDatabase.attachments.readableDatabase
|
||||
.select(AttachmentTable.TRANSFER_STATE)
|
||||
.from(AttachmentTable.TABLE_NAME)
|
||||
.where("${AttachmentTable.ID} = ?", attachmentId.id)
|
||||
.run()
|
||||
.readToSingleInt(-1)
|
||||
}
|
||||
|
||||
private fun needsRestoreCount(): Int {
|
||||
return SignalDatabase.attachments.readableDatabase
|
||||
.select(AttachmentTable.ID)
|
||||
.from(AttachmentTable.TABLE_NAME)
|
||||
.where("${AttachmentTable.TRANSFER_STATE} = ?", AttachmentTable.TRANSFER_NEEDS_RESTORE)
|
||||
.run()
|
||||
.readToList { it.requireLong(AttachmentTable.ID) }
|
||||
.size
|
||||
}
|
||||
|
||||
private fun createAttachment(contentType: String): Attachment {
|
||||
return ArchivedAttachment(
|
||||
contentType = contentType,
|
||||
size = 1024,
|
||||
cdn = 3,
|
||||
uploadTimestamp = 0,
|
||||
key = Random.nextBytes(32),
|
||||
cdnKey = "password",
|
||||
archiveCdn = 3,
|
||||
plaintextHash = Random.nextBytes(32),
|
||||
incrementalMac = null,
|
||||
incrementalMacChunkSize = null,
|
||||
width = 0,
|
||||
height = 0,
|
||||
caption = null,
|
||||
blurHash = null,
|
||||
voiceNote = false,
|
||||
borderless = false,
|
||||
stickerLocator = null,
|
||||
gif = false,
|
||||
quote = false,
|
||||
quoteTargetContentType = null,
|
||||
uuid = UUID.randomUUID(),
|
||||
fileName = null,
|
||||
localBackupKey = null
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,379 @@
|
||||
/*
|
||||
* Copyright 2026 Signal Messenger, LLC
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
package org.thoughtcrime.securesms.jobs
|
||||
|
||||
import android.app.Application
|
||||
import assertk.assertThat
|
||||
import assertk.assertions.isEqualTo
|
||||
import assertk.assertions.isNotNull
|
||||
import assertk.assertions.isNull
|
||||
import io.mockk.every
|
||||
import io.mockk.verify
|
||||
import org.junit.Before
|
||||
import org.junit.BeforeClass
|
||||
import org.junit.Rule
|
||||
import org.junit.Test
|
||||
import org.junit.runner.RunWith
|
||||
import org.robolectric.RobolectricTestRunner
|
||||
import org.robolectric.annotation.Config
|
||||
import org.signal.core.models.backup.MediaName
|
||||
import org.signal.core.models.backup.MediaRootBackupKey
|
||||
import org.signal.core.models.database.AttachmentId
|
||||
import org.signal.core.util.Base64
|
||||
import org.signal.core.util.logging.Log
|
||||
import org.signal.core.util.readToSingleInt
|
||||
import org.signal.core.util.readToSingleObject
|
||||
import org.signal.core.util.requireString
|
||||
import org.signal.core.util.select
|
||||
import org.signal.core.util.update
|
||||
import org.thoughtcrime.securesms.attachments.ArchivedAttachment
|
||||
import org.thoughtcrime.securesms.attachments.Attachment
|
||||
import org.thoughtcrime.securesms.database.AttachmentTable
|
||||
import org.thoughtcrime.securesms.database.BackupMediaSnapshotTable.MediaEntry
|
||||
import org.thoughtcrime.securesms.database.MessageTable
|
||||
import org.thoughtcrime.securesms.database.MessageType
|
||||
import org.thoughtcrime.securesms.database.SignalDatabase
|
||||
import org.thoughtcrime.securesms.dependencies.AppDependencies
|
||||
import org.thoughtcrime.securesms.mms.IncomingMessage
|
||||
import org.thoughtcrime.securesms.recipients.Recipient
|
||||
import org.thoughtcrime.securesms.testutil.RecipientTestRule
|
||||
import org.thoughtcrime.securesms.testutil.SystemOutLogger
|
||||
import java.util.UUID
|
||||
import kotlin.random.Random
|
||||
import kotlin.time.Duration
|
||||
import kotlin.time.Duration.Companion.days
|
||||
import kotlin.time.Duration.Companion.hours
|
||||
|
||||
/**
|
||||
* Coverage for the gate that decides whether a local copy may be deleted. The local copy is usually the only copy, so every refusal here is the difference
|
||||
* between a recoverable mistake and permanent data loss.
|
||||
*/
|
||||
@RunWith(RobolectricTestRunner::class)
|
||||
@Config(manifest = Config.NONE, application = Application::class)
|
||||
class OptimizeMediaJobTest {
|
||||
|
||||
@get:Rule val recipients = RecipientTestRule()
|
||||
|
||||
companion object {
|
||||
private const val NO_COMPLETED_CRAWL = -1L
|
||||
private val RECONCILIATION_INTERVAL = 7.days
|
||||
private val MEDIA_ROOT_BACKUP_KEY = MediaRootBackupKey(Random.nextBytes(32))
|
||||
private val PLAINTEXT_HASH = Random.nextBytes(32)
|
||||
private val REMOTE_KEY = Random.nextBytes(32)
|
||||
|
||||
@BeforeClass
|
||||
@JvmStatic
|
||||
fun setUpClass() {
|
||||
Log.initialize(SystemOutLogger())
|
||||
}
|
||||
}
|
||||
|
||||
@Before
|
||||
fun setUp() {
|
||||
every { recipients.signalStore.backup.mediaRootBackupKey } returns MEDIA_ROOT_BACKUP_KEY
|
||||
}
|
||||
|
||||
@Test
|
||||
fun givenNoCrawlHasEverCompleted_whenIOffload_thenIExpectNothingOffloaded() {
|
||||
val attachmentId = givenAnEligibleAttachmentConfirmedOnTheCdn()
|
||||
givenCompletedCrawl(snapshotVersion = NO_COMPLETED_CRAWL, age = 1.days)
|
||||
|
||||
offload()
|
||||
|
||||
assertNotOffloaded(attachmentId)
|
||||
}
|
||||
|
||||
/**
|
||||
* Refusing is safe but permanent on its own, since only a completed crawl can produce the evidence that lifts the refusal. Without this the feature would
|
||||
* appear to do nothing until the periodic crawl happened to run.
|
||||
*/
|
||||
@Test
|
||||
fun givenNoCrawlHasEverCompleted_whenIOffload_thenIExpectACrawlRequested() {
|
||||
givenAnEligibleAttachmentConfirmedOnTheCdn()
|
||||
givenCompletedCrawl(snapshotVersion = NO_COMPLETED_CRAWL, age = 1.days)
|
||||
|
||||
offload()
|
||||
|
||||
assertACrawlWasRequested()
|
||||
}
|
||||
|
||||
@Test
|
||||
fun givenFreshEvidence_whenIOffload_thenIExpectNoCrawlRequested() {
|
||||
givenAnEligibleAttachmentConfirmedOnTheCdn()
|
||||
givenCompletedCrawl(snapshotVersion = 1, age = 1.days)
|
||||
|
||||
offload()
|
||||
|
||||
assertNoCrawlWasRequested()
|
||||
}
|
||||
|
||||
/** The caller runs after every backup, so without a rate limit an unverified device would force a full walk of the server listing daily. */
|
||||
@Test
|
||||
fun givenACrawlWasForcedRecently_whenIOffload_thenIExpectNoCrawlRequested() {
|
||||
givenAnEligibleAttachmentConfirmedOnTheCdn()
|
||||
givenCompletedCrawl(snapshotVersion = NO_COMPLETED_CRAWL, age = 1.days)
|
||||
givenLastForcedAttempt(age = 1.days)
|
||||
|
||||
offload()
|
||||
|
||||
assertNoCrawlWasRequested()
|
||||
}
|
||||
|
||||
@Test
|
||||
fun givenTheLastForcedCrawlIsOlderThanTheInterval_whenIOffload_thenIExpectACrawlRequested() {
|
||||
givenAnEligibleAttachmentConfirmedOnTheCdn()
|
||||
givenCompletedCrawl(snapshotVersion = NO_COMPLETED_CRAWL, age = 1.days)
|
||||
givenLastForcedAttempt(age = RECONCILIATION_INTERVAL * 2)
|
||||
|
||||
offload()
|
||||
|
||||
assertACrawlWasRequested()
|
||||
}
|
||||
|
||||
/** Media can disappear from the CDN after a crawl confirmed it, so a confirmation has a shelf life even though it was genuine when recorded. */
|
||||
@Test
|
||||
fun givenTheEvidenceIsTooOld_whenIOffload_thenIExpectNothingOffloaded() {
|
||||
val attachmentId = givenAnEligibleAttachmentConfirmedOnTheCdn()
|
||||
givenCompletedCrawl(snapshotVersion = 1, age = RECONCILIATION_INTERVAL * 5)
|
||||
|
||||
offload()
|
||||
|
||||
assertNotOffloaded(attachmentId)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun givenTheEvidenceIsTooOld_whenIOffload_thenIExpectACrawlRequested() {
|
||||
givenAnEligibleAttachmentConfirmedOnTheCdn()
|
||||
givenCompletedCrawl(snapshotVersion = 1, age = RECONCILIATION_INTERVAL * 5)
|
||||
|
||||
offload()
|
||||
|
||||
assertACrawlWasRequested()
|
||||
}
|
||||
|
||||
/**
|
||||
* A remotely configured interval of zero parses successfully rather than falling back to the default, so without a floor it would make every confirmation
|
||||
* instantly stale and stop offloading on every device at once.
|
||||
*/
|
||||
@Test
|
||||
fun givenTheConfiguredIntervalIsZero_whenIOffload_thenIExpectItOffloadedAnyway() {
|
||||
val attachmentId = givenAnEligibleAttachmentConfirmedOnTheCdn()
|
||||
givenCompletedCrawl(snapshotVersion = 1, age = 1.days)
|
||||
|
||||
OptimizeMediaJob().offloadVerifiedMedia(minimumAge = 30.days, reconciliationInterval = Duration.ZERO)
|
||||
|
||||
assertThat(transferStateOf(attachmentId)).isEqualTo(AttachmentTable.TRANSFER_RESTORE_OFFLOADED)
|
||||
}
|
||||
|
||||
/** The same floor has to reach the rate limit, or a zero interval would force a full walk of the server listing after every backup. */
|
||||
@Test
|
||||
fun givenTheConfiguredIntervalIsZeroAndACrawlWasJustForced_whenIOffload_thenIExpectNoCrawlRequested() {
|
||||
givenAnEligibleAttachmentConfirmedOnTheCdn()
|
||||
givenCompletedCrawl(snapshotVersion = NO_COMPLETED_CRAWL, age = 1.days)
|
||||
givenLastForcedAttempt(age = 1.hours)
|
||||
|
||||
OptimizeMediaJob().offloadVerifiedMedia(minimumAge = 30.days, reconciliationInterval = Duration.ZERO)
|
||||
|
||||
assertNoCrawlWasRequested()
|
||||
}
|
||||
|
||||
/** A clock that moved backwards makes the evidence age negative, which has to read as untrustworthy rather than as brand new. */
|
||||
@Test
|
||||
fun givenTheClockRolledBack_whenIOffload_thenIExpectNothingOffloaded() {
|
||||
val attachmentId = givenAnEligibleAttachmentConfirmedOnTheCdn()
|
||||
givenCompletedCrawl(snapshotVersion = 1, age = -(30.days))
|
||||
|
||||
offload()
|
||||
|
||||
assertNotOffloaded(attachmentId)
|
||||
}
|
||||
|
||||
/** Refusing on its own would leave a far-future timestamp gating offloading until the clock caught up to it, so the evidence has to be discarded outright. */
|
||||
@Test
|
||||
fun givenTheClockRolledBack_whenIOffload_thenIExpectTheVerificationStateCleared() {
|
||||
givenAnEligibleAttachmentConfirmedOnTheCdn()
|
||||
givenCompletedCrawl(snapshotVersion = 1, age = -(30.days))
|
||||
|
||||
offload()
|
||||
|
||||
val backup = recipients.signalStore.backup
|
||||
verify { backup.clearArchiveVerificationState() }
|
||||
}
|
||||
|
||||
@Test
|
||||
fun givenTheClockRolledBack_whenIOffload_thenIExpectACrawlRequested() {
|
||||
givenAnEligibleAttachmentConfirmedOnTheCdn()
|
||||
givenCompletedCrawl(snapshotVersion = 1, age = -(30.days))
|
||||
|
||||
offload()
|
||||
|
||||
assertACrawlWasRequested()
|
||||
}
|
||||
|
||||
@Test
|
||||
fun givenFreshEvidenceForTheMedia_whenIOffload_thenIExpectItOffloaded() {
|
||||
val attachmentId = givenAnEligibleAttachmentConfirmedOnTheCdn()
|
||||
givenCompletedCrawl(snapshotVersion = 1, age = 1.days)
|
||||
|
||||
offload()
|
||||
|
||||
assertThat(transferStateOf(attachmentId)).isEqualTo(AttachmentTable.TRANSFER_RESTORE_OFFLOADED)
|
||||
assertThat(dataFileOf(attachmentId)).isNull()
|
||||
}
|
||||
|
||||
/** A snapshot row on its own is local bookkeeping. Only the crawl marking it seen makes it evidence the server produced. */
|
||||
@Test
|
||||
fun givenTheMediaWasNeverSeenOnTheCdn_whenIOffload_thenIExpectNothingOffloaded() {
|
||||
val attachmentId = givenAnEligibleAttachment()
|
||||
commitSnapshotFor(PLAINTEXT_HASH, REMOTE_KEY, markSeen = false)
|
||||
givenCompletedCrawl(snapshotVersion = 1, age = 1.days)
|
||||
|
||||
offload()
|
||||
|
||||
assertNotOffloaded(attachmentId)
|
||||
}
|
||||
|
||||
private fun offload() {
|
||||
OptimizeMediaJob().offloadVerifiedMedia(minimumAge = 30.days, reconciliationInterval = RECONCILIATION_INTERVAL)
|
||||
}
|
||||
|
||||
private fun givenCompletedCrawl(snapshotVersion: Long, age: Duration) {
|
||||
every { recipients.signalStore.backup.lastCompletedReconciliationSnapshotVersion } returns snapshotVersion
|
||||
every { recipients.signalStore.backup.lastCompletedReconciliationTime } returns System.currentTimeMillis() - age.inWholeMilliseconds
|
||||
givenLastForcedAttempt(age = null)
|
||||
}
|
||||
|
||||
/** A null [age] means no crawl has ever been forced, which is what a device that just enabled this looks like. */
|
||||
private fun givenLastForcedAttempt(age: Duration?) {
|
||||
val timestamp = if (age == null) 0 else System.currentTimeMillis() - age.inWholeMilliseconds
|
||||
every { recipients.signalStore.backup.lastForcedReconciliationAttemptTime } returns timestamp
|
||||
}
|
||||
|
||||
private fun givenAnEligibleAttachmentConfirmedOnTheCdn(): AttachmentId {
|
||||
val attachmentId = givenAnEligibleAttachment()
|
||||
commitSnapshotFor(PLAINTEXT_HASH, REMOTE_KEY, markSeen = true)
|
||||
return attachmentId
|
||||
}
|
||||
|
||||
/** The offload path derives the media id from the hash and key, so the snapshot row has to carry the same derived value or it can never match. */
|
||||
private fun commitSnapshotFor(plaintextHash: ByteArray, remoteKey: ByteArray, markSeen: Boolean) {
|
||||
val mediaId = MediaName.fromPlaintextHashAndRemoteKey(plaintextHash, remoteKey).toMediaId(MEDIA_ROOT_BACKUP_KEY).encode()
|
||||
|
||||
SignalDatabase.backupMediaSnapshots.writePendingMediaEntries(
|
||||
listOf(MediaEntry(mediaId = mediaId, cdn = 3, plaintextHash = plaintextHash, remoteKey = remoteKey, isThumbnail = false))
|
||||
)
|
||||
SignalDatabase.backupMediaSnapshots.commitPendingRows()
|
||||
|
||||
if (markSeen) {
|
||||
SignalDatabase.backupMediaSnapshots.markSeenOnRemote(listOf(mediaId), 1)
|
||||
}
|
||||
}
|
||||
|
||||
private fun givenAnEligibleAttachment(): AttachmentId {
|
||||
val from = recipients.createRecipient("Some Contact")
|
||||
val threadId = SignalDatabase.threads.getOrCreateThreadIdFor(Recipient.resolved(from))
|
||||
|
||||
val message = IncomingMessage(
|
||||
type = MessageType.NORMAL,
|
||||
from = from,
|
||||
body = null,
|
||||
sentTimeMillis = 100L,
|
||||
serverTimeMillis = 100L,
|
||||
receivedTimeMillis = 200L,
|
||||
attachments = listOf(createAttachment())
|
||||
)
|
||||
|
||||
val messageId = SignalDatabase.messages.insertMessageInbox(message, threadId).get().messageId
|
||||
val attachmentId = SignalDatabase.attachments.getAttachmentsForMessage(messageId).first().attachmentId
|
||||
|
||||
SignalDatabase.messages.writableDatabase
|
||||
.update(MessageTable.TABLE_NAME)
|
||||
.values(MessageTable.DATE_RECEIVED to System.currentTimeMillis() - 60.days.inWholeMilliseconds)
|
||||
.where("${MessageTable.ID} = ?", messageId)
|
||||
.run()
|
||||
|
||||
SignalDatabase.attachments.writableDatabase
|
||||
.update(AttachmentTable.TABLE_NAME)
|
||||
.values(
|
||||
AttachmentTable.TRANSFER_STATE to AttachmentTable.TRANSFER_PROGRESS_DONE,
|
||||
AttachmentTable.ARCHIVE_TRANSFER_STATE to AttachmentTable.ArchiveTransferState.FINISHED.value,
|
||||
AttachmentTable.DATA_FILE to "/not/a/real/file/${attachmentId.id}",
|
||||
AttachmentTable.DATA_HASH_END to Base64.encodeWithPadding(PLAINTEXT_HASH),
|
||||
AttachmentTable.REMOTE_KEY to Base64.encodeWithPadding(REMOTE_KEY),
|
||||
AttachmentTable.OFFLOAD_RESTORED_AT to 0
|
||||
)
|
||||
.where("${AttachmentTable.ID} = ?", attachmentId.id)
|
||||
.run()
|
||||
|
||||
return attachmentId
|
||||
}
|
||||
|
||||
/** Deliberately not an image or video, so eligibility does not additionally depend on a local thumbnail existing. */
|
||||
private fun createAttachment(): Attachment {
|
||||
return ArchivedAttachment(
|
||||
contentType = "application/pdf",
|
||||
size = 1024,
|
||||
cdn = 3,
|
||||
uploadTimestamp = 0,
|
||||
key = Random.nextBytes(32),
|
||||
cdnKey = "password",
|
||||
archiveCdn = 3,
|
||||
plaintextHash = Random.nextBytes(32),
|
||||
incrementalMac = null,
|
||||
incrementalMacChunkSize = null,
|
||||
width = 0,
|
||||
height = 0,
|
||||
caption = null,
|
||||
blurHash = null,
|
||||
voiceNote = false,
|
||||
borderless = false,
|
||||
stickerLocator = null,
|
||||
gif = false,
|
||||
quote = false,
|
||||
quoteTargetContentType = null,
|
||||
uuid = UUID.randomUUID(),
|
||||
fileName = null,
|
||||
localBackupKey = null
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* The job manager is hoisted out of the verify block deliberately. [AppDependencies] is statically mocked, so referencing it inside the block would record
|
||||
* the accessor itself as a call to verify.
|
||||
*/
|
||||
private fun assertACrawlWasRequested() {
|
||||
val jobManager = AppDependencies.jobManager
|
||||
verify { jobManager.add(ofType<ArchiveAttachmentReconciliationJob>()) }
|
||||
}
|
||||
|
||||
private fun assertNoCrawlWasRequested() {
|
||||
val jobManager = AppDependencies.jobManager
|
||||
verify(exactly = 0) { jobManager.add(ofType<ArchiveAttachmentReconciliationJob>()) }
|
||||
}
|
||||
|
||||
private fun assertNotOffloaded(attachmentId: AttachmentId) {
|
||||
assertThat(transferStateOf(attachmentId)).isEqualTo(AttachmentTable.TRANSFER_PROGRESS_DONE)
|
||||
assertThat(dataFileOf(attachmentId)).isNotNull()
|
||||
}
|
||||
|
||||
private fun transferStateOf(attachmentId: AttachmentId): Int {
|
||||
return SignalDatabase.attachments.readableDatabase
|
||||
.select(AttachmentTable.TRANSFER_STATE)
|
||||
.from(AttachmentTable.TABLE_NAME)
|
||||
.where("${AttachmentTable.ID} = ?", attachmentId.id)
|
||||
.run()
|
||||
.readToSingleInt(-1)
|
||||
}
|
||||
|
||||
private fun dataFileOf(attachmentId: AttachmentId): String? {
|
||||
return SignalDatabase.attachments.readableDatabase
|
||||
.select(AttachmentTable.DATA_FILE)
|
||||
.from(AttachmentTable.TABLE_NAME)
|
||||
.where("${AttachmentTable.ID} = ?", attachmentId.id)
|
||||
.run()
|
||||
.readToSingleObject { it.requireString(AttachmentTable.DATA_FILE) }
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user