Upload media restored from local backups to the archive CDN.

This commit is contained in:
Alex Hart
2026-07-07 16:08:51 -03:00
committed by GitHub
parent e687718ec3
commit 384344c91b
15 changed files with 747 additions and 9 deletions
@@ -11,6 +11,7 @@ import assertk.assertions.isEmpty
import assertk.assertions.isEqualTo
import assertk.assertions.isNotEmpty
import assertk.assertions.isNotEqualTo
import assertk.assertions.isNull
import assertk.assertions.isTrue
import org.junit.Assert.assertEquals
import org.junit.Assert.assertNotEquals
@@ -219,6 +220,26 @@ class AttachmentTableTest {
assertThat(SignalDatabase.attachments.getAttachment(attachmentId)!!.archiveTransferState).isEqualTo(AttachmentTable.ArchiveTransferState.NONE)
}
@Test
fun resetArchiveTransferStateForLocalBackupMedia_onlyResetsLocalBackupMedia() {
// Given one archive-finished attachment restored from a local backup, and one that wasn't
val localBackupMessageId = SignalDatabase.messages.insertMessageInbox(createIncomingMessage(serverTime = 0.days, attachment = createArchivedAttachment(localBackupKey = Random.nextBytes(32)))).map { it.messageId }.get()
val localBackupAttachmentId = SignalDatabase.attachments.getAttachmentsForMessage(localBackupMessageId).first().attachmentId
val nonLocalBackupMessageId = SignalDatabase.messages.insertMessageInbox(createIncomingMessage(serverTime = 1.days, attachment = createArchivedAttachment())).map { it.messageId }.get()
val nonLocalBackupAttachmentId = SignalDatabase.attachments.getAttachmentsForMessage(nonLocalBackupMessageId).first().attachmentId
SignalDatabase.attachments.setArchiveTransferState(localBackupAttachmentId, AttachmentTable.ArchiveTransferState.FINISHED)
SignalDatabase.attachments.setArchiveTransferState(nonLocalBackupAttachmentId, AttachmentTable.ArchiveTransferState.FINISHED)
val resetCount = SignalDatabase.attachments.resetArchiveTransferStateForLocalBackupMedia()
// Only the local-backup attachment is reset
assertThat(resetCount).isEqualTo(1)
assertThat(SignalDatabase.attachments.getAttachment(localBackupAttachmentId)!!.archiveTransferState).isEqualTo(AttachmentTable.ArchiveTransferState.NONE)
assertThat(SignalDatabase.attachments.getAttachment(nonLocalBackupAttachmentId)!!.archiveTransferState).isEqualTo(AttachmentTable.ArchiveTransferState.FINISHED)
}
@Test
fun given10NewerAnd10OlderAttachments_whenIGetEachBatch_thenIExpectProperBucketing() {
val now = System.currentTimeMillis().milliseconds
@@ -418,6 +439,39 @@ class AttachmentTableTest {
assertThat(dbAttachment2.archiveTransferState).isEqualTo(AttachmentTable.ArchiveTransferState.FINISHED)
}
@Test
fun givenLocalBackupRestore_whenIFinalizeAttachment_thenIExpectArchiveStateNoneSoItGetsUploaded() {
val data = byteArrayOf(1, 2, 3, 4, 5)
val attachment = createAttachmentPointer("remote-key-1".toByteArray(), data.size)
val messageResult = SignalDatabase.messages.insertMessageInbox(createIncomingMessage(serverTime = 0.days, attachment = attachment)).get()
val attachmentId = messageResult.insertedAttachments!![attachment]!!
SignalDatabase.attachments.setTransferState(messageResult.messageId, attachmentId, AttachmentTable.TRANSFER_PROGRESS_STARTED)
// Data is restored from a local backup file, not the archive CDN
SignalDatabase.attachments.finalizeAttachmentAfterDownload(messageResult.messageId, attachmentId, ByteArrayInputStream(data), archiveRestore = true, restoredFromArchiveCdn = false)
val result = SignalDatabase.attachments.getAttachment(attachmentId)!!
assertThat(result.archiveTransferState).isEqualTo(AttachmentTable.ArchiveTransferState.NONE)
assertThat(result.archiveCdn).isNull()
}
@Test
fun givenArchiveCdnRestore_whenIFinalizeAttachment_thenIExpectArchiveStateFinished() {
val data = byteArrayOf(1, 2, 3, 4, 5)
val attachment = createAttachmentPointer("remote-key-1".toByteArray(), data.size)
val messageResult = SignalDatabase.messages.insertMessageInbox(createIncomingMessage(serverTime = 0.days, attachment = attachment)).get()
val attachmentId = messageResult.insertedAttachments!![attachment]!!
SignalDatabase.attachments.setTransferState(messageResult.messageId, attachmentId, AttachmentTable.TRANSFER_PROGRESS_STARTED)
// Data is restored directly from the archive CDN
SignalDatabase.attachments.finalizeAttachmentAfterDownload(messageResult.messageId, attachmentId, ByteArrayInputStream(data), archiveRestore = true, restoredFromArchiveCdn = true)
val result = SignalDatabase.attachments.getAttachment(attachmentId)!!
assertThat(result.archiveTransferState).isEqualTo(AttachmentTable.ArchiveTransferState.FINISHED)
}
@Test
fun givenAttachmentsWithMatchingMediaId_whenISetArchiveFinishedForMatchingMediaObjects_thenIExpectThoseAttachmentsToBeMarkedFinished() {
// GIVEN
@@ -585,7 +639,7 @@ class AttachmentTableTest {
).get()
}
private fun createArchivedAttachment(): Attachment {
private fun createArchivedAttachment(localBackupKey: ByteArray? = null): Attachment {
return ArchivedAttachment(
contentType = "image/jpeg",
size = 1024,
@@ -609,7 +663,7 @@ class AttachmentTableTest {
quoteTargetContentType = null,
uuid = UUID.randomUUID(),
fileName = null,
localBackupKey = null
localBackupKey = localBackupKey
)
}
@@ -0,0 +1,279 @@
/*
* Copyright 2026 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.thoughtcrime.securesms.jobs
import androidx.test.ext.junit.runners.AndroidJUnit4
import assertk.assertThat
import assertk.assertions.isEqualTo
import assertk.assertions.isFalse
import assertk.assertions.isNull
import io.mockk.Runs
import io.mockk.every
import io.mockk.just
import io.mockk.mockkObject
import io.mockk.unmockkAll
import io.mockk.verify
import org.junit.After
import org.junit.Before
import org.junit.Rule
import org.junit.Test
import org.junit.runner.RunWith
import org.signal.core.models.backup.MediaName
import org.signal.core.models.database.AttachmentId
import org.signal.core.util.Base64.decodeBase64OrThrow
import org.signal.network.NetworkResult
import org.thoughtcrime.securesms.attachments.Attachment
import org.thoughtcrime.securesms.attachments.PointerAttachment
import org.thoughtcrime.securesms.backup.v2.BackupRepository
import org.thoughtcrime.securesms.backup.v2.MessageBackupTier
import org.thoughtcrime.securesms.database.AttachmentTable
import org.thoughtcrime.securesms.database.BackupMediaSnapshotTable.MediaEntry
import org.thoughtcrime.securesms.database.MessageType
import org.thoughtcrime.securesms.database.SignalDatabase
import org.thoughtcrime.securesms.keyvalue.SignalStore
import org.thoughtcrime.securesms.mms.IncomingMessage
import org.thoughtcrime.securesms.testing.SignalActivityRule
import org.thoughtcrime.securesms.util.MediaUtil
import org.whispersystems.signalservice.api.archive.ArchiveGetMediaItemsResponse
import org.whispersystems.signalservice.api.archive.ArchiveGetMediaItemsResponse.StoredMediaObject
import org.whispersystems.signalservice.api.messages.SignalServiceAttachmentPointer
import org.whispersystems.signalservice.api.messages.SignalServiceAttachmentRemoteId
import java.io.ByteArrayInputStream
import java.util.Optional
import kotlin.time.Duration
import kotlin.time.Duration.Companion.days
@RunWith(AndroidJUnit4::class)
class ArchiveAttachmentReconciliationJobTest {
@get:Rule
val harness = SignalActivityRule()
@Before
fun setUp() {
SignalStore.backup.backupTier = MessageBackupTier.PAID
SignalStore.backup.hasBackupBeenUploaded = true
SignalStore.backup.lastAttachmentReconciliationTime = System.currentTimeMillis()
SignalStore.backup.localRestoreReconcilePending = false
mockkObject(BackupRepository)
mockkObject(ArchiveCommitAttachmentDeletesJob)
every { ArchiveCommitAttachmentDeletesJob.deleteMediaObjectsFromCdn(any(), any(), any(), any()) } returns null
}
@After
fun tearDown() {
unmockkAll()
}
/**
* The core of the reconcile-first restore flow: a local restore resets everything to [AttachmentTable.ArchiveTransferState.NONE], so media that genuinely is
* on the CDN must be promoted back to FINISHED during reconciliation -- otherwise the backfill would needlessly re-upload it. This only happens while
* [localRestoreReconcilePending] is set, so it never runs in the common periodic reconciliation.
*/
@Test
fun givenLocalRestorePendingAndAttachmentOnCdn_whenIReconcile_thenIExpectItMarkedFinished() {
SignalStore.backup.localRestoreReconcilePending = true
val attachmentId = seedFinalizedAttachment("remote-key-1".toByteArray(), byteArrayOf(1, 2, 3, 4, 5))
commitSnapshotFor(attachmentId, cdn = 3)
fakeCdnContains(attachmentId, cdn = 3)
ArchiveAttachmentReconciliationJob(forced = true).run()
assertThat(SignalDatabase.attachments.getAttachment(attachmentId)!!.archiveTransferState).isEqualTo(AttachmentTable.ArchiveTransferState.FINISHED)
}
/**
* Guards the reconcile-first promotion above: outside the local-restore flow (the common periodic reconciliation), NONE media that happens to be on the CDN is
* left alone, so we don't do the expensive mark-finished scan in the common case.
*/
@Test
fun givenNoLocalRestorePendingAndNoneAttachmentOnCdn_whenIReconcile_thenItStaysNone() {
val attachmentId = seedFinalizedAttachment("remote-key-common".toByteArray(), byteArrayOf(2, 3, 4, 5, 6))
commitSnapshotFor(attachmentId, cdn = 3)
fakeCdnContains(attachmentId, cdn = 3)
ArchiveAttachmentReconciliationJob(forced = true).run()
assertThat(SignalDatabase.attachments.getAttachment(attachmentId)!!.archiveTransferState).isEqualTo(AttachmentTable.ArchiveTransferState.NONE)
}
@Test
fun givenFinishedAttachmentMissingFromCdn_whenIReconcile_thenIExpectItResetToNone() {
val attachmentId = seedFinalizedAttachment("remote-key-2".toByteArray(), byteArrayOf(6, 7, 8, 9, 10))
SignalDatabase.attachments.setArchiveTransferState(attachmentId, AttachmentTable.ArchiveTransferState.FINISHED)
commitSnapshotFor(attachmentId, cdn = 3)
fakeCdnEmpty()
ArchiveAttachmentReconciliationJob(forced = true).run()
assertThat(SignalDatabase.attachments.getAttachment(attachmentId)!!.archiveTransferState).isEqualTo(AttachmentTable.ArchiveTransferState.NONE)
}
/**
* The eventual safety net: an ordinary (non-forced) periodic reconciliation, run after the sync interval has elapsed, heals the bad state on its own -- media
* in the snapshot but missing from the CDN is reset to [AttachmentTable.ArchiveTransferState.NONE] and a re-upload is enqueued -- with no help from the
* migration or the reconcile-first flow.
*/
@Test
fun givenFinishedMediaMissingFromCdn_whenAnOrdinaryPeriodicReconciliationRuns_thenItHealsToNoneAndReUploads() {
SignalStore.backup.lastAttachmentReconciliationTime = System.currentTimeMillis() - 60.days.inWholeMilliseconds
mockkObject(BackupMessagesJob)
every { BackupMessagesJob.enqueue() } just Runs
val attachmentId = seedFinalizedAttachment("remote-key-periodic".toByteArray(), byteArrayOf(1, 2, 3, 4, 5))
SignalDatabase.attachments.setArchiveTransferState(attachmentId, AttachmentTable.ArchiveTransferState.FINISHED)
commitSnapshotFor(attachmentId, cdn = 3)
fakeCdnEmpty()
ArchiveAttachmentReconciliationJob(forced = false).run()
val healed = SignalDatabase.attachments.getAttachment(attachmentId)!!
assertThat(healed.archiveTransferState).isEqualTo(AttachmentTable.ArchiveTransferState.NONE)
assertThat(healed.archiveCdn).isNull()
verify(exactly = 1) { BackupMessagesJob.enqueue() }
}
/**
* Reconciliation must only repair genuinely-broken state. Media that is actually present on the CDN stays [AttachmentTable.ArchiveTransferState.FINISHED], so
* we never needlessly reset (and therefore re-upload) media that was correctly archived.
*/
@Test
fun givenFinishedMediaStillOnCdn_whenIReconcile_thenItStaysFinished() {
val attachmentId = seedFinalizedAttachment("remote-key-on-cdn".toByteArray(), byteArrayOf(6, 7, 8, 9, 10))
SignalDatabase.attachments.setArchiveTransferState(attachmentId, AttachmentTable.ArchiveTransferState.FINISHED)
commitSnapshotFor(attachmentId, cdn = 3)
fakeCdnContains(attachmentId, cdn = 3)
ArchiveAttachmentReconciliationJob(forced = true).run()
assertThat(SignalDatabase.attachments.getAttachment(attachmentId)!!.archiveTransferState).isEqualTo(AttachmentTable.ArchiveTransferState.FINISHED)
}
/**
* The other healing direction: media that is on the CDN but locally marked [AttachmentTable.ArchiveTransferState.NONE] and absent from the current snapshot is
* treated as a delete-candidate. Before deleting, reconciliation confirms it's still referenced locally and recovers it to
* [AttachmentTable.ArchiveTransferState.FINISHED] rather than deleting it from the CDN.
*/
@Test
fun givenNoneMediaOnCdnButNotInSnapshot_whenIReconcile_thenItIsRecoveredToFinished() {
val attachmentId = seedFinalizedAttachment("remote-key-flow2".toByteArray(), byteArrayOf(11, 12, 13, 14, 15))
SignalDatabase.attachments.setArchiveTransferState(attachmentId, AttachmentTable.ArchiveTransferState.NONE)
fakeCdnContains(attachmentId, cdn = 3)
ArchiveAttachmentReconciliationJob(forced = true).run()
assertThat(SignalDatabase.attachments.getAttachment(attachmentId)!!.archiveTransferState).isEqualTo(AttachmentTable.ArchiveTransferState.FINISHED)
}
@Test
fun givenFirstEverReconciliation_whenIForceIt_thenItStillRunsAndRepairs() {
SignalStore.backup.lastAttachmentReconciliationTime = -1
val attachmentId = seedFinalizedAttachment("remote-key-3".toByteArray(), byteArrayOf(11, 12, 13, 14, 15))
SignalDatabase.attachments.setArchiveTransferState(attachmentId, AttachmentTable.ArchiveTransferState.FINISHED)
commitSnapshotFor(attachmentId, cdn = 3)
fakeCdnEmpty()
ArchiveAttachmentReconciliationJob(forced = true).run()
assertThat(SignalDatabase.attachments.getAttachment(attachmentId)!!.archiveTransferState).isEqualTo(AttachmentTable.ArchiveTransferState.NONE)
}
@Test
fun givenLocalRestoreReconcilePending_whenReconcileCompletes_thenIExpectFlagCleared() {
SignalStore.backup.localRestoreReconcilePending = true
fakeCdnEmpty()
ArchiveAttachmentReconciliationJob(forced = true).run()
assertThat(SignalStore.backup.localRestoreReconcilePending).isFalse()
}
private fun seedFinalizedAttachment(remoteKey: ByteArray, data: ByteArray): AttachmentId {
val attachment = createAttachmentPointer(remoteKey, data.size)
val messageResult = SignalDatabase.messages.insertMessageInbox(createIncomingMessage(serverTime = 0.days, attachment = attachment)).get()
val attachmentId = messageResult.insertedAttachments!![attachment]!!
SignalDatabase.attachments.setTransferState(messageResult.messageId, attachmentId, AttachmentTable.TRANSFER_PROGRESS_STARTED)
SignalDatabase.attachments.finalizeAttachmentAfterDownload(messageResult.messageId, attachmentId, ByteArrayInputStream(data))
return attachmentId
}
private fun commitSnapshotFor(attachmentId: AttachmentId, cdn: Int) {
val attachment = SignalDatabase.attachments.getAttachment(attachmentId)!!
val plaintextHash = attachment.dataHash!!.decodeBase64OrThrow()
val remoteKey = attachment.remoteKey!!.decodeBase64OrThrow()
val mediaId = MediaName.fromPlaintextHashAndRemoteKey(plaintextHash, remoteKey).toMediaId(SignalStore.backup.mediaRootBackupKey).encode()
SignalDatabase.backupMediaSnapshots.writePendingMediaEntries(
listOf(MediaEntry(mediaId = mediaId, cdn = cdn, plaintextHash = plaintextHash, remoteKey = remoteKey, isThumbnail = false))
)
SignalDatabase.backupMediaSnapshots.commitPendingRows()
}
private fun fakeCdnContains(attachmentId: AttachmentId, cdn: Int) {
val attachment = SignalDatabase.attachments.getAttachment(attachmentId)!!
val plaintextHash = attachment.dataHash!!.decodeBase64OrThrow()
val remoteKey = attachment.remoteKey!!.decodeBase64OrThrow()
val mediaId = MediaName.fromPlaintextHashAndRemoteKey(plaintextHash, remoteKey).toMediaId(SignalStore.backup.mediaRootBackupKey).encode()
every { BackupRepository.listRemoteMediaObjects(any(), any()) } returns NetworkResult.Success(
ArchiveGetMediaItemsResponse(
storedMediaObjects = listOf(StoredMediaObject(cdn = cdn, mediaId = mediaId, objectLength = attachment.size)),
backupDir = null,
mediaDir = null,
cursor = null
)
)
}
private fun fakeCdnEmpty() {
every { BackupRepository.listRemoteMediaObjects(any(), any()) } returns NetworkResult.Success(
ArchiveGetMediaItemsResponse(storedMediaObjects = emptyList(), backupDir = null, mediaDir = null, cursor = null)
)
}
private fun createIncomingMessage(serverTime: Duration, attachment: Attachment): IncomingMessage {
return IncomingMessage(
type = MessageType.NORMAL,
from = harness.others[0],
body = null,
sentTimeMillis = serverTime.inWholeMilliseconds,
serverTimeMillis = serverTime.inWholeMilliseconds,
receivedTimeMillis = serverTime.inWholeMilliseconds,
attachments = listOf(attachment)
)
}
private fun createAttachmentPointer(key: ByteArray, size: Int): Attachment {
return PointerAttachment.forPointer(
pointer = Optional.of(
SignalServiceAttachmentPointer(
cdnNumber = 3,
remoteId = SignalServiceAttachmentRemoteId.V4("asdf"),
contentType = MediaUtil.IMAGE_JPEG,
key = key,
size = Optional.of(size),
preview = Optional.empty(),
width = 2,
height = 2,
digest = Optional.of(byteArrayOf()),
incrementalDigest = Optional.empty(),
incrementalMacChunkSize = 0,
fileName = Optional.of("file.jpg"),
voiceNote = false,
isBorderless = false,
isGif = false,
caption = Optional.empty(),
blurHash = Optional.empty(),
uploadTimestamp = 0,
uuid = null
)
)
).get()
}
}
@@ -960,6 +960,34 @@ class AttachmentTable(
.run()
}
/**
* Whether the user has any archive-finished media that came from a local backup. Used to detect the local-restore bad state where restored media was
* incorrectly marked as already archived, so we only run an expensive reconciliation for users who could actually be affected.
*/
fun hasArchiveFinishedLocalBackupMedia(): Boolean {
return readableDatabase
.exists("$TABLE_NAME INNER JOIN ${AttachmentMetadataTable.TABLE_NAME} ON $TABLE_NAME.$METADATA_ID = ${AttachmentMetadataTable.TABLE_NAME}.${AttachmentMetadataTable.ID}")
.where("$TABLE_NAME.$ARCHIVE_TRANSFER_STATE = ? AND ${AttachmentMetadataTable.TABLE_NAME}.${AttachmentMetadataTable.LOCAL_BACKUP_KEY} NOT NULL", ArchiveTransferState.FINISHED.value)
.run()
}
/**
* Resets archive-finished media that came from a local backup, returning the number of attachments repaired. Safe only when the user doesn't back up media,
* as they can't have anything legitimately on the archive CDN. See [hasArchiveFinishedLocalBackupMedia].
*/
fun resetArchiveTransferStateForLocalBackupMedia(): Int {
return writableDatabase
.update(TABLE_NAME)
.values(
ARCHIVE_TRANSFER_STATE to ArchiveTransferState.NONE.value,
ARCHIVE_CDN to null
)
.where(
"$ARCHIVE_TRANSFER_STATE = ${ArchiveTransferState.FINISHED.value} AND $METADATA_ID IN (SELECT ${AttachmentMetadataTable.ID} FROM ${AttachmentMetadataTable.TABLE_NAME} WHERE ${AttachmentMetadataTable.LOCAL_BACKUP_KEY} NOT NULL)"
)
.run()
}
/**
* Returns whether or not there are thumbnails that need to be uploaded to the archive.
*/
@@ -1790,7 +1818,7 @@ class AttachmentTable(
* that the content of the attachment will never change.
*/
@Throws(MmsException::class)
fun finalizeAttachmentAfterDownload(mmsId: Long, attachmentId: AttachmentId, inputStream: InputStream, offloadRestoredAt: Duration? = null, archiveRestore: Boolean = false, notify: Boolean = true) {
fun finalizeAttachmentAfterDownload(mmsId: Long, attachmentId: AttachmentId, inputStream: InputStream, offloadRestoredAt: Duration? = null, archiveRestore: Boolean = false, restoredFromArchiveCdn: Boolean = false, notify: Boolean = true) {
Log.i(TAG, "[finalizeAttachmentAfterDownload] Finalizing downloaded data for $attachmentId. (MessageId: $mmsId, $attachmentId)")
val existingPlaceholder: DatabaseAttachment = getAttachment(attachmentId) ?: throw MmsException("No attachment found for id: $attachmentId")
@@ -1835,8 +1863,11 @@ class AttachmentTable(
values.put(DATA_HASH_START, fileWriteResult.hash)
values.put(DATA_HASH_END, fileWriteResult.hash)
if (archiveRestore) {
if (restoredFromArchiveCdn) {
values.put(ARCHIVE_TRANSFER_STATE, ArchiveTransferState.FINISHED.value)
} else if (archiveRestore) {
values.putNull(ARCHIVE_CDN)
values.put(ARCHIVE_TRANSFER_STATE, ArchiveTransferState.NONE.value)
}
}
@@ -82,6 +82,19 @@ class ArchiveAttachmentReconciliationJob private constructor(
Log.i(TAG, "Skip enqueueing reconciliation job: attempt limit exceeded.")
}
}
/**
* Reconcile-first entry point for after a local restore. Runs a [BackupMessagesJob] (to capture a snapshot) chained into a forced reconciliation.
* Sets [BackupValues.localRestoreReconcilePending] so the backup holds off on the bulk attachment backfill, letting reconciliation run first. Reconciliation
* then clears the flag and re-triggers the backfill for whatever genuinely still needs uploading.
*/
fun enqueueReconcileFirstForLocalRestore() {
SignalStore.backup.localRestoreReconcilePending = true
AppDependencies.jobManager
.startChain(BackupMessagesJob())
.then(ArchiveAttachmentReconciliationJob(forced = true))
.enqueue()
}
}
constructor(forced: Boolean = false) : this(
@@ -116,7 +129,7 @@ class ArchiveAttachmentReconciliationJob private constructor(
return Result.success()
}
if (SignalStore.backup.lastAttachmentReconciliationTime < 0) {
if (!forced && SignalStore.backup.lastAttachmentReconciliationTime < 0) {
Log.w(TAG, "First ever time we're attempting a reconciliation. Setting the last sync time to now, so we'll run at the proper interval. Skipping this iteration.", true)
SignalStore.backup.lastAttachmentReconciliationTime = System.currentTimeMillis()
return Result.success()
@@ -138,10 +151,27 @@ class ArchiveAttachmentReconciliationJob private constructor(
// we use to determine which attachments need to be re-uploaded will possibly result in us unnecessarily re-uploading attachments.
snapshotVersion = snapshotVersion ?: SignalDatabase.backupMediaSnapshots.getCurrentSnapshotVersion()
return syncDataFromCdn(snapshotVersion!!) ?: Result.success()
syncDataFromCdn(snapshotVersion!!)?.let { return it }
clearPendingLocalRestoreReconcile()
return Result.success()
}
override fun onFailure() = Unit
/**
* Once a reconciliation has fully crawled the CDN, any media that was already archived has been marked finished, so the bulk backfill that was held off
* during a local restore can proceed for whatever genuinely still needs uploading.
*/
private fun clearPendingLocalRestoreReconcile() {
if (SignalStore.backup.localRestoreReconcilePending) {
Log.i(TAG, "Local restore reconciliation complete. Clearing the pending flag and enqueueing a backup to upload any remaining media.", true)
SignalStore.backup.localRestoreReconcilePending = false
BackupMessagesJob.enqueue()
}
}
override fun onFailure() {
clearPendingLocalRestoreReconcile()
}
/**
* Fetches all attachment metadata from the archive CDN and ensures that our local store is in sync with it.
@@ -302,6 +332,8 @@ class ArchiveAttachmentReconciliationJob private constructor(
* - Mark that page as seen on the remote.
* - Fix any CDN mismatches by updating our local store with the correct CDN.
* - Delete any orphaned attachments that are on the CDN but not in our local store.
* - During the local-restore reconcile-first flow, mark media confirmed present on the CDN as finished. A local restore resets everything to NONE (we don't
* trust the backup's CDN claims), so this is what promotes the media that genuinely is on the CDN back to finished, preventing a needless re-upload of it.
*
* @return A list of media objects that should be deleted (after being verified)
*/
@@ -329,6 +361,13 @@ class ArchiveAttachmentReconciliationJob private constructor(
}
}
if (SignalStore.backup.localRestoreReconcilePending) {
val markedFinished = SignalDatabase.attachments.setArchiveFinishedForMatchingMediaObjects(mediaObjectsOnBothRemoteAndLocal.toSet())
if (markedFinished > 0) {
Log.i(TAG, "Marked $markedFinished media object group(s) as finished after confirming they are present on the CDN.", true)
}
}
return mediaOnRemoteButNotLocal
}
@@ -417,7 +417,10 @@ class BackupMessagesJob private constructor(
return Result.failure()
}
if (SignalStore.backup.backsUpMedia && SignalDatabase.attachments.doAnyAttachmentsNeedArchiveUpload()) {
if (SignalStore.backup.localRestoreReconcilePending) {
Log.i(TAG, "A local restore reconciliation is pending. Holding off on the attachment backfill until reconciliation has marked already-archived media as finished.", true)
ArchiveUploadProgress.onMessageBackupFinishedEarly()
} else if (SignalStore.backup.backsUpMedia && SignalDatabase.attachments.doAnyAttachmentsNeedArchiveUpload()) {
Log.i(TAG, "Enqueuing attachment backfill job.", true)
AppDependencies.jobManager.add(ArchiveAttachmentBackfillJob())
} else {
@@ -103,6 +103,15 @@ class CheckRestoreMediaLeftJob private constructor(parameters: Parameters) : Job
SignalStore.backup.deletionState = DeletionState.MEDIA_DOWNLOAD_FINISHED
}
if (SignalStore.backup.localRestoreReconcilePending) {
if (SignalStore.backup.backsUpMedia) {
Log.i(TAG, "Local restore complete. Reconciling restored media against the archive CDN before uploading. (Flag cleared by the reconciliation job.)")
ArchiveAttachmentReconciliationJob.enqueueReconcileFirstForLocalRestore()
} else {
SignalStore.backup.localRestoreReconcilePending = false
}
}
if (!SignalStore.backup.backsUpMedia) {
SignalDatabase.attachments.markQuotesThatNeedReconstruction()
AppDependencies.jobManager.add(QuoteThumbnailReconstructionJob())
@@ -77,6 +77,7 @@ import org.thoughtcrime.securesms.migrations.GooglePlayBillingPurchaseTokenMigra
import org.thoughtcrime.securesms.migrations.IdentityTableCleanupMigrationJob;
import org.thoughtcrime.securesms.migrations.KeyTransparencyUsernameMigrationJob;
import org.thoughtcrime.securesms.migrations.LegacyMigrationJob;
import org.thoughtcrime.securesms.migrations.LocalArchiveReconciliationMigrationJob;
import org.thoughtcrime.securesms.migrations.MigrationCompleteJob;
import org.thoughtcrime.securesms.migrations.OptimizeMessageSearchIndexMigrationJob;
import org.thoughtcrime.securesms.migrations.PassingMigrationJob;
@@ -344,6 +345,7 @@ public final class JobManagerFactories {
put(IdentityTableCleanupMigrationJob.KEY, new IdentityTableCleanupMigrationJob.Factory());
put(KeyTransparencyUsernameMigrationJob.KEY, new KeyTransparencyUsernameMigrationJob.Factory());
put(LegacyMigrationJob.KEY, new LegacyMigrationJob.Factory());
put(LocalArchiveReconciliationMigrationJob.KEY, new LocalArchiveReconciliationMigrationJob.Factory());
put(MigrationCompleteJob.KEY, new MigrationCompleteJob.Factory());
put(OptimizeMessageSearchIndexMigrationJob.KEY, new OptimizeMessageSearchIndexMigrationJob.Factory());
put(PinOptOutMigration.KEY, new PinOptOutMigration.Factory());
@@ -410,6 +410,7 @@ class RestoreAttachmentJob private constructor(
inputStream = input,
offloadRestoredAt = if (manual) System.currentTimeMillis().milliseconds else null,
archiveRestore = true,
restoredFromArchiveCdn = useArchiveCdn,
notify = manual
)
ArchiveDatabaseExecutor.throttledNotifyAttachmentAndChatListObservers()
@@ -21,6 +21,7 @@ import org.thoughtcrime.securesms.database.SignalDatabase
import org.thoughtcrime.securesms.dependencies.AppDependencies
import org.thoughtcrime.securesms.jobmanager.Job
import org.thoughtcrime.securesms.jobs.protos.RestoreLocalAttachmentJobData
import org.thoughtcrime.securesms.keyvalue.SignalStore
import org.thoughtcrime.securesms.mms.MmsException
import org.whispersystems.signalservice.api.crypto.AttachmentCipherInputStream
import org.whispersystems.signalservice.api.crypto.AttachmentCipherInputStream.IntegrityCheck
@@ -46,6 +47,8 @@ class RestoreLocalAttachmentJob private constructor(
fun enqueueRestoreLocalAttachmentsJobs(mediaNameToFileInfo: Map<String, DocumentFileInfo>) {
val jobManager = AppDependencies.jobManager
SignalStore.backup.localRestoreReconcilePending = true
val orphanedCount = SignalDatabase.attachments.markRestorableAttachmentsWithoutMessageAsFailed()
if (orphanedCount > 0) {
Log.w(TAG, "Failed $orphanedCount orphaned restorable attachment(s) with no backing message before enqueueing restores.")
@@ -85,6 +85,7 @@ class BackupValues(store: KeyValueStore) : SignalStoreValues(store) {
private const val KEY_BACKUP_DELETION_STATE = "backup.deletion.state"
private const val KEY_REMOTE_STORAGE_GARBAGE_COLLECTION_PENDING = "backup.remoteStorageGarbageCollectionPending"
private const val KEY_ARCHIVE_ATTACHMENT_RECONCILIATION_ATTEMPTS = "backup.archiveAttachmentReconciliationAttempts"
private const val KEY_LOCAL_RESTORE_RECONCILE_PENDING = "backup.localRestoreReconcilePending"
private const val KEY_MEDIA_ROOT_BACKUP_KEY = "backup.mediaRootBackupKey"
@@ -186,6 +187,11 @@ class BackupValues(store: KeyValueStore) : SignalStoreValues(store) {
var userManuallySkippedMediaRestore: Boolean by booleanValue(KEY_USER_MANUALLY_SKIPPED_MEDIA_RESTORE, false)
/**
* Set when a local backup restore is kicked off so that, once media restore completes, we reconcile the restored media against the archive CDN.
*/
var localRestoreReconcilePending: Boolean by booleanValue(KEY_LOCAL_RESTORE_RECONCILE_PENDING, false)
var backupExpiredAndDowngraded: Boolean by booleanValue(KEY_BACKUP_EXPIRED_AND_DOWNGRADED, false)
/**
@@ -43,6 +43,7 @@ class LogSectionRemoteBackups : LogSection {
output.append("Optimize storage : ${SignalStore.backup.optimizeStorage}\n")
output.append("Detected subscription state mismatch: ${SignalStore.backup.subscriptionStateMismatchDetected}\n")
output.append("Last verified key time : ${SignalStore.backup.lastVerifyKeyTime}\n")
output.append("Local restore reconcile pending : ${SignalStore.backup.localRestoreReconcilePending}\n")
output.append("Restore state : ${ArchiveRestoreProgress.state}\n")
output.append("\n -- Subscription State\n")
@@ -207,9 +207,10 @@ public class ApplicationMigrations {
static final int NOTIFICATION_STATE_CLEANUP = 163;
static final int KT_USERNAME_CAPABILITY = 164;
static final int FIX_CHANGE_NUMBER_ERROR_2 = 165;
static final int LOCAL_ARCHIVE_RECONCILE = 166;
}
public static final int CURRENT_VERSION = 165;
public static final int CURRENT_VERSION = 166;
/**
* This *must* be called after the {@link JobManager} has been instantiated, but *before* the call
@@ -960,6 +961,10 @@ public class ApplicationMigrations {
jobs.put(Version.KT_USERNAME_CAPABILITY, new KeyTransparencyUsernameMigrationJob());
}
if (lastSeenVersion < Version.LOCAL_ARCHIVE_RECONCILE) {
jobs.put(Version.LOCAL_ARCHIVE_RECONCILE, new LocalArchiveReconciliationMigrationJob());
}
return jobs;
}
@@ -0,0 +1,61 @@
/*
* Copyright 2026 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.thoughtcrime.securesms.migrations
import org.signal.core.util.logging.Log
import org.thoughtcrime.securesms.database.SignalDatabase
import org.thoughtcrime.securesms.jobmanager.Job
import org.thoughtcrime.securesms.jobs.ArchiveAttachmentReconciliationJob
import org.thoughtcrime.securesms.keyvalue.SignalStore
/**
* There was a bug where media restored from a local backup was incorrectly marked as already being on the archive CDN, which prevented it from ever being
* uploaded. This migration repairs that state:
*
* - Non-media-backup users can't have anything legitimately on the CDN, so we just reset the bogus state locally; the normal backfill re-uploads it if they
* later enable media backups.
* - Media-backup users may have some of that media genuinely on the CDN, so we can't tell locally which entries are bogus, and instead reconcile against it.
*
* Reconciliation is expensive server-side, so we only expedite it for media-backup users who are actually in the bad state (i.e. still have local-restore media
* marked as archived), rather than for everyone.
*/
internal class LocalArchiveReconciliationMigrationJob(
parameters: Parameters = Parameters.Builder().build()
) : MigrationJob(parameters) {
companion object {
val TAG = Log.tag(LocalArchiveReconciliationMigrationJob::class.java)
const val KEY = "LocalArchiveReconciliationMigrationJob"
}
override fun getFactoryKey(): String = KEY
override fun isUiBlocking(): Boolean = false
override fun performMigration() {
if (!SignalStore.backup.backsUpMedia) {
val resetCount = SignalDatabase.attachments.resetArchiveTransferStateForLocalBackupMedia()
Log.i(TAG, "User does not back up media. Reset $resetCount local-restore attachment(s) incorrectly marked as archived so they'll upload if media backups are enabled later.")
return
}
if (!SignalDatabase.attachments.hasArchiveFinishedLocalBackupMedia()) {
Log.i(TAG, "No archive-finished media from a local backup. Not in the bad state, so skipping.")
return
}
Log.i(TAG, "Expediting an archive reconciliation to repair any media incorrectly marked as archived after a local restore.")
ArchiveAttachmentReconciliationJob.enqueueReconcileFirstForLocalRestore()
}
override fun shouldRetry(e: Exception): Boolean = false
class Factory : Job.Factory<LocalArchiveReconciliationMigrationJob> {
override fun create(parameters: Parameters, serializedData: ByteArray?): LocalArchiveReconciliationMigrationJob {
return LocalArchiveReconciliationMigrationJob(parameters)
}
}
}
@@ -0,0 +1,139 @@
/*
* Copyright 2026 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.thoughtcrime.securesms.database
import android.app.Application
import assertk.assertThat
import assertk.assertions.isEqualTo
import assertk.assertions.isFalse
import assertk.assertions.isNull
import assertk.assertions.isTrue
import org.junit.BeforeClass
import org.junit.Rule
import org.junit.Test
import org.junit.runner.RunWith
import org.robolectric.RobolectricTestRunner
import org.robolectric.annotation.Config
import org.signal.core.models.database.AttachmentId
import org.signal.core.util.logging.Log
import org.thoughtcrime.securesms.attachments.ArchivedAttachment
import org.thoughtcrime.securesms.attachments.Attachment
import org.thoughtcrime.securesms.mms.IncomingMessage
import org.thoughtcrime.securesms.recipients.Recipient
import org.thoughtcrime.securesms.testutil.RecipientTestRule
import org.thoughtcrime.securesms.testutil.SystemOutLogger
import java.util.UUID
import kotlin.random.Random
/**
* JVM (Robolectric) coverage for the archive transfer state of media imported from a local backup. Verifies the invariant that a locally-imported attachment is
* FINISHED only when it carries an archive CDN, and that [AttachmentTable.hasArchiveFinishedLocalBackupMedia] detects exactly the bad state the local-restore
* migration repairs.
*/
@Suppress("ClassName")
@RunWith(RobolectricTestRunner::class)
@Config(manifest = Config.NONE, application = Application::class)
class AttachmentTableTest_localRestoreArchiveState {
@get:Rule val recipients = RecipientTestRule()
companion object {
@BeforeClass
@JvmStatic
fun setUpClass() {
Log.initialize(SystemOutLogger())
}
}
@Test
fun givenLocallyImportedMediaThatCarriesAnArchiveCdn_whenInserted_thenIExpectArchiveStateFinished() {
val attachmentId = insertArchivedAttachment(archiveCdn = 3, localBackupKey = Random.nextBytes(32))
val attachment = SignalDatabase.attachments.getAttachment(attachmentId)!!
assertThat(attachment.archiveTransferState).isEqualTo(AttachmentTable.ArchiveTransferState.FINISHED)
assertThat(attachment.archiveCdn).isEqualTo(3)
}
@Test
fun givenLocallyImportedMediaThatCarriesNoArchiveCdn_whenInserted_thenIExpectArchiveStateNone() {
val attachmentId = insertArchivedAttachment(archiveCdn = null, localBackupKey = Random.nextBytes(32))
val attachment = SignalDatabase.attachments.getAttachment(attachmentId)!!
assertThat(attachment.archiveTransferState).isEqualTo(AttachmentTable.ArchiveTransferState.NONE)
assertThat(attachment.archiveCdn).isNull()
}
@Test
fun hasArchiveFinishedLocalBackupMedia_trueForFinishedLocalBackupMedia() {
insertArchivedAttachment(archiveCdn = 3, localBackupKey = Random.nextBytes(32))
assertThat(SignalDatabase.attachments.hasArchiveFinishedLocalBackupMedia()).isTrue()
}
@Test
fun hasArchiveFinishedLocalBackupMedia_falseWhenFinishedButNotFromLocalBackup() {
insertArchivedAttachment(archiveCdn = 3, localBackupKey = null)
assertThat(SignalDatabase.attachments.hasArchiveFinishedLocalBackupMedia()).isFalse()
}
@Test
fun hasArchiveFinishedLocalBackupMedia_falseOnceLocalBackupMediaIsNoLongerFinished() {
insertArchivedAttachment(archiveCdn = 3, localBackupKey = Random.nextBytes(32))
assertThat(SignalDatabase.attachments.hasArchiveFinishedLocalBackupMedia()).isTrue()
SignalDatabase.attachments.resetArchiveTransferStateForLocalBackupMedia()
assertThat(SignalDatabase.attachments.hasArchiveFinishedLocalBackupMedia()).isFalse()
}
private fun insertArchivedAttachment(archiveCdn: Int?, localBackupKey: ByteArray?): AttachmentId {
val from = recipients.createRecipient("Some Contact")
val threadId = SignalDatabase.threads.getOrCreateThreadIdFor(Recipient.resolved(from))
val attachment = createArchivedAttachment(archiveCdn = archiveCdn, localBackupKey = localBackupKey)
val message = IncomingMessage(
type = MessageType.NORMAL,
from = from,
body = null,
sentTimeMillis = 100L,
serverTimeMillis = 100L,
receivedTimeMillis = 200L,
attachments = listOf(attachment)
)
val messageId = SignalDatabase.messages.insertMessageInbox(message, threadId).get().messageId
return SignalDatabase.attachments.getAttachmentsForMessage(messageId).first().attachmentId
}
private fun createArchivedAttachment(archiveCdn: Int?, localBackupKey: ByteArray?): Attachment {
return ArchivedAttachment(
contentType = "image/jpeg",
size = 1024,
cdn = 3,
uploadTimestamp = 0,
key = Random.nextBytes(8),
cdnKey = "password",
archiveCdn = archiveCdn,
plaintextHash = Random.nextBytes(8),
incrementalMac = Random.nextBytes(8),
incrementalMacChunkSize = 8,
width = 100,
height = 100,
caption = null,
blurHash = null,
voiceNote = false,
borderless = false,
stickerLocator = null,
gif = false,
quote = false,
quoteTargetContentType = null,
uuid = UUID.randomUUID(),
fileName = null,
localBackupKey = localBackupKey
)
}
}
@@ -0,0 +1,105 @@
/*
* Copyright 2026 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.thoughtcrime.securesms.migrations
import android.app.Application
import io.mockk.every
import io.mockk.mockk
import io.mockk.mockkObject
import io.mockk.unmockkObject
import io.mockk.verify
import org.junit.After
import org.junit.Before
import org.junit.Rule
import org.junit.Test
import org.junit.runner.RunWith
import org.robolectric.RobolectricTestRunner
import org.robolectric.annotation.Config
import org.signal.core.util.logging.Log
import org.thoughtcrime.securesms.database.AttachmentTable
import org.thoughtcrime.securesms.database.SignalDatabase
import org.thoughtcrime.securesms.dependencies.AppDependencies
import org.thoughtcrime.securesms.jobmanager.JobManager
import org.thoughtcrime.securesms.jobs.BackupMessagesJob
import org.thoughtcrime.securesms.testutil.MockAppDependenciesRule
import org.thoughtcrime.securesms.testutil.MockSignalStoreRule
import org.thoughtcrime.securesms.testutil.SystemOutLogger
import kotlin.time.Duration.Companion.days
@RunWith(RobolectricTestRunner::class)
@Config(manifest = Config.NONE, application = Application::class)
class LocalArchiveReconciliationMigrationJobTest {
@get:Rule
val mockSignalStore = MockSignalStoreRule()
@get:Rule
val appDependencies = MockAppDependenciesRule()
private lateinit var jobManager: JobManager
private lateinit var attachments: AttachmentTable
@Before
fun setUp() {
Log.initialize(SystemOutLogger())
jobManager = AppDependencies.jobManager
attachments = mockk(relaxed = true)
mockkObject(SignalDatabase.Companion)
every { SignalDatabase.attachments } returns attachments
}
@After
fun tearDown() {
unmockkObject(SignalDatabase.Companion)
}
@Test
fun givenUserDoesNotBackUpMedia_whenIRunMigration_thenIResetLocalBackupMediaAndDoNotEnqueueReconciliation() {
every { mockSignalStore.backup.backsUpMedia } returns false
LocalArchiveReconciliationMigrationJob().run()
verify(exactly = 1) { attachments.resetArchiveTransferStateForLocalBackupMedia() }
verify(exactly = 0) { jobManager.startChain(any<BackupMessagesJob>()) }
}
@Test
fun givenMediaBackupUserNotInBadState_whenIRunMigration_thenIDoNotEnqueueReconciliation() {
every { mockSignalStore.backup.backsUpMedia } returns true
every { attachments.hasArchiveFinishedLocalBackupMedia() } returns false
LocalArchiveReconciliationMigrationJob().run()
verify(exactly = 0) { jobManager.startChain(any<BackupMessagesJob>()) }
}
@Test
fun givenMediaBackupUserInBadState_whenIRunMigration_thenIEnqueueReconciliation() {
every { mockSignalStore.backup.backsUpMedia } returns true
every { attachments.hasArchiveFinishedLocalBackupMedia() } returns true
LocalArchiveReconciliationMigrationJob().run()
verify(exactly = 1) { mockSignalStore.backup.localRestoreReconcilePending = true }
verify(exactly = 1) { jobManager.startChain(any<BackupMessagesJob>()) }
}
/**
* Guards against reintroducing a registration-age skip. A user who restored from a local backup and only later enabled remote backups may have registered
* long ago yet never had a reconciliation heal them, so being in the bad state must expedite reconciliation regardless of how long ago they registered.
*/
@Test
fun givenMediaBackupUserInBadStateWhoRegisteredLongAgo_whenIRunMigration_thenIStillEnqueueReconciliation() {
every { mockSignalStore.backup.backsUpMedia } returns true
every { mockSignalStore.account.registeredAtTimestamp } returns System.currentTimeMillis() - 365.days.inWholeMilliseconds
every { attachments.hasArchiveFinishedLocalBackupMedia() } returns true
LocalArchiveReconciliationMigrationJob().run()
verify(exactly = 1) { jobManager.startChain(any<BackupMessagesJob>()) }
}
}