Upload media restored from local backups to the archive CDN.

2026-08-06 05:14:50 +01:00 · 2026-07-07 16:08:51 -03:00
parent e687718ec3
commit 384344c91b
15 changed files with 747 additions and 9 deletions
@@ -960,6 +960,34 @@ class AttachmentTable(
      .run()
  }

+  /**
+   * Whether the user has any archive-finished media that came from a local backup. Used to detect the local-restore bad state where restored media was
+   * incorrectly marked as already archived, so we only run an expensive reconciliation for users who could actually be affected.
+   */
+  fun hasArchiveFinishedLocalBackupMedia(): Boolean {
+    return readableDatabase
+      .exists("$TABLE_NAME INNER JOIN ${AttachmentMetadataTable.TABLE_NAME} ON $TABLE_NAME.$METADATA_ID = ${AttachmentMetadataTable.TABLE_NAME}.${AttachmentMetadataTable.ID}")
+      .where("$TABLE_NAME.$ARCHIVE_TRANSFER_STATE = ? AND ${AttachmentMetadataTable.TABLE_NAME}.${AttachmentMetadataTable.LOCAL_BACKUP_KEY} NOT NULL", ArchiveTransferState.FINISHED.value)
+      .run()
+  }
+
+  /**
+   * Resets archive-finished media that came from a local backup, returning the number of attachments repaired. Safe only when the user doesn't back up media,
+   * as they can't have anything legitimately on the archive CDN. See [hasArchiveFinishedLocalBackupMedia].
+   */
+  fun resetArchiveTransferStateForLocalBackupMedia(): Int {
+    return writableDatabase
+      .update(TABLE_NAME)
+      .values(
+        ARCHIVE_TRANSFER_STATE to ArchiveTransferState.NONE.value,
+        ARCHIVE_CDN to null
+      )
+      .where(
+        "$ARCHIVE_TRANSFER_STATE = ${ArchiveTransferState.FINISHED.value} AND $METADATA_ID IN (SELECT ${AttachmentMetadataTable.ID} FROM ${AttachmentMetadataTable.TABLE_NAME} WHERE ${AttachmentMetadataTable.LOCAL_BACKUP_KEY} NOT NULL)"
+      )
+      .run()
+  }
+
  /**
   * Returns whether or not there are thumbnails that need to be uploaded to the archive.
   */
@@ -1790,7 +1818,7 @@ class AttachmentTable(
   * that the content of the attachment will never change.
   */
  @Throws(MmsException::class)
-  fun finalizeAttachmentAfterDownload(mmsId: Long, attachmentId: AttachmentId, inputStream: InputStream, offloadRestoredAt: Duration? = null, archiveRestore: Boolean = false, notify: Boolean = true) {
+  fun finalizeAttachmentAfterDownload(mmsId: Long, attachmentId: AttachmentId, inputStream: InputStream, offloadRestoredAt: Duration? = null, archiveRestore: Boolean = false, restoredFromArchiveCdn: Boolean = false, notify: Boolean = true) {
    Log.i(TAG, "[finalizeAttachmentAfterDownload] Finalizing downloaded data for $attachmentId. (MessageId: $mmsId, $attachmentId)")

    val existingPlaceholder: DatabaseAttachment = getAttachment(attachmentId) ?: throw MmsException("No attachment found for id: $attachmentId")
@@ -1835,8 +1863,11 @@ class AttachmentTable(
        values.put(DATA_HASH_START, fileWriteResult.hash)
        values.put(DATA_HASH_END, fileWriteResult.hash)

-        if (archiveRestore) {
+        if (restoredFromArchiveCdn) {
          values.put(ARCHIVE_TRANSFER_STATE, ArchiveTransferState.FINISHED.value)
+        } else if (archiveRestore) {
+          values.putNull(ARCHIVE_CDN)
+          values.put(ARCHIVE_TRANSFER_STATE, ArchiveTransferState.NONE.value)
        }
      }

@@ -82,6 +82,19 @@ class ArchiveAttachmentReconciliationJob private constructor(
        Log.i(TAG, "Skip enqueueing reconciliation job: attempt limit exceeded.")
      }
    }
+
+    /**
+     * Reconcile-first entry point for after a local restore. Runs a [BackupMessagesJob] (to capture a snapshot) chained into a forced reconciliation.
+     * Sets [BackupValues.localRestoreReconcilePending] so the backup holds off on the bulk attachment backfill, letting reconciliation run first. Reconciliation
+     * then clears the flag and re-triggers the backfill for whatever genuinely still needs uploading.
+     */
+    fun enqueueReconcileFirstForLocalRestore() {
+      SignalStore.backup.localRestoreReconcilePending = true
+      AppDependencies.jobManager
+        .startChain(BackupMessagesJob())
+        .then(ArchiveAttachmentReconciliationJob(forced = true))
+        .enqueue()
+    }
  }

  constructor(forced: Boolean = false) : this(
@@ -116,7 +129,7 @@ class ArchiveAttachmentReconciliationJob private constructor(
      return Result.success()
    }

-    if (SignalStore.backup.lastAttachmentReconciliationTime < 0) {
+    if (!forced && SignalStore.backup.lastAttachmentReconciliationTime < 0) {
      Log.w(TAG, "First ever time we're attempting a reconciliation. Setting the last sync time to now, so we'll run at the proper interval. Skipping this iteration.", true)
      SignalStore.backup.lastAttachmentReconciliationTime = System.currentTimeMillis()
      return Result.success()
@@ -138,10 +151,27 @@ class ArchiveAttachmentReconciliationJob private constructor(
    // we use to determine which attachments need to be re-uploaded will possibly result in us unnecessarily re-uploading attachments.
    snapshotVersion = snapshotVersion ?: SignalDatabase.backupMediaSnapshots.getCurrentSnapshotVersion()

-    return syncDataFromCdn(snapshotVersion!!) ?: Result.success()
+    syncDataFromCdn(snapshotVersion!!)?.let { return it }
+
+    clearPendingLocalRestoreReconcile()
+    return Result.success()
  }

-  override fun onFailure() = Unit
+  /**
+   * Once a reconciliation has fully crawled the CDN, any media that was already archived has been marked finished, so the bulk backfill that was held off
+   * during a local restore can proceed for whatever genuinely still needs uploading.
+   */
+  private fun clearPendingLocalRestoreReconcile() {
+    if (SignalStore.backup.localRestoreReconcilePending) {
+      Log.i(TAG, "Local restore reconciliation complete. Clearing the pending flag and enqueueing a backup to upload any remaining media.", true)
+      SignalStore.backup.localRestoreReconcilePending = false
+      BackupMessagesJob.enqueue()
+    }
+  }
+
+  override fun onFailure() {
+    clearPendingLocalRestoreReconcile()
+  }

  /**
   * Fetches all attachment metadata from the archive CDN and ensures that our local store is in sync with it.
@@ -302,6 +332,8 @@ class ArchiveAttachmentReconciliationJob private constructor(
   * - Mark that page as seen on the remote.
   * - Fix any CDN mismatches by updating our local store with the correct CDN.
   * - Delete any orphaned attachments that are on the CDN but not in our local store.
+   * - During the local-restore reconcile-first flow, mark media confirmed present on the CDN as finished. A local restore resets everything to NONE (we don't
+   *   trust the backup's CDN claims), so this is what promotes the media that genuinely is on the CDN back to finished, preventing a needless re-upload of it.
   *
   * @return A list of media objects that should be deleted (after being verified)
   */
@@ -329,6 +361,13 @@ class ArchiveAttachmentReconciliationJob private constructor(
      }
    }

+    if (SignalStore.backup.localRestoreReconcilePending) {
+      val markedFinished = SignalDatabase.attachments.setArchiveFinishedForMatchingMediaObjects(mediaObjectsOnBothRemoteAndLocal.toSet())
+      if (markedFinished > 0) {
+        Log.i(TAG, "Marked $markedFinished media object group(s) as finished after confirming they are present on the CDN.", true)
+      }
+    }
+
    return mediaOnRemoteButNotLocal
  }

@@ -417,7 +417,10 @@ class BackupMessagesJob private constructor(
      return Result.failure()
    }

-    if (SignalStore.backup.backsUpMedia && SignalDatabase.attachments.doAnyAttachmentsNeedArchiveUpload()) {
+    if (SignalStore.backup.localRestoreReconcilePending) {
+      Log.i(TAG, "A local restore reconciliation is pending. Holding off on the attachment backfill until reconciliation has marked already-archived media as finished.", true)
+      ArchiveUploadProgress.onMessageBackupFinishedEarly()
+    } else if (SignalStore.backup.backsUpMedia && SignalDatabase.attachments.doAnyAttachmentsNeedArchiveUpload()) {
      Log.i(TAG, "Enqueuing attachment backfill job.", true)
      AppDependencies.jobManager.add(ArchiveAttachmentBackfillJob())
    } else {
@@ -103,6 +103,15 @@ class CheckRestoreMediaLeftJob private constructor(parameters: Parameters) : Job
      SignalStore.backup.deletionState = DeletionState.MEDIA_DOWNLOAD_FINISHED
    }

+    if (SignalStore.backup.localRestoreReconcilePending) {
+      if (SignalStore.backup.backsUpMedia) {
+        Log.i(TAG, "Local restore complete. Reconciling restored media against the archive CDN before uploading. (Flag cleared by the reconciliation job.)")
+        ArchiveAttachmentReconciliationJob.enqueueReconcileFirstForLocalRestore()
+      } else {
+        SignalStore.backup.localRestoreReconcilePending = false
+      }
+    }
+
    if (!SignalStore.backup.backsUpMedia) {
      SignalDatabase.attachments.markQuotesThatNeedReconstruction()
      AppDependencies.jobManager.add(QuoteThumbnailReconstructionJob())
@@ -77,6 +77,7 @@ import org.thoughtcrime.securesms.migrations.GooglePlayBillingPurchaseTokenMigra
 import org.thoughtcrime.securesms.migrations.IdentityTableCleanupMigrationJob;
 import org.thoughtcrime.securesms.migrations.KeyTransparencyUsernameMigrationJob;
 import org.thoughtcrime.securesms.migrations.LegacyMigrationJob;
+import org.thoughtcrime.securesms.migrations.LocalArchiveReconciliationMigrationJob;
 import org.thoughtcrime.securesms.migrations.MigrationCompleteJob;
 import org.thoughtcrime.securesms.migrations.OptimizeMessageSearchIndexMigrationJob;
 import org.thoughtcrime.securesms.migrations.PassingMigrationJob;
@@ -344,6 +345,7 @@ public final class JobManagerFactories {
      put(IdentityTableCleanupMigrationJob.KEY,           new IdentityTableCleanupMigrationJob.Factory());
      put(KeyTransparencyUsernameMigrationJob.KEY,        new KeyTransparencyUsernameMigrationJob.Factory());
      put(LegacyMigrationJob.KEY,                         new LegacyMigrationJob.Factory());
+      put(LocalArchiveReconciliationMigrationJob.KEY,     new LocalArchiveReconciliationMigrationJob.Factory());
      put(MigrationCompleteJob.KEY,                       new MigrationCompleteJob.Factory());
      put(OptimizeMessageSearchIndexMigrationJob.KEY,     new OptimizeMessageSearchIndexMigrationJob.Factory());
      put(PinOptOutMigration.KEY,                         new PinOptOutMigration.Factory());
@@ -410,6 +410,7 @@ class RestoreAttachmentJob private constructor(
            inputStream = input,
            offloadRestoredAt = if (manual) System.currentTimeMillis().milliseconds else null,
            archiveRestore = true,
+            restoredFromArchiveCdn = useArchiveCdn,
            notify = manual
          )
        ArchiveDatabaseExecutor.throttledNotifyAttachmentAndChatListObservers()
@@ -21,6 +21,7 @@ import org.thoughtcrime.securesms.database.SignalDatabase
 import org.thoughtcrime.securesms.dependencies.AppDependencies
 import org.thoughtcrime.securesms.jobmanager.Job
 import org.thoughtcrime.securesms.jobs.protos.RestoreLocalAttachmentJobData
+import org.thoughtcrime.securesms.keyvalue.SignalStore
 import org.thoughtcrime.securesms.mms.MmsException
 import org.whispersystems.signalservice.api.crypto.AttachmentCipherInputStream
 import org.whispersystems.signalservice.api.crypto.AttachmentCipherInputStream.IntegrityCheck
@@ -46,6 +47,8 @@ class RestoreLocalAttachmentJob private constructor(
    fun enqueueRestoreLocalAttachmentsJobs(mediaNameToFileInfo: Map<String, DocumentFileInfo>) {
      val jobManager = AppDependencies.jobManager

+      SignalStore.backup.localRestoreReconcilePending = true
+
      val orphanedCount = SignalDatabase.attachments.markRestorableAttachmentsWithoutMessageAsFailed()
      if (orphanedCount > 0) {
        Log.w(TAG, "Failed $orphanedCount orphaned restorable attachment(s) with no backing message before enqueueing restores.")
@@ -85,6 +85,7 @@ class BackupValues(store: KeyValueStore) : SignalStoreValues(store) {
    private const val KEY_BACKUP_DELETION_STATE = "backup.deletion.state"
    private const val KEY_REMOTE_STORAGE_GARBAGE_COLLECTION_PENDING = "backup.remoteStorageGarbageCollectionPending"
    private const val KEY_ARCHIVE_ATTACHMENT_RECONCILIATION_ATTEMPTS = "backup.archiveAttachmentReconciliationAttempts"
+    private const val KEY_LOCAL_RESTORE_RECONCILE_PENDING = "backup.localRestoreReconcilePending"

    private const val KEY_MEDIA_ROOT_BACKUP_KEY = "backup.mediaRootBackupKey"

@@ -186,6 +187,11 @@ class BackupValues(store: KeyValueStore) : SignalStoreValues(store) {

  var userManuallySkippedMediaRestore: Boolean by booleanValue(KEY_USER_MANUALLY_SKIPPED_MEDIA_RESTORE, false)

+  /**
+   * Set when a local backup restore is kicked off so that, once media restore completes, we reconcile the restored media against the archive CDN.
+   */
+  var localRestoreReconcilePending: Boolean by booleanValue(KEY_LOCAL_RESTORE_RECONCILE_PENDING, false)
+
  var backupExpiredAndDowngraded: Boolean by booleanValue(KEY_BACKUP_EXPIRED_AND_DOWNGRADED, false)

  /**
@@ -43,6 +43,7 @@ class LogSectionRemoteBackups : LogSection {
    output.append("Optimize storage                    : ${SignalStore.backup.optimizeStorage}\n")
    output.append("Detected subscription state mismatch: ${SignalStore.backup.subscriptionStateMismatchDetected}\n")
    output.append("Last verified key time              : ${SignalStore.backup.lastVerifyKeyTime}\n")
+    output.append("Local restore reconcile pending     : ${SignalStore.backup.localRestoreReconcilePending}\n")
    output.append("Restore state                       : ${ArchiveRestoreProgress.state}\n")
    output.append("\n -- Subscription State\n")

@@ -207,9 +207,10 @@ public class ApplicationMigrations {
    static final int NOTIFICATION_STATE_CLEANUP    = 163;
    static final int KT_USERNAME_CAPABILITY        = 164;
    static final int FIX_CHANGE_NUMBER_ERROR_2     = 165;
+    static final int LOCAL_ARCHIVE_RECONCILE       = 166;
  }

-  public static final int CURRENT_VERSION = 165;
+  public static final int CURRENT_VERSION = 166;

  /**
   * This *must* be called after the {@link JobManager} has been instantiated, but *before* the call
@@ -960,6 +961,10 @@ public class ApplicationMigrations {
      jobs.put(Version.KT_USERNAME_CAPABILITY, new KeyTransparencyUsernameMigrationJob());
    }

+    if (lastSeenVersion < Version.LOCAL_ARCHIVE_RECONCILE) {
+      jobs.put(Version.LOCAL_ARCHIVE_RECONCILE, new LocalArchiveReconciliationMigrationJob());
+    }
+
    return jobs;
  }

@@ -0,0 +1,61 @@
+/*
+ * Copyright 2026 Signal Messenger, LLC
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+package org.thoughtcrime.securesms.migrations
+
+import org.signal.core.util.logging.Log
+import org.thoughtcrime.securesms.database.SignalDatabase
+import org.thoughtcrime.securesms.jobmanager.Job
+import org.thoughtcrime.securesms.jobs.ArchiveAttachmentReconciliationJob
+import org.thoughtcrime.securesms.keyvalue.SignalStore
+
+/**
+ * There was a bug where media restored from a local backup was incorrectly marked as already being on the archive CDN, which prevented it from ever being
+ * uploaded. This migration repairs that state:
+ *
+ * - Non-media-backup users can't have anything legitimately on the CDN, so we just reset the bogus state locally; the normal backfill re-uploads it if they
+ *   later enable media backups.
+ * - Media-backup users may have some of that media genuinely on the CDN, so we can't tell locally which entries are bogus, and instead reconcile against it.
+ *
+ * Reconciliation is expensive server-side, so we only expedite it for media-backup users who are actually in the bad state (i.e. still have local-restore media
+ * marked as archived), rather than for everyone.
+ */
+internal class LocalArchiveReconciliationMigrationJob(
+  parameters: Parameters = Parameters.Builder().build()
+) : MigrationJob(parameters) {
+
+  companion object {
+    val TAG = Log.tag(LocalArchiveReconciliationMigrationJob::class.java)
+    const val KEY = "LocalArchiveReconciliationMigrationJob"
+  }
+
+  override fun getFactoryKey(): String = KEY
+
+  override fun isUiBlocking(): Boolean = false
+
+  override fun performMigration() {
+    if (!SignalStore.backup.backsUpMedia) {
+      val resetCount = SignalDatabase.attachments.resetArchiveTransferStateForLocalBackupMedia()
+      Log.i(TAG, "User does not back up media. Reset $resetCount local-restore attachment(s) incorrectly marked as archived so they'll upload if media backups are enabled later.")
+      return
+    }
+
+    if (!SignalDatabase.attachments.hasArchiveFinishedLocalBackupMedia()) {
+      Log.i(TAG, "No archive-finished media from a local backup. Not in the bad state, so skipping.")
+      return
+    }
+
+    Log.i(TAG, "Expediting an archive reconciliation to repair any media incorrectly marked as archived after a local restore.")
+    ArchiveAttachmentReconciliationJob.enqueueReconcileFirstForLocalRestore()
+  }
+
+  override fun shouldRetry(e: Exception): Boolean = false
+
+  class Factory : Job.Factory<LocalArchiveReconciliationMigrationJob> {
+    override fun create(parameters: Parameters, serializedData: ByteArray?): LocalArchiveReconciliationMigrationJob {
+      return LocalArchiveReconciliationMigrationJob(parameters)
+    }
+  }
+}