Add key reuse to create keys operation in backup job.

app/src/main/java/org/thoughtcrime/securesms/database/AttachmentTable.kt

@@ -172,7 +172,6 @@ class AttachmentTable(
     const val DISPLAY_ORDER = "display_order"
     const val UPLOAD_TIMESTAMP = "upload_timestamp"
     const val ARCHIVE_CDN = "archive_cdn"
-    const val ARCHIVE_TRANSFER_FILE = "archive_transfer_file"
     const val ARCHIVE_TRANSFER_STATE = "archive_transfer_state"
     const val THUMBNAIL_RESTORE_STATE = "thumbnail_restore_state"
     const val ATTACHMENT_UUID = "attachment_uuid"
@@ -652,10 +651,13 @@ class AttachmentTable(
    * At archive creation time, we need to ensure that all relevant attachments have populated [REMOTE_KEY]s.
    * This does that.
    */
-  fun createRemoteKeyForAttachmentsThatNeedArchiveUpload(): Int {
-    var count = 0
+  fun createRemoteKeyForAttachmentsThatNeedArchiveUpload(): CreateRemoteKeyResult {
+    var totalCount = 0
+    var notQuoteOrStickerDupeNotFoundCount = 0
+    var notQuoteOrStickerDupeFoundCount = 0
 
-    writableDatabase.select(ID, REMOTE_KEY, DATA_FILE, DATA_RANDOM)
+    val missingKeys = readableDatabase
+      .select(ID, DATA_FILE, QUOTE, STICKER_ID)
       .from(TABLE_NAME)
       .where(
         """
@@ -666,21 +668,75 @@ class AttachmentTable(
         """
       )
       .run()
-      .forEach { cursor ->
-        val attachmentId = AttachmentId(cursor.requireLong(ID))
-        Log.w(TAG, "[createRemoteKeyForAttachmentsThatNeedArchiveUpload][$attachmentId] Missing key. Generating.")
+      .readToList { Triple(AttachmentId(it.requireLong(ID)), it.requireBoolean(QUOTE), it.requireInt(STICKER_ID) >= 0) to it.requireNonNullString(DATA_FILE) }
+      .groupBy({ (_, dataFile) -> dataFile }, { (record, _) -> record })
 
-        val key = cursor.requireString(REMOTE_KEY)?.let { Base64.decode(it) } ?: Util.getSecretBytes(64)
+    missingKeys.forEach { dataFile, ids ->
+      val duplicateAttachmentWithRemoteData = readableDatabase
+        .select()
+        .from(TABLE_NAME)
+        .where("$DATA_FILE = ? AND $DATA_RANDOM NOT NULL AND $REMOTE_KEY NOT NULL AND $REMOTE_LOCATION NOT NULL AND $REMOTE_DIGEST NOT NULL", dataFile)
+        .orderBy("$ID DESC")
+        .limit(1)
+        .run()
+        .readToSingleObject { cursor ->
+          val duplicateAttachment = cursor.readAttachment()
+          val dataFileInfo = cursor.readDataFileInfo()!!
 
-        writableDatabase.update(TABLE_NAME)
-          .values(REMOTE_KEY to Base64.encodeWithPadding(key))
-          .where("$ID = ?", attachmentId.id)
-          .run()
+          duplicateAttachment to dataFileInfo
+        }
 
-        count++
+      if (duplicateAttachmentWithRemoteData != null) {
+        val (duplicateAttachment, duplicateAttachmentDataInfo) = duplicateAttachmentWithRemoteData
+
+        ids.forEach { (attachmentId, isQuote, isSticker) ->
+          Log.w(TAG, "[createRemoteKeyForAttachmentsThatNeedArchiveUpload][$attachmentId] Missing key but found same data file with remote data. Updating. isQuote:$isQuote isSticker:$isSticker")
+
+          writableDatabase
+            .update(TABLE_NAME)
+            .values(
+              REMOTE_KEY to duplicateAttachment.remoteKey,
+              REMOTE_LOCATION to duplicateAttachment.remoteLocation,
+              REMOTE_DIGEST to duplicateAttachment.remoteDigest,
+              REMOTE_INCREMENTAL_DIGEST to duplicateAttachment.incrementalDigest?.takeIf { it.isNotEmpty() },
+              REMOTE_INCREMENTAL_DIGEST_CHUNK_SIZE to duplicateAttachment.incrementalMacChunkSize,
+              UPLOAD_TIMESTAMP to duplicateAttachment.uploadTimestamp,
+              ARCHIVE_CDN to duplicateAttachment.archiveCdn,
+              ARCHIVE_TRANSFER_STATE to duplicateAttachment.archiveTransferState.value,
+              THUMBNAIL_FILE to duplicateAttachmentDataInfo.thumbnailFile,
+              THUMBNAIL_RANDOM to duplicateAttachmentDataInfo.thumbnailRandom,
+              THUMBNAIL_RESTORE_STATE to duplicateAttachmentDataInfo.thumbnailRestoreState
+            )
+            .where("$ID = ?", attachmentId.id)
+            .run()
+
+          if (!isQuote && !isSticker) {
+            notQuoteOrStickerDupeFoundCount++
+          }
+
+          totalCount++
+        }
+      } else {
+        ids.forEach { (attachmentId, isQuote, isSticker) ->
+          Log.w(TAG, "[createRemoteKeyForAttachmentsThatNeedArchiveUpload][$attachmentId] Missing key. Generating. isQuote:$isQuote isSticker:$isSticker")
+
+          val key = Util.getSecretBytes(64)
+
+          writableDatabase.update(TABLE_NAME)
+            .values(REMOTE_KEY to Base64.encodeWithPadding(key))
+            .where("$ID = ?", attachmentId.id)
+            .run()
+
+          totalCount++
+
+          if (!isQuote && !isSticker) {
+            notQuoteOrStickerDupeNotFoundCount++
+          }
+        }
       }
+    }
 
-    return count
+    return CreateRemoteKeyResult(totalCount, notQuoteOrStickerDupeNotFoundCount, notQuoteOrStickerDupeFoundCount)
   }
 
   /**
@@ -3088,4 +3144,8 @@ class AttachmentTable(
     val validForArchiveTransferStateCounts: Map<String, Long>,
     val estimatedThumbnailCount: Long
   )
+
+  data class CreateRemoteKeyResult(val totalCount: Int, val notQuoteOrSickerDupeNotFoundCount: Int, val notQuoteOrSickerDupeFoundCount: Int) {
+    val unexpectedKeyCreation = notQuoteOrSickerDupeFoundCount > 0 || notQuoteOrSickerDupeNotFoundCount > 0
+  }
 }

app/src/main/java/org/thoughtcrime/securesms/database/helpers/SignalDatabaseMigrations.kt

@@ -141,6 +141,7 @@ import org.thoughtcrime.securesms.database.helpers.migration.V283_ViewOnceRemote
 import org.thoughtcrime.securesms.database.helpers.migration.V284_SetPlaceholderGroupFlag
 import org.thoughtcrime.securesms.database.helpers.migration.V285_AddEpochToCallLinksTable
 import org.thoughtcrime.securesms.database.helpers.migration.V286_FixRemoteKeyEncoding
+import org.thoughtcrime.securesms.database.helpers.migration.V287_FixInvalidArchiveState
 import org.thoughtcrime.securesms.database.SQLiteDatabase as SignalSqliteDatabase
 
 /**
@@ -287,10 +288,11 @@ object SignalDatabaseMigrations {
     283 to V283_ViewOnceRemoteDataCleanup,
     284 to V284_SetPlaceholderGroupFlag,
     285 to V285_AddEpochToCallLinksTable,
-    286 to V286_FixRemoteKeyEncoding
+    286 to V286_FixRemoteKeyEncoding,
+    287 to V287_FixInvalidArchiveState
   )
 
-  const val DATABASE_VERSION = 286
+  const val DATABASE_VERSION = 287
 
   @JvmStatic
   fun migrate(context: Application, db: SignalSqliteDatabase, oldVersion: Int, newVersion: Int) {

app/src/main/java/org/thoughtcrime/securesms/database/helpers/migration/V287_FixInvalidArchiveState.kt

@@ -0,0 +1,19 @@
+/*
+ * Copyright 2024 Signal Messenger, LLC
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+package org.thoughtcrime.securesms.database.helpers.migration
+
+import android.app.Application
+import org.thoughtcrime.securesms.database.SQLiteDatabase
+
+/**
+ * Ensure archive_transfer_state is clear if an attachment is missing a remote_key.
+ */
+@Suppress("ClassName")
+object V287_FixInvalidArchiveState : SignalDatabaseMigration {
+  override fun migrate(context: Application, db: SQLiteDatabase, oldVersion: Int, newVersion: Int) {
+    db.execSQL("UPDATE attachment SET archive_cdn = null, archive_transfer_state = 0 WHERE remote_key IS NULL AND archive_transfer_state = 3")
+  }
+}

app/src/main/java/org/thoughtcrime/securesms/jobs/BackupMessagesJob.kt

@@ -5,6 +5,12 @@
 
 package org.thoughtcrime.securesms.jobs
 
+import android.app.Notification
+import android.app.PendingIntent
+import android.content.Intent
+import androidx.core.app.NotificationCompat
+import androidx.core.app.NotificationManagerCompat
+import org.signal.core.util.PendingIntentFlags
 import org.signal.core.util.Stopwatch
 import org.signal.core.util.isNotNullOrBlank
 import org.signal.core.util.logging.Log
@@ -12,6 +18,7 @@ import org.signal.core.util.logging.logW
 import org.signal.libsignal.messagebackup.BackupForwardSecrecyToken
 import org.signal.libsignal.net.SvrBStoreResponse
 import org.signal.protos.resumableuploads.ResumableUpload
+import org.thoughtcrime.securesms.R
 import org.thoughtcrime.securesms.backup.ArchiveUploadProgress
 import org.thoughtcrime.securesms.backup.RestoreState
 import org.thoughtcrime.securesms.backup.v2.ArchiveMediaItemIterator
@@ -25,7 +32,10 @@ import org.thoughtcrime.securesms.jobmanager.impl.BackupMessagesConstraint
 import org.thoughtcrime.securesms.jobs.protos.BackupMessagesJobData
 import org.thoughtcrime.securesms.keyvalue.SignalStore
 import org.thoughtcrime.securesms.keyvalue.isDecisionPending
+import org.thoughtcrime.securesms.logsubmit.SubmitDebugLogActivity
 import org.thoughtcrime.securesms.net.SignalNetwork
+import org.thoughtcrime.securesms.notifications.NotificationChannels
+import org.thoughtcrime.securesms.notifications.NotificationIds
 import org.thoughtcrime.securesms.providers.BlobProvider
 import org.thoughtcrime.securesms.recipients.Recipient
 import org.thoughtcrime.securesms.storage.StorageSyncHelper
@@ -179,7 +189,13 @@ class BackupMessagesJob private constructor(
     Log.i(TAG, "Successfully stored data on SVRB.")
     stopwatch.split("svrb")
 
-    SignalDatabase.attachments.createRemoteKeyForAttachmentsThatNeedArchiveUpload().takeIf { it > 0 }?.let { count -> Log.w(TAG, "Needed to create $count remote keys.") }
+    val createKeyResult = SignalDatabase.attachments.createRemoteKeyForAttachmentsThatNeedArchiveUpload()
+    if (createKeyResult.totalCount > 0) {
+      Log.w(TAG, "Needed to create remote keys. $createKeyResult")
+      if (createKeyResult.unexpectedKeyCreation) {
+        maybePostRemoteKeyMissingNotification()
+      }
+    }
     stopwatch.split("keygen")
 
     SignalDatabase.attachments.clearIncrementalMacsForAttachmentsThatNeedArchiveUpload().takeIf { it > 0 }?.let { count -> Log.w(TAG, "Needed to clear $count incrementalMacs.") }
@@ -409,6 +425,21 @@ class BackupMessagesJob private constructor(
     }
   }
 
+  private fun maybePostRemoteKeyMissingNotification() {
+    if (!RemoteConfig.internalUser || !SignalStore.backup.backsUpMedia) {
+      return
+    }
+
+    val notification: Notification = NotificationCompat.Builder(context, NotificationChannels.getInstance().FAILURES)
+      .setSmallIcon(R.drawable.ic_notification)
+      .setContentTitle("[Internal-only] Unexpected remote key missing!")
+      .setContentText("Tap to send a debug log")
+      .setContentIntent(PendingIntent.getActivity(context, 0, Intent(context, SubmitDebugLogActivity::class.java), PendingIntentFlags.mutable()))
+      .build()
+
+    NotificationManagerCompat.from(context).notify(NotificationIds.INTERNAL_ERROR, notification)
+  }
+
   class Factory : Job.Factory<BackupMessagesJob> {
     override fun create(parameters: Parameters, serializedData: ByteArray?): BackupMessagesJob {
       val jobData = if (serializedData != null) {