Add a job to backfill attachment uploads to the archive service.

app/src/main/java/org/thoughtcrime/securesms/database/AttachmentTable.kt

@@ -98,7 +98,6 @@ import java.security.NoSuchAlgorithmException
 import java.util.LinkedList
 import java.util.Optional
 import java.util.UUID
-import kotlin.time.Duration.Companion.days
 
 class AttachmentTable(
   context: Context,
@@ -147,6 +146,7 @@ class AttachmentTable(
     const val ARCHIVE_MEDIA_NAME = "archive_media_name"
     const val ARCHIVE_MEDIA_ID = "archive_media_id"
     const val ARCHIVE_TRANSFER_FILE = "archive_transfer_file"
+    const val ARCHIVE_TRANSFER_STATE = "archive_transfer_state"
 
     const val ATTACHMENT_JSON_ALIAS = "attachment_json"
 
@@ -201,7 +201,8 @@ class AttachmentTable(
       ARCHIVE_TRANSFER_FILE
     )
 
-    const val CREATE_TABLE = """
+    @JvmField
+    val CREATE_TABLE = """
       CREATE TABLE $TABLE_NAME (
         $ID INTEGER PRIMARY KEY AUTOINCREMENT,
         $MESSAGE_ID INTEGER,
@@ -239,7 +240,8 @@ class AttachmentTable(
         $ARCHIVE_CDN INTEGER DEFAULT 0,
         $ARCHIVE_MEDIA_NAME TEXT DEFAULT NULL,
         $ARCHIVE_MEDIA_ID TEXT DEFAULT NULL,
-        $ARCHIVE_TRANSFER_FILE TEXT DEFAULT NULL
+        $ARCHIVE_TRANSFER_FILE TEXT DEFAULT NULL,
+        $ARCHIVE_TRANSFER_STATE INTEGER DEFAULT ${ArchiveTransferState.NONE.value}
       )
       """
 
@@ -254,8 +256,6 @@ class AttachmentTable(
       "CREATE INDEX IF NOT EXISTS attachment_archive_media_id_index ON $TABLE_NAME ($ARCHIVE_MEDIA_ID);"
     )
 
-    val ATTACHMENT_POINTER_REUSE_THRESHOLD = 7.days.inWholeMilliseconds
-
     @JvmStatic
     @Throws(IOException::class)
     fun newDataFile(context: Context): File {
@@ -426,6 +426,78 @@ class AttachmentTable(
       }.flatten()
   }
 
+  /**
+   * Finds the next eligible attachment that needs to be uploaded to the archive service.
+   * If it exists, it'll also atomically be marked as [ArchiveTransferState.BACKFILL_UPLOAD_IN_PROGRESS].
+   */
+  fun getNextAttachmentToArchiveAndMarkUploadInProgress(): DatabaseAttachment? {
+    return writableDatabase.withinTransaction {
+      val record: DatabaseAttachment? = readableDatabase
+        .select(*PROJECTION)
+        .from(TABLE_NAME)
+        .where("$ARCHIVE_TRANSFER_STATE = ? AND $DATA_FILE NOT NULL AND $TRANSFER_STATE = $TRANSFER_PROGRESS_DONE", ArchiveTransferState.NONE.value)
+        .orderBy("$ID DESC")
+        .limit(1)
+        .run()
+        .readToSingleObject { it.readAttachment() }
+
+      if (record != null) {
+        writableDatabase
+          .update(TABLE_NAME)
+          .values(ARCHIVE_TRANSFER_STATE to ArchiveTransferState.BACKFILL_UPLOAD_IN_PROGRESS.value)
+          .where("$ID = ?", record.attachmentId)
+          .run()
+      }
+
+      record
+    }
+  }
+
+  /**
+   * Returns the current archive transfer state, if the attachment can be found.
+   */
+  fun getArchiveTransferState(id: AttachmentId): ArchiveTransferState? {
+    return readableDatabase
+      .select(ARCHIVE_TRANSFER_STATE)
+      .from(TABLE_NAME)
+      .where("$ID = ?", id.id)
+      .run()
+      .readToSingleObject { ArchiveTransferState.deserialize(it.requireInt(ARCHIVE_TRANSFER_STATE)) }
+  }
+
+  /**
+   * Sets the archive transfer state for the given attachment and all other attachments that share the same data file.
+   */
+  fun setArchiveTransferState(id: AttachmentId, state: ArchiveTransferState) {
+    writableDatabase.withinTransaction {
+      val dataFile: String = readableDatabase
+        .select(DATA_FILE)
+        .from(TABLE_NAME)
+        .where("$ID = ?", id.id)
+        .run()
+        .readToSingleObject { it.requireString(DATA_FILE) } ?: return@withinTransaction
+
+      writableDatabase
+        .update(TABLE_NAME)
+        .values(ARCHIVE_TRANSFER_STATE to state.value)
+        .where("$DATA_FILE = ?", dataFile)
+        .run()
+    }
+  }
+
+  /**
+   * Resets any in-progress archive backfill states to [ArchiveTransferState.NONE], returning the number that had to be reset.
+   * This should only be called if you believe the backfill process has finished. In this case, if this returns a value > 0,
+   * it indicates that state was mis-tracked and you should try uploading again.
+   */
+  fun resetPendingArchiveBackfills(): Int {
+    return writableDatabase
+      .update(TABLE_NAME)
+      .values(ARCHIVE_TRANSFER_STATE to ArchiveTransferState.NONE.value)
+      .where("$ARCHIVE_TRANSFER_STATE == ${ArchiveTransferState.BACKFILL_UPLOAD_IN_PROGRESS.value} || $ARCHIVE_TRANSFER_STATE == ${ArchiveTransferState.BACKFILL_UPLOADED.value}")
+      .run()
+  }
+
   fun deleteAttachmentsForMessage(mmsId: Long): Boolean {
     Log.d(TAG, "[deleteAttachmentsForMessage] mmsId: $mmsId")
 
@@ -1992,4 +2064,44 @@ class AttachmentTable(
       }
     }
   }
+
+  /**
+   * This maintains two different state paths for uploading attachments to the archive.
+   *
+   * The first is the backfill process, which will happen after newly-enabling backups. That process will go:
+   * 1. [NONE]
+   * 2. [BACKFILL_UPLOAD_IN_PROGRESS]
+   * 3. [BACKFILL_UPLOADED]
+   * 4. [FINISHED] or [PERMANENT_FAILURE]
+   *
+   * The second is when newly sending/receiving an attachment after enabling backups. That process will go:
+   * 1. [NONE]
+   * 2. [ATTACHMENT_TRANSFER_PENDING]
+   * 3. [FINISHED] or [PERMANENT_FAILURE]
+   */
+  enum class ArchiveTransferState(val value: Int) {
+    /** Not backed up at all. */
+    NONE(0),
+
+    /** The upload to the attachment service is in progress. */
+    BACKFILL_UPLOAD_IN_PROGRESS(1),
+
+    /** Successfully uploaded to the attachment service during the backfill process. Still need to tell the service to move the file over to the archive service. */
+    BACKFILL_UPLOADED(2),
+
+    /** Completely finished backing up the attachment. */
+    FINISHED(3),
+
+    /** It is impossible to upload this attachment. */
+    PERMANENT_FAILURE(4),
+
+    /** We sent/received this attachment after enabling backups, but still need to transfer the file to the archive service. */
+    ATTACHMENT_TRANSFER_PENDING(5);
+
+    companion object {
+      fun deserialize(value: Int): ArchiveTransferState {
+        return values().firstOrNull { it.value == value } ?: NONE
+      }
+    }
+  }
 }

app/src/main/java/org/thoughtcrime/securesms/database/helpers/SignalDatabaseMigrations.kt

@@ -84,6 +84,7 @@ import org.thoughtcrime.securesms.database.helpers.migration.V223_AddNicknameAnd
 import org.thoughtcrime.securesms.database.helpers.migration.V224_AddAttachmentArchiveColumns
 import org.thoughtcrime.securesms.database.helpers.migration.V225_AddLocalUserJoinedStateAndGroupCallActiveState
 import org.thoughtcrime.securesms.database.helpers.migration.V226_AddAttachmentMediaIdIndex
+import org.thoughtcrime.securesms.database.helpers.migration.V227_AddAttachmentArchiveTransferState
 
 /**
  * Contains all of the database migrations for [SignalDatabase]. Broken into a separate file for cleanliness.
@@ -170,10 +171,11 @@ object SignalDatabaseMigrations {
     223 to V223_AddNicknameAndNoteFieldsToRecipientTable,
     224 to V224_AddAttachmentArchiveColumns,
     225 to V225_AddLocalUserJoinedStateAndGroupCallActiveState,
-    226 to V226_AddAttachmentMediaIdIndex
+    226 to V226_AddAttachmentMediaIdIndex,
+    227 to V227_AddAttachmentArchiveTransferState
   )
 
-  const val DATABASE_VERSION = 226
+  const val DATABASE_VERSION = 227
 
   @JvmStatic
   fun migrate(context: Application, db: SQLiteDatabase, oldVersion: Int, newVersion: Int) {

app/src/main/java/org/thoughtcrime/securesms/database/helpers/migration/V227_AddAttachmentArchiveTransferState.kt

@@ -0,0 +1,19 @@
+/*
+ * Copyright 2024 Signal Messenger, LLC
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+package org.thoughtcrime.securesms.database.helpers.migration
+
+import android.app.Application
+import net.zetetic.database.sqlcipher.SQLiteDatabase
+
+/**
+ * Adds a new column to track the status of transferring attachments to the archive service.
+ */
+object V227_AddAttachmentArchiveTransferState : SignalDatabaseMigration {
+
+  override fun migrate(context: Application, db: SQLiteDatabase, oldVersion: Int, newVersion: Int) {
+    db.execSQL("ALTER TABLE attachment ADD COLUMN archive_transfer_state INTEGER DEFAULT 0")
+  }
+}