Attempt backing up a subset of messages if you hit the limit.

2026-02-21 02:08:40 +00:00 · 2025-11-11 14:48:09 -05:00
parent f4e82e6aab
commit b047f8bc0a
11 changed files with 112 additions and 24 deletions
--- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/BackupRepository.kt
+++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/BackupRepository.kt
@@ -417,7 +417,7 @@ object BackupRepository {
  }

  fun clearBackupFailure() {
-    SignalStore.backup.clearBackupCreationFailed()
+    SignalStore.backup.backupCreationError = null
    ServiceUtil.getNotificationManager(AppDependencies.application).cancel(NotificationIds.INITIAL_BACKUP_FAILED)
  }

@@ -767,7 +767,8 @@ object BackupRepository {
      progressEmitter = localBackupProgressEmitter,
      cancellationSignal = cancellationSignal,
      forTransfer = false,
-      extraFrameOperation = null
+      extraFrameOperation = null,
+      messageInclusionCutoffTime = 0
    ) { dbSnapshot ->
      val localArchivableAttachments = dbSnapshot
        .attachmentTable
@@ -801,6 +802,7 @@ object BackupRepository {
    forwardSecrecyToken: BackupForwardSecrecyToken,
    forwardSecrecyMetadata: ByteArray,
    currentTime: Long,
+    messageInclusionCutoffTime: Long = 0,
    progressEmitter: ExportProgressListener? = null,
    cancellationSignal: () -> Boolean = { false },
    extraFrameOperation: ((Frame) -> Unit)?
@@ -822,7 +824,8 @@ object BackupRepository {
      progressEmitter = progressEmitter,
      cancellationSignal = cancellationSignal,
      extraFrameOperation = extraFrameOperation,
-      endingExportOperation = null
+      endingExportOperation = null,
+      messageInclusionCutoffTime = messageInclusionCutoffTime
    )
  }

@@ -852,7 +855,8 @@ object BackupRepository {
      progressEmitter = progressEmitter,
      cancellationSignal = cancellationSignal,
      extraFrameOperation = null,
-      endingExportOperation = null
+      endingExportOperation = null,
+      messageInclusionCutoffTime = 0
    )
  }

@@ -887,7 +891,8 @@ object BackupRepository {
      progressEmitter = progressEmitter,
      cancellationSignal = cancellationSignal,
      extraFrameOperation = null,
-      endingExportOperation = null
+      endingExportOperation = null,
+      messageInclusionCutoffTime = 0
    )
  }

@@ -907,6 +912,7 @@ object BackupRepository {
    isLocal: Boolean,
    writer: BackupExportWriter,
    forTransfer: Boolean,
+    messageInclusionCutoffTime: Long,
    progressEmitter: ExportProgressListener?,
    cancellationSignal: () -> Boolean,
    extraFrameOperation: ((Frame) -> Unit)?,
@@ -1033,7 +1039,7 @@ object BackupRepository {
          val approximateMessageCount = dbSnapshot.messageTable.getApproximateExportableMessageCount(exportState.threadIds)
          val frameCountStart = frameCount
          progressEmitter?.onMessage(0, approximateMessageCount)
-          ChatItemArchiveProcessor.export(dbSnapshot, exportState, selfRecipientId, cancellationSignal) { frame ->
+          ChatItemArchiveProcessor.export(dbSnapshot, exportState, selfRecipientId, messageInclusionCutoffTime, cancellationSignal) { frame ->
            writer.write(frame)
            extraFrameOperation?.invoke(frame)
            eventTimer.emit("message")
--- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/MessageTableArchiveExtensions.kt
+++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/MessageTableArchiveExtensions.kt
@@ -24,7 +24,7 @@ import kotlin.time.Duration.Companion.days

 private val TAG = "MessageTableArchiveExtensions"

-fun MessageTable.getMessagesForBackup(db: SignalDatabase, backupTime: Long, selfRecipientId: RecipientId, exportState: ExportState): ChatItemArchiveExporter {
+fun MessageTable.getMessagesForBackup(db: SignalDatabase, backupTime: Long, selfRecipientId: RecipientId, messageInclusionCutoffTime: Long, exportState: ExportState): ChatItemArchiveExporter {
  // We create a covering index for the query to drastically speed up perf here.
  // Remember that we're working on a temporary snapshot of the database, so we can create an index and not worry about cleaning it up.
  val startTime = System.currentTimeMillis()
@@ -105,6 +105,12 @@ fun MessageTable.getMessagesForBackup(db: SignalDatabase, backupTime: Long, self
  )
  Log.d(TAG, "Cleanup took ${System.currentTimeMillis() - cleanupStartTime} ms")

+  val cutoffQuery = if (messageInclusionCutoffTime > 0) {
+    " AND $DATE_RECEIVED >= $messageInclusionCutoffTime"
+  } else {
+    ""
+  }
+
  return ChatItemArchiveExporter(
    db = db,
    selfRecipientId = selfRecipientId,
@@ -152,7 +158,7 @@ fun MessageTable.getMessagesForBackup(db: SignalDatabase, backupTime: Long, self
          PARENT_STORY_ID
        )
        .from("${MessageTable.TABLE_NAME} INDEXED BY $dateReceivedIndex")
-        .where("$STORY_TYPE = 0 AND $PARENT_STORY_ID <= 0 AND $SCHEDULED_DATE = -1 AND ($EXPIRES_IN == 0 OR $EXPIRES_IN > ${1.days.inWholeMilliseconds}) AND $DATE_RECEIVED >= $lastSeenReceivedTime")
+        .where("$STORY_TYPE = 0 AND $PARENT_STORY_ID <= 0 AND $SCHEDULED_DATE = -1 AND ($EXPIRES_IN == 0 OR $EXPIRES_IN > ${1.days.inWholeMilliseconds}) AND $DATE_RECEIVED >= $lastSeenReceivedTime $cutoffQuery")
        .limit(count)
        .orderBy("$DATE_RECEIVED ASC")
        .run()
--- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/ChatItemArchiveProcessor.kt
+++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/ChatItemArchiveProcessor.kt
@@ -23,8 +23,8 @@ import org.thoughtcrime.securesms.recipients.RecipientId
 object ChatItemArchiveProcessor {
  val TAG = Log.tag(ChatItemArchiveProcessor::class.java)

-  fun export(db: SignalDatabase, exportState: ExportState, selfRecipientId: RecipientId, cancellationSignal: () -> Boolean, emitter: BackupFrameEmitter) {
-    db.messageTable.getMessagesForBackup(db, exportState.backupTime, selfRecipientId, exportState).use { chatItems ->
+  fun export(db: SignalDatabase, exportState: ExportState, selfRecipientId: RecipientId, messageInclusionCutoffTime: Long, cancellationSignal: () -> Boolean, emitter: BackupFrameEmitter) {
+    db.messageTable.getMessagesForBackup(db, exportState.backupTime, selfRecipientId, messageInclusionCutoffTime, exportState).use { chatItems ->
      var count = 0
      while (chatItems.hasNext()) {
        if (count % 1000 == 0 && cancellationSignal()) {
--- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/ui/status/BackupCreateErrorRow.kt
+++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/ui/status/BackupCreateErrorRow.kt
@@ -19,6 +19,7 @@ import androidx.compose.material3.Text
 import androidx.compose.runtime.Composable
 import androidx.compose.ui.Modifier
 import androidx.compose.ui.graphics.Color
+import androidx.compose.ui.platform.LocalContext
 import androidx.compose.ui.res.dimensionResource
 import androidx.compose.ui.res.stringResource
 import androidx.compose.ui.text.AnnotatedString.Builder
@@ -35,6 +36,9 @@ import org.signal.core.ui.compose.DayNightPreviews
 import org.signal.core.ui.compose.Previews
 import org.thoughtcrime.securesms.R
 import org.thoughtcrime.securesms.keyvalue.BackupValues
+import org.thoughtcrime.securesms.util.DateUtils
+import java.util.Locale
+import kotlin.time.Duration.Companion.days
 import org.signal.core.ui.R as CoreUiR

 private val YELLOW_DOT = Color(0xFFFFCC00)
@@ -45,8 +49,12 @@ private val YELLOW_DOT = Color(0xFFFFCC00)
@Composable
 fun BackupCreateErrorRow(
  error: BackupValues.BackupCreationError,
+  lastMessageCutoffTime: Long = 0,
  onLearnMoreClick: () -> Unit = {}
 ) {
+  val context = LocalContext.current
+  val locale = Locale.getDefault()
+
  when (error) {
    BackupValues.BackupCreationError.TRANSIENT -> {
      BackupAlertText {
@@ -73,7 +81,11 @@ fun BackupCreateErrorRow(

    BackupValues.BackupCreationError.BACKUP_FILE_TOO_LARGE -> {
      BackupAlertText {
-        append(stringResource(R.string.BackupStatusRow__backup_file_too_large))
+        if (lastMessageCutoffTime > 0) {
+          append(stringResource(R.string.BackupStatusRow__not_backing_up_old_messages, DateUtils.getDayPrecisionTimeString(context, locale, lastMessageCutoffTime)))
+        } else {
+          append(stringResource(R.string.BackupStatusRow__backup_file_too_large))
+        }
      }
    }
  }
@@ -114,6 +126,10 @@ fun BackupStatusRowCouldNotCompleteBackupPreview() {
        BackupCreateErrorRow(error = error, onLearnMoreClick = {})
        Spacer(modifier = Modifier.size(8.dp))
      }
+
+      Text(BackupValues.BackupCreationError.BACKUP_FILE_TOO_LARGE.name + " with cutoff duration")
+      BackupCreateErrorRow(error = BackupValues.BackupCreationError.BACKUP_FILE_TOO_LARGE, lastMessageCutoffTime = System.currentTimeMillis() - 365.days.inWholeMilliseconds, onLearnMoreClick = {})
+      Spacer(modifier = Modifier.size(8.dp))
    }
  }
 }
--- a/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/backups/remote/RemoteBackupsSettingsFragment.kt
+++ b/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/backups/remote/RemoteBackupsSettingsFragment.kt
@@ -543,6 +543,7 @@ private fun RemoteBackupsSettingsContent(
          item {
            BackupCreateErrorRow(
              error = state.backupCreationError,
+              lastMessageCutoffTime = state.lastMessageCutoffTime,
              onLearnMoreClick = contentCallbacks::onLearnMoreAboutBackupFailure
            )
          }
@@ -891,6 +892,7 @@ private fun LazyListScope.appendBackupDetailsItems(
    item {
      BackupCreateErrorRow(
        error = state.backupCreationError,
+        lastMessageCutoffTime = state.lastMessageCutoffTime,
        onLearnMoreClick = contentCallbacks::onLearnMoreAboutBackupFailure
      )
    }
--- a/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/backups/remote/RemoteBackupsSettingsState.kt
+++ b/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/backups/remote/RemoteBackupsSettingsState.kt
@@ -32,6 +32,7 @@ data class RemoteBackupsSettingsState(
  val canBackupMessagesJobRun: Boolean = false,
  val backupMediaDetails: BackupMediaDetails? = null,
  val backupCreationError: BackupValues.BackupCreationError? = null,
+  val lastMessageCutoffTime: Long = 0,
  val freeTierMediaRetentionDays: Int = -1,
  val isGooglePlayServicesAvailable: Boolean = false
 ) {
--- a/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/backups/remote/RemoteBackupsSettingsViewModel.kt
+++ b/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/backups/remote/RemoteBackupsSettingsViewModel.kt
@@ -77,7 +77,8 @@ class RemoteBackupsSettingsViewModel : ViewModel() {
      canBackUpUsingCellular = SignalStore.backup.backupWithCellular,
      canRestoreUsingCellular = SignalStore.backup.restoreWithCellular,
      includeDebuglog = SignalStore.internal.includeDebuglogInBackup.takeIf { RemoteConfig.internalUser },
-      backupCreationError = SignalStore.backup.backupCreationError
+      backupCreationError = SignalStore.backup.backupCreationError,
+      lastMessageCutoffTime = SignalStore.backup.lastUsedMessageCutoffTime
    )
  )

@@ -348,7 +349,8 @@ class RemoteBackupsSettingsViewModel : ViewModel() {
        canRestoreUsingCellular = SignalStore.backup.restoreWithCellular,
        isOutOfStorageSpace = BackupRepository.shouldDisplayOutOfRemoteStorageSpaceUx(),
        hasRedemptionError = lastPurchase?.data?.error?.data_ == "409",
-        backupCreationError = SignalStore.backup.backupCreationError
+        backupCreationError = SignalStore.backup.backupCreationError,
+        lastMessageCutoffTime = SignalStore.backup.lastUsedMessageCutoffTime
      )
    }
  }
--- a/app/src/main/java/org/thoughtcrime/securesms/jobs/BackupMessagesJob.kt
+++ b/app/src/main/java/org/thoughtcrime/securesms/jobs/BackupMessagesJob.kt
@@ -57,6 +57,7 @@ import org.whispersystems.signalservice.internal.push.AttachmentUploadForm
 import java.io.File
 import java.io.FileInputStream
 import java.io.FileOutputStream
+import kotlin.time.Duration.Companion.days
 import kotlin.time.Duration.Companion.hours
 import kotlin.time.Duration.Companion.milliseconds

@@ -75,6 +76,7 @@ class BackupMessagesJob private constructor(
    private val TAG = Log.tag(BackupMessagesJob::class.java)
    private val FILE_REUSE_TIMEOUT = 1.hours
    private const val ATTACHMENT_SNAPSHOT_BUFFER_SIZE = 10_000
+    private val TOO_LARGE_MESSAGE_CUTTOFF_DURATION = 365.days

    const val KEY = "BackupMessagesJob"

@@ -265,7 +267,7 @@ class BackupMessagesJob private constructor(
      return Result.failure()
    }

-    val (tempBackupFile, currentTime) = when (val generateBackupFileResult = getOrCreateBackupFile(stopwatch, svrBMetadata.forwardSecrecyToken, svrBMetadata.metadata)) {
+    val (tempBackupFile, currentTime, messageCutoffTime) = when (val generateBackupFileResult = getOrCreateBackupFile(stopwatch, svrBMetadata.forwardSecrecyToken, svrBMetadata.metadata)) {
      is BackupFileResult.Success -> generateBackupFileResult
      BackupFileResult.Failure -> return Result.failure()
      BackupFileResult.Retry -> return Result.retry(defaultBackoff())
@@ -294,12 +296,19 @@ class BackupMessagesJob private constructor(
      is NetworkResult.StatusCodeError -> {
        when (result.code) {
          413 -> {
-            Log.i(TAG, "Backup file is too large! Size: ${tempBackupFile.length()} bytes", result.getCause(), true)
+            Log.i(TAG, "Backup file is too large! Size: ${tempBackupFile.length()} bytes. Current threshold: ${SignalStore.backup.messageCuttoffDuration}", result.getCause(), true)
            tempBackupFile.delete()
            this.dataFile = ""
            BackupRepository.markBackupCreationFailed(BackupValues.BackupCreationError.BACKUP_FILE_TOO_LARGE)
            backupErrorHandled = true
-            return Result.failure()
+
+            if (SignalStore.backup.messageCuttoffDuration == null) {
+              Log.i(TAG, "Setting message cuttoff duration to $TOO_LARGE_MESSAGE_CUTTOFF_DURATION", true)
+              SignalStore.backup.messageCuttoffDuration = TOO_LARGE_MESSAGE_CUTTOFF_DURATION
+              return Result.retry(defaultBackoff())
+            } else {
+              return Result.failure()
+            }
          }
          else -> {
            Log.i(TAG, "Status code failure", result.getCause(), true)
@@ -394,7 +403,11 @@ class BackupMessagesJob private constructor(
      Log.i(TAG, "No thumbnails need to be uploaded: ${SignalStore.backup.backupTier}", true)
    }

-    BackupRepository.clearBackupFailure()
+    SignalStore.backup.messageCuttoffDuration = null
+    SignalStore.backup.lastUsedMessageCutoffTime = messageCutoffTime
+    if (messageCutoffTime == 0L) {
+      BackupRepository.clearBackupFailure()
+    }
    SignalDatabase.backupMediaSnapshots.commitPendingRows()

    if (SignalStore.backup.backsUpMedia) {
@@ -416,7 +429,7 @@ class BackupMessagesJob private constructor(

      if (file.exists() && file.canRead() && elapsed < FILE_REUSE_TIMEOUT) {
        Log.d(TAG, "File exists and is new enough to utilize.", true)
-        return BackupFileResult.Success(file, syncTime)
+        return BackupFileResult.Success(file, syncTime, messageInclusionCutoffTime = SignalStore.backup.lastUsedMessageCutoffTime)
      }
    }

@@ -430,6 +443,7 @@ class BackupMessagesJob private constructor(
    val currentTime = System.currentTimeMillis()

    val attachmentInfoBuffer: MutableSet<ArchiveAttachmentInfo> = mutableSetOf()
+    val messageInclusionCutoffTime = SignalStore.backup.messageCuttoffDuration?.let { currentTime - it.inWholeMilliseconds } ?: 0

    BackupRepository.exportForSignalBackup(
      outputStream = outputStream,
@@ -439,7 +453,8 @@ class BackupMessagesJob private constructor(
      progressEmitter = ArchiveUploadProgress.ArchiveBackupProgressListener,
      append = { tempBackupFile.appendBytes(it) },
      cancellationSignal = { this.isCanceled },
-      currentTime = currentTime
+      currentTime = currentTime,
+      messageInclusionCutoffTime = messageInclusionCutoffTime
    ) { frame ->
      attachmentInfoBuffer += frame.getAllReferencedArchiveAttachmentInfos()
      if (attachmentInfoBuffer.size > ATTACHMENT_SNAPSHOT_BUFFER_SIZE) {
@@ -496,7 +511,7 @@ class BackupMessagesJob private constructor(
      return BackupFileResult.Failure
    }

-    return BackupFileResult.Success(tempBackupFile, currentTime)
+    return BackupFileResult.Success(tempBackupFile, currentTime, messageInclusionCutoffTime)
  }

  private fun AttachmentUploadForm.toUploadSpec(): ResumableUpload {
@@ -601,7 +616,8 @@ class BackupMessagesJob private constructor(
  private sealed interface BackupFileResult {
    data class Success(
      val tempBackupFile: File,
-      val currentTime: Long
+      val currentTime: Long,
+      val messageInclusionCutoffTime: Long
    ) : BackupFileResult

    data object Failure : BackupFileResult
--- a/app/src/main/java/org/thoughtcrime/securesms/keyvalue/BackupValues.kt
+++ b/app/src/main/java/org/thoughtcrime/securesms/keyvalue/BackupValues.kt
@@ -96,6 +96,9 @@ class BackupValues(store: KeyValueStore) : SignalStoreValues(store) {

    private const val KEY_RESTORING_VIA_QR = "backup.restore_via_qr"

+    private const val KEY_MESSAGE_CUTOFF_DURATION = "backup.message_cutoff_duration"
+    private const val KEY_LAST_USED_MESSAGE_CUTOFF_TIME = "backup.last_used_message_cutoff_time"
+
    private val cachedCdnCredentialsExpiresIn: Duration = 12.hours

    private val lock = ReentrantLock()
@@ -259,8 +262,8 @@ class BackupValues(store: KeyValueStore) : SignalStoreValues(store) {

        if (storedValue != value) {
          clearNotEnoughRemoteStorageSpace()
-          clearBackupCreationFailed()
          clearMessageBackupFailureSheetWatermark()
+          backupCreationError = null
        }

        deletionState = DeletionState.NONE
@@ -303,7 +306,7 @@ class BackupValues(store: KeyValueStore) : SignalStoreValues(store) {
  var hasBackupBeenUploaded: Boolean by booleanValue(KEY_BACKUP_UPLOADED, false)

  val hasBackupCreationError: Boolean get() = backupCreationError != null
-  val backupCreationError: BackupCreationError? by enumValue(KEY_BACKUP_CREATION_ERROR, null, BackupCreationError.serializer)
+  var backupCreationError: BackupCreationError? by enumValue(KEY_BACKUP_CREATION_ERROR, null, BackupCreationError.serializer)
  val nextBackupFailureSnoozeTime: Duration get() = getLong(KEY_BACKUP_FAIL_ACKNOWLEDGED_SNOOZE_TIME, 0L).milliseconds
  val nextBackupFailureSheetSnoozeTime: Duration get() = getLong(KEY_BACKUP_FAIL_SHEET_SNOOZE_TIME, getNextBackupFailureSheetSnoozeTime(lastBackupTime.milliseconds).inWholeMilliseconds).milliseconds

@@ -419,6 +422,20 @@ class BackupValues(store: KeyValueStore) : SignalStoreValues(store) {
        .apply()
    }

+  /**
+   * If set, this represents how far back we should backup messages. For instance, if the returned value is 1 year in milliseconds, you should back up
+   * every message within the last year. If unset, back up all messages. We only cutoff old messages for users whose backup is over the
+   * size limit, which is *extraordinarily* rare, so this value is almost always null.
+   */
+  var messageCuttoffDuration: Duration? by durationValue(KEY_MESSAGE_CUTOFF_DURATION, null)
+
+  /**
+   * The last threshold we used for backing up messages. Messages sent before this time were not included in the backup.
+   * A value of 0 indicates that we included all messages. We only cutoff old messages for users whose backup is over the
+   * size limit, which is *extraordinarily* rare, so this value is almost always 0.
+   */
+  var lastUsedMessageCutoffTime: Long by longValue(KEY_LAST_USED_MESSAGE_CUTOFF_TIME, 0)
+
  /**
   * When we are told by the server that we are out of storage space, we should show
   * UX treatment to make the user aware of this.
--- a/app/src/main/java/org/thoughtcrime/securesms/keyvalue/SignalStoreValueDelegates.kt
+++ b/app/src/main/java/org/thoughtcrime/securesms/keyvalue/SignalStoreValueDelegates.kt
@@ -5,6 +5,8 @@ import kotlinx.coroutines.flow.Flow
 import kotlinx.coroutines.flow.MutableStateFlow
 import org.signal.core.util.LongSerializer
 import kotlin.reflect.KProperty
+import kotlin.time.Duration
+import kotlin.time.Duration.Companion.nanoseconds

 internal fun SignalStoreValues.longValue(key: String, default: Long): SignalStoreValueDelegate<Long> {
  return LongValue(key, default, this.store)
@@ -46,6 +48,10 @@ internal fun <M> SignalStoreValues.protoValue(key: String, default: M, adapter:
  return KeyValueProtoWithDefaultValue(key, default, adapter, this.store, onSet)
 }

+internal fun SignalStoreValues.durationValue(key: String, default: Duration?): SignalStoreValueDelegate<Duration?> {
+  return DurationValue(key, default, this.store)
+}
+
 internal fun <T> SignalStoreValueDelegate<T>.withPrecondition(precondition: () -> Boolean): SignalStoreValueDelegate<T> {
  return PreconditionDelegate(
    delegate = this,
@@ -159,6 +165,20 @@ private class NullableBlobValue(private val key: String, default: ByteArray?, st
  }
 }

+private class DurationValue(private val key: String, default: Duration?, store: KeyValueStore) : SignalStoreValueDelegate<Duration?>(store, default) {
+  companion object {
+    private const val UNSET: Long = -1
+  }
+
+  override fun getValue(values: KeyValueStore): Duration? {
+    return values.getLong(key, default?.inWholeNanoseconds ?: UNSET).takeUnless { it == UNSET }?.nanoseconds
+  }
+
+  override fun setValue(values: KeyValueStore, value: Duration?) {
+    values.beginWrite().putLong(key, value?.inWholeNanoseconds ?: UNSET).apply()
+  }
+}
+
 private class KeyValueProtoWithDefaultValue<M>(
  private val key: String,
  default: M,