Signal/Android
1
0
Fork 0
mirror of https://github.com/signalapp/Signal-Android.git synced 2026-02-26 12:44:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

Filter messages that will expire within 24hrs from including attachments in backups.

Browse Source
This commit is contained in:
Alex Hart
2025-07-22 16:28:28 -03:00
committed by Michelle Tang
parent 635aa8791f
commit 16776ad843
8 changed files with 162 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
app/src/main/java/org/thoughtcrime/securesms/backup/v2/BackupRepository.kt
View File

@@ -38,7 +38,6 @@ import org.signal.core.util.getAllIndexDefinitions
import org.signal.core.util.getAllTableDefinitions
import org.signal.core.util.getAllTriggerDefinitions
import org.signal.core.util.getForeignKeyViolations
import org.signal.core.util.isNotEmpty
import org.signal.core.util.logging.Log
import org.signal.core.util.money.FiatMoney
import org.signal.core.util.requireIntOrNull
@@ -1425,6 +1424,7 @@ object BackupRepository {
!DatabaseAttachmentArchiveUtil.hadIntegrityCheckPerformed(attachment) -> false
messageId == AttachmentTable.PREUPLOAD_MESSAGE_ID -> false
SignalDatabase.messages.isStory(messageId) -> false
SignalDatabase.messages.willMessageExpireBeforeCutoff(messageId) -> false
else -> true
}
}

50
app/src/main/java/org/thoughtcrime/securesms/database/AttachmentTable.kt
View File

@@ -77,6 +77,11 @@ import org.thoughtcrime.securesms.crypto.AttachmentSecret
import org.thoughtcrime.securesms.crypto.ClassicDecryptingPartInputStream
import org.thoughtcrime.securesms.crypto.ModernDecryptingPartInputStream
import org.thoughtcrime.securesms.crypto.ModernEncryptingPartOutputStream
import org.thoughtcrime.securesms.database.AttachmentTable.ArchiveTransferState.COPY_PENDING
import org.thoughtcrime.securesms.database.AttachmentTable.ArchiveTransferState.FINISHED
import org.thoughtcrime.securesms.database.AttachmentTable.ArchiveTransferState.NONE
import org.thoughtcrime.securesms.database.AttachmentTable.ArchiveTransferState.PERMANENT_FAILURE
import org.thoughtcrime.securesms.database.AttachmentTable.ArchiveTransferState.UPLOAD_IN_PROGRESS
import org.thoughtcrime.securesms.database.AttachmentTable.Companion.DATA_FILE
import org.thoughtcrime.securesms.database.AttachmentTable.Companion.DATA_HASH_END
import org.thoughtcrime.securesms.database.AttachmentTable.Companion.PREUPLOAD_MESSAGE_ID
@@ -625,6 +630,12 @@ class AttachmentTable(
.readToSingleLong()
}
private fun getMessageDoesNotExpireWithinTimeoutClause(): String {
val messageHasExpiration = "${MessageTable.TABLE_NAME}.${MessageTable.EXPIRES_IN} > 0"
val messageExpiresInOneDayAfterViewing = "$messageHasExpiration AND ${MessageTable.TABLE_NAME}.${MessageTable.EXPIRES_IN} < ${1.days.inWholeMilliseconds}"
return "NOT $messageExpiresInOneDayAfterViewing"
}
/**
* Finds all of the attachmentIds of attachments that need to be uploaded to the archive cdn.
*/
@@ -632,7 +643,11 @@ class AttachmentTable(
return readableDatabase
.select("$TABLE_NAME.$ID")
.from("$TABLE_NAME LEFT JOIN ${MessageTable.TABLE_NAME} ON $TABLE_NAME.$MESSAGE_ID = ${MessageTable.TABLE_NAME}.${MessageTable.ID}")
.where("($ARCHIVE_TRANSFER_STATE = ? or $ARCHIVE_TRANSFER_STATE = ?) AND $DATA_FILE NOT NULL AND $TRANSFER_STATE = $TRANSFER_PROGRESS_DONE AND (${MessageTable.STORY_TYPE} = 0 OR ${MessageTable.STORY_TYPE} IS NULL)", ArchiveTransferState.NONE.value, ArchiveTransferState.TEMPORARY_FAILURE.value)
.where(
"($ARCHIVE_TRANSFER_STATE = ? or $ARCHIVE_TRANSFER_STATE = ?) AND $DATA_FILE NOT NULL AND $TRANSFER_STATE = $TRANSFER_PROGRESS_DONE AND (${MessageTable.STORY_TYPE} = 0 OR ${MessageTable.STORY_TYPE} IS NULL) AND ${getMessageDoesNotExpireWithinTimeoutClause()}",
ArchiveTransferState.NONE.value,
ArchiveTransferState.TEMPORARY_FAILURE.value
)
.orderBy("$TABLE_NAME.$ID DESC")
.run()
.readToList { AttachmentId(it.requireLong(ID)) }
@@ -679,7 +694,11 @@ class AttachmentTable(
fun doAnyAttachmentsNeedArchiveUpload(): Boolean {
return readableDatabase
.exists("$TABLE_NAME LEFT JOIN ${MessageTable.TABLE_NAME} ON $TABLE_NAME.$MESSAGE_ID = ${MessageTable.TABLE_NAME}.${MessageTable.ID}")
.where("($ARCHIVE_TRANSFER_STATE = ? OR $ARCHIVE_TRANSFER_STATE = ?) AND $DATA_FILE NOT NULL AND $TRANSFER_STATE = $TRANSFER_PROGRESS_DONE AND (${MessageTable.STORY_TYPE} = 0 OR ${MessageTable.STORY_TYPE} IS NULL)", ArchiveTransferState.NONE.value, ArchiveTransferState.TEMPORARY_FAILURE.value)
.where(
"($ARCHIVE_TRANSFER_STATE = ? OR $ARCHIVE_TRANSFER_STATE = ?) AND $DATA_FILE NOT NULL AND $TRANSFER_STATE = $TRANSFER_PROGRESS_DONE AND (${MessageTable.STORY_TYPE} = 0 OR ${MessageTable.STORY_TYPE} IS NULL) AND ${getMessageDoesNotExpireWithinTimeoutClause()}",
ArchiveTransferState.NONE.value,
ArchiveTransferState.TEMPORARY_FAILURE.value
)
.run()
}
@@ -836,7 +855,8 @@ class AttachmentTable(
$DATA_HASH_END NOT NULL AND
$REMOTE_KEY NOT NULL AND
$ARCHIVE_TRANSFER_STATE NOT IN (${ArchiveTransferState.FINISHED.value}, ${ArchiveTransferState.PERMANENT_FAILURE.value}) AND
(${MessageTable.STORY_TYPE} = 0 OR ${MessageTable.STORY_TYPE} IS NULL)
(${MessageTable.STORY_TYPE} = 0 OR ${MessageTable.STORY_TYPE} IS NULL) AND
${getMessageDoesNotExpireWithinTimeoutClause()}
)
""".trimIndent()
)
@@ -1153,6 +1173,7 @@ class AttachmentTable(
ThumbnailRestoreState.IN_PROGRESS -> {
"($THUMBNAIL_RESTORE_STATE = ${ThumbnailRestoreState.NEEDS_RESTORE.value} OR $THUMBNAIL_RESTORE_STATE = ${ThumbnailRestoreState.IN_PROGRESS.value}) AND"
}
else -> ""
}
@@ -2336,9 +2357,11 @@ class AttachmentTable(
hashMatch.hashStart -> {
Log.i(TAG, "[insertAttachmentWithData] Found that the new attachment hash matches the DATA_HASH_START of ${hashMatch.id}. Using all of it's fields. (MessageId: $messageId, ${attachment.uri})")
}
hashMatch.hashEnd -> {
Log.i(TAG, "[insertAttachmentWithData] Found that the new attachment hash matches the DATA_HASH_END of ${hashMatch.id}. Using all of it's fields. (MessageId: $messageId, ${attachment.uri})")
}
else -> {
throw IllegalStateException("Should not be possible based on query.")
}
@@ -2380,7 +2403,10 @@ class AttachmentTable(
}
if (uploadTemplate != null) {
Log.i(TAG, "[insertAttachmentWithData] Found a valid template we could use to skip upload. Template: ${uploadTemplate.attachmentId}, TemplateUploadTimestamp: ${hashMatch?.uploadTimestamp}, CurrentTime: ${System.currentTimeMillis()}, InsertingAttachment: (MessageId: $messageId, ${attachment.uri})")
Log.i(
TAG,
"[insertAttachmentWithData] Found a valid template we could use to skip upload. Template: ${uploadTemplate.attachmentId}, TemplateUploadTimestamp: ${hashMatch?.uploadTimestamp}, CurrentTime: ${System.currentTimeMillis()}, InsertingAttachment: (MessageId: $messageId, ${attachment.uri})"
)
transformProperties = (uploadTemplate.transformProperties ?: transformProperties).copy(skipTransform = true)
}
@@ -2647,11 +2673,19 @@ class AttachmentTable(
.associate { it to readableDatabase.count().from(TABLE_NAME).where("$ARCHIVE_TRANSFER_STATE = ${it.value} AND $DATA_HASH_END NOT NULL AND $REMOTE_KEY NOT NULL").run().readToSingleLong(-1L) }
val attachmentFileCount = readableDatabase.query("SELECT COUNT(DISTINCT $DATA_FILE) FROM $TABLE_NAME WHERE $DATA_FILE NOT NULL AND $DATA_HASH_END NOT NULL AND $REMOTE_KEY NOT NULL").readToSingleLong(-1L)
val finishedAttachmentFileCount = readableDatabase.query("SELECT COUNT(DISTINCT $DATA_FILE) FROM $TABLE_NAME WHERE $DATA_FILE NOT NULL AND $DATA_HASH_END NOT NULL AND $REMOTE_KEY NOT NULL AND $ARCHIVE_TRANSFER_STATE = ${ArchiveTransferState.FINISHED.value}").readToSingleLong(-1L)
val attachmentPlaintextHashAndKeyCount = readableDatabase.query("SELECT COUNT(*) FROM (SELECT DISTINCT $DATA_HASH_END, $REMOTE_KEY FROM $TABLE_NAME WHERE $DATA_HASH_END NOT NULL AND $REMOTE_KEY NOT NULL AND $TRANSFER_STATE in ($TRANSFER_PROGRESS_DONE, $TRANSFER_RESTORE_OFFLOADED, $TRANSFER_RESTORE_IN_PROGRESS, $TRANSFER_NEEDS_RESTORE))").readToSingleLong(-1L)
val finishedAttachmentDigestCount = readableDatabase.query("SELECT COUNT(*) FROM (SELECT DISTINCT $DATA_HASH_END, $REMOTE_KEY FROM $TABLE_NAME WHERE $DATA_HASH_END NOT NULL AND $REMOTE_KEY NOT NULL AND $ARCHIVE_TRANSFER_STATE = ${ArchiveTransferState.FINISHED.value})").readToSingleLong(-1L)
val finishedAttachmentFileCount =
readableDatabase.query("SELECT COUNT(DISTINCT $DATA_FILE) FROM $TABLE_NAME WHERE $DATA_FILE NOT NULL AND $DATA_HASH_END NOT NULL AND $REMOTE_KEY NOT NULL AND $ARCHIVE_TRANSFER_STATE = ${ArchiveTransferState.FINISHED.value}")
.readToSingleLong(-1L)
val attachmentPlaintextHashAndKeyCount =
readableDatabase.query("SELECT COUNT(*) FROM (SELECT DISTINCT $DATA_HASH_END, $REMOTE_KEY FROM $TABLE_NAME WHERE $DATA_HASH_END NOT NULL AND $REMOTE_KEY NOT NULL AND $TRANSFER_STATE in ($TRANSFER_PROGRESS_DONE, $TRANSFER_RESTORE_OFFLOADED, $TRANSFER_RESTORE_IN_PROGRESS, $TRANSFER_NEEDS_RESTORE))")
.readToSingleLong(-1L)
val finishedAttachmentDigestCount =
readableDatabase.query("SELECT COUNT(*) FROM (SELECT DISTINCT $DATA_HASH_END, $REMOTE_KEY FROM $TABLE_NAME WHERE $DATA_HASH_END NOT NULL AND $REMOTE_KEY NOT NULL AND $ARCHIVE_TRANSFER_STATE = ${ArchiveTransferState.FINISHED.value})")
.readToSingleLong(-1L)
val thumbnailFileCount = readableDatabase.query("SELECT COUNT(DISTINCT $THUMBNAIL_FILE) FROM $TABLE_NAME WHERE $THUMBNAIL_FILE IS NOT NULL").readToSingleLong(-1L)
val estimatedThumbnailCount = readableDatabase.query("SELECT COUNT(*) FROM (SELECT DISTINCT $DATA_HASH_END, $REMOTE_KEY FROM $TABLE_NAME WHERE $ARCHIVE_TRANSFER_STATE = ${ArchiveTransferState.FINISHED.value} AND $DATA_HASH_END NOT NULL AND $REMOTE_KEY NOT NULL AND ($CONTENT_TYPE LIKE 'image/%' OR $CONTENT_TYPE LIKE 'video/%'))").readToSingleLong(-1L)
val estimatedThumbnailCount =
readableDatabase.query("SELECT COUNT(*) FROM (SELECT DISTINCT $DATA_HASH_END, $REMOTE_KEY FROM $TABLE_NAME WHERE $ARCHIVE_TRANSFER_STATE = ${ArchiveTransferState.FINISHED.value} AND $DATA_HASH_END NOT NULL AND $REMOTE_KEY NOT NULL AND ($CONTENT_TYPE LIKE 'image/%' OR $CONTENT_TYPE LIKE 'video/%'))")
.readToSingleLong(-1L)
val pendingUploadBytes = getPendingArchiveUploadBytes()
val uploadedAttachmentBytes = readableDatabase

18
app/src/main/java/org/thoughtcrime/securesms/database/MessageTable.kt
View File

@@ -68,6 +68,7 @@ import org.thoughtcrime.securesms.attachments.Attachment
import org.thoughtcrime.securesms.attachments.AttachmentId
import org.thoughtcrime.securesms.attachments.DatabaseAttachment
import org.thoughtcrime.securesms.attachments.DatabaseAttachment.DisplayOrderComparator
import org.thoughtcrime.securesms.backup.v2.exporters.ChatItemArchiveExporter
import org.thoughtcrime.securesms.contactshare.Contact
import org.thoughtcrime.securesms.conversation.MessageStyler
import org.thoughtcrime.securesms.database.EarlyDeliveryReceiptCache.Receipt
@@ -632,6 +633,23 @@ open class MessageTable(context: Context?, databaseHelper: SignalDatabase) : Dat
return rawQueryWithAttachments(where, null)
}
/**
* Returns true iff
* - the message will expire within [ChatItemArchiveExporter.EXPIRATION_CUTOFF] once viewed
*/
fun willMessageExpireBeforeCutoff(messageId: Long): Boolean {
val expiresIn = readableDatabase
.select(EXPIRES_IN)
.from(TABLE_NAME)
.where(ID_WHERE, messageId)
.run()
.readToSingleObject {
it.requireLong(EXPIRES_IN)
} ?: 0L
return expiresIn > 0L && expiresIn < ChatItemArchiveExporter.EXPIRATION_CUTOFF.inWholeMilliseconds
}
fun getMessagesBySentTimestamp(sentTimestamp: Long): List<MessageRecord> {
return readableDatabase
.select(*MMS_PROJECTION)

4
app/src/main/java/org/thoughtcrime/securesms/jobs/AttachmentDownloadJob.kt
View File

@@ -235,6 +235,10 @@ class AttachmentDownloadJob private constructor(
Log.i(TAG, "[$attachmentId] Attachment is a story. Skipping.")
}
SignalDatabase.messages.willMessageExpireBeforeCutoff(messageId) -> {
Log.i(TAG, "[$attachmentId] Message will expire within 24hrs. Skipping.")
}
else -> {
Log.i(TAG, "[$attachmentId] Enqueuing job to copy to archive.")
AppDependencies.jobManager.add(CopyAttachmentToArchiveJob(attachmentId))

4
app/src/main/java/org/thoughtcrime/securesms/jobs/AttachmentUploadJob.kt
View File

@@ -198,6 +198,10 @@ class AttachmentUploadJob private constructor(
messageId == AttachmentTable.PREUPLOAD_MESSAGE_ID -> {
Log.i(TAG, "[$attachmentId] Avoid uploading preuploaded attachments to archive. Skipping.")
}
SignalDatabase.messages.willMessageExpireBeforeCutoff(messageId) -> {
Log.i(TAG, "[$attachmentId] Message will expire within 24hrs. Skipping.")
}
else -> {
Log.i(TAG, "[$attachmentId] Enqueuing job to copy to archive.")
AppDependencies.jobManager.add(CopyAttachmentToArchiveJob(attachmentId))

6
app/src/main/java/org/thoughtcrime/securesms/jobs/CopyAttachmentToArchiveJob.kt
View File

@@ -96,6 +96,12 @@ class CopyAttachmentToArchiveJob private constructor(private val attachmentId: A
return Result.success()
}
if (SignalDatabase.messages.willMessageExpireBeforeCutoff(attachment.mmsId)) {
Log.i(TAG, "[$attachmentId] Message will expire in less than 24 hours. Resetting transfer state to none and skipping.")
SignalDatabase.attachments.setArchiveTransferState(attachmentId, AttachmentTable.ArchiveTransferState.NONE)
return Result.success()
}
if (isCanceled) {
Log.w(TAG, "[$attachmentId] Canceled. Refusing to proceed.")
return Result.failure()

6
app/src/main/java/org/thoughtcrime/securesms/jobs/UploadAttachmentToArchiveJob.kt
View File

@@ -131,6 +131,12 @@ class UploadAttachmentToArchiveJob private constructor(
return Result.success()
}
if (SignalDatabase.messages.willMessageExpireBeforeCutoff(attachment.mmsId)) {
Log.i(TAG, "[$attachmentId] Message will expire within 24 hours. Resetting transfer state to none and skipping.")
SignalDatabase.attachments.setArchiveTransferState(attachmentId, AttachmentTable.ArchiveTransferState.NONE)
return Result.success()
}
if (attachment.remoteKey == null) {
Log.w(TAG, "[$attachmentId] Attachment is missing remote key! Cannot upload.")
return Result.failure()