From 5324290fab50cb3880bd1f79798b9c30b18fcd50 Mon Sep 17 00:00:00 2001 From: Cody Henthorne Date: Wed, 1 Oct 2025 14:20:31 -0400 Subject: [PATCH] Dedupe attachment downloads for matching attachments and fix size calculations. --- .../securesms/database/AttachmentTable.kt | 25 ++++++++++++++----- .../securesms/jobs/BackupRestoreMediaJob.kt | 3 ++- .../securesms/jobs/RestoreAttachmentJob.kt | 17 ++++++++++--- .../jobs/RestoreOptimizedMediaJob.kt | 3 ++- 4 files changed, 36 insertions(+), 12 deletions(-) diff --git a/app/src/main/java/org/thoughtcrime/securesms/database/AttachmentTable.kt b/app/src/main/java/org/thoughtcrime/securesms/database/AttachmentTable.kt index dff80c34e2..44f104f642 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/database/AttachmentTable.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/database/AttachmentTable.kt @@ -687,11 +687,24 @@ class AttachmentTable( fun getRemainingRestorableAttachmentSize(): Long { return readableDatabase - .select("SUM($DATA_SIZE)") - .from(TABLE_NAME) - .where("$TRANSFER_STATE = ? OR $TRANSFER_STATE = ?", TRANSFER_NEEDS_RESTORE, TRANSFER_RESTORE_IN_PROGRESS) - .run() - .readToSingleLong() + .rawQuery( + """ + SELECT $DATA_SIZE + FROM ( + SELECT DISTINCT $DATA_HASH_END, $REMOTE_KEY, $DATA_SIZE + FROM $TABLE_NAME + WHERE ($TRANSFER_STATE = $TRANSFER_NEEDS_RESTORE OR $TRANSFER_STATE = $TRANSFER_RESTORE_IN_PROGRESS) + ) + """ + ) + .readToList { it.requireLong(DATA_SIZE) } + .sumOf { + val paddedSize = PaddingInputStream.getPaddedSize(it) + val clientEncryptedSize = AttachmentCipherStreamUtil.getCiphertextLength(paddedSize) + val serverEncryptedSize = AttachmentCipherStreamUtil.getCiphertextLength(clientEncryptedSize) + + serverEncryptedSize + } } fun getOptimizedMediaAttachmentSize(): Long { @@ -706,7 +719,7 @@ class AttachmentTable( private fun getMessageDoesNotExpireWithinTimeoutClause(tablePrefix: String = MessageTable.TABLE_NAME): String { val messageHasExpiration = "$tablePrefix.${MessageTable.EXPIRES_IN} > 0" val messageExpiresInOneDayAfterViewing = "$messageHasExpiration AND $tablePrefix.${MessageTable.EXPIRES_IN} < ${1.days.inWholeMilliseconds}" - return "NOT $messageExpiresInOneDayAfterViewing" + return "NOT ($messageExpiresInOneDayAfterViewing)" } /** diff --git a/app/src/main/java/org/thoughtcrime/securesms/jobs/BackupRestoreMediaJob.kt b/app/src/main/java/org/thoughtcrime/securesms/jobs/BackupRestoreMediaJob.kt index a1511a1cbb..844125bc86 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/jobs/BackupRestoreMediaJob.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/jobs/BackupRestoreMediaJob.kt @@ -95,7 +95,8 @@ class BackupRestoreMediaJob private constructor(parameters: Parameters) : BaseJo restoreFullAttachmentJobs += RestoreAttachmentJob.forInitialRestore( messageId = attachment.mmsId, attachmentId = attachment.attachmentId, - stickerPackId = attachment.stickerPackId + stickerPackId = attachment.stickerPackId, + queueHash = attachment.plaintextHash?.contentHashCode() ?: attachment.remoteKey?.contentHashCode() ) } else { restoreThumbnailJobs += RestoreAttachmentThumbnailJob( diff --git a/app/src/main/java/org/thoughtcrime/securesms/jobs/RestoreAttachmentJob.kt b/app/src/main/java/org/thoughtcrime/securesms/jobs/RestoreAttachmentJob.kt index 738e246eed..bcdf9cd25a 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/jobs/RestoreAttachmentJob.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/jobs/RestoreAttachmentJob.kt @@ -61,6 +61,7 @@ import java.io.File import java.io.IOException import java.util.concurrent.TimeUnit import kotlin.jvm.optionals.getOrNull +import kotlin.math.abs import kotlin.math.max import kotlin.math.pow import kotlin.time.Duration.Companion.days @@ -106,6 +107,14 @@ class RestoreAttachmentJob private constructor( /** All possible queues used by this job. */ val ALL = INITIAL_RESTORE + OFFLOAD_RESTORE + MANUAL_RESTORE + + fun random(queues: Set, queueHash: Int?): String { + return if (queueHash != null) { + queues.elementAt(abs(queueHash) % queues.size) + } else { + queues.random() + } + } } companion object { @@ -116,12 +125,12 @@ class RestoreAttachmentJob private constructor( * Create a restore job for the initial large batch of media on a fresh restore. * Will enqueue with some amount of parallelization with low job priority. */ - fun forInitialRestore(attachmentId: AttachmentId, messageId: Long, stickerPackId: String?): RestoreAttachmentJob { + fun forInitialRestore(attachmentId: AttachmentId, messageId: Long, stickerPackId: String?, queueHash: Int?): RestoreAttachmentJob { return RestoreAttachmentJob( attachmentId = attachmentId, messageId = messageId, manual = false, - queue = Queues.INITIAL_RESTORE.random(), + queue = Queues.random(Queues.INITIAL_RESTORE, queueHash), priority = Parameters.PRIORITY_LOW, stickerPackId = stickerPackId ) @@ -132,12 +141,12 @@ class RestoreAttachmentJob private constructor( * * See [RestoreOptimizedMediaJob]. */ - fun forOffloadedRestore(attachmentId: AttachmentId, messageId: Long): RestoreAttachmentJob { + fun forOffloadedRestore(attachmentId: AttachmentId, messageId: Long, queueHash: Int?): RestoreAttachmentJob { return RestoreAttachmentJob( attachmentId = attachmentId, messageId = messageId, manual = false, - queue = Queues.OFFLOAD_RESTORE.random(), + queue = Queues.random(Queues.OFFLOAD_RESTORE, queueHash), priority = Parameters.PRIORITY_LOW ) } diff --git a/app/src/main/java/org/thoughtcrime/securesms/jobs/RestoreOptimizedMediaJob.kt b/app/src/main/java/org/thoughtcrime/securesms/jobs/RestoreOptimizedMediaJob.kt index b3dc768738..b4d7bbc7ef 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/jobs/RestoreOptimizedMediaJob.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/jobs/RestoreOptimizedMediaJob.kt @@ -70,7 +70,8 @@ class RestoreOptimizedMediaJob private constructor(parameters: Parameters) : Job .forEach { val job = RestoreAttachmentJob.forOffloadedRestore( messageId = it.mmsId, - attachmentId = it.attachmentId + attachmentId = it.attachmentId, + queueHash = it.plaintextHash?.contentHashCode() ?: it.remoteKey?.contentHashCode() ) // Intentionally enqueues one at a time for safer attachment transfer state management