diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/BackupRepository.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/BackupRepository.kt index 3cf5c5121e..feda926ac0 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/BackupRepository.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/BackupRepository.kt @@ -49,6 +49,7 @@ import org.signal.libsignal.zkgroup.backups.BackupLevel import org.signal.libsignal.zkgroup.profiles.ProfileKey import org.thoughtcrime.securesms.R import org.thoughtcrime.securesms.attachments.Attachment +import org.thoughtcrime.securesms.attachments.AttachmentId import org.thoughtcrime.securesms.attachments.Cdn import org.thoughtcrime.securesms.attachments.DatabaseAttachment import org.thoughtcrime.securesms.backup.ArchiveUploadProgress @@ -1370,6 +1371,28 @@ object BackupRepository { } } + /** + * Returns if an attachment should be copied to the archive if it meets certain requirements eg + * not a story, not already uploaded to the archive cdn, not a preuploaded attachment, etc. + */ + @JvmStatic + fun shouldCopyAttachmentToArchive(attachmentId: AttachmentId, messageId: Long): Boolean { + if (!SignalStore.backup.backsUpMedia) { + return false + } + + val attachment = SignalDatabase.attachments.getAttachment(attachmentId) + + return when { + attachment == null -> false + attachment.archiveTransferState == AttachmentTable.ArchiveTransferState.FINISHED -> false + !DatabaseAttachmentArchiveUtil.hadIntegrityCheckPerformed(attachment) -> false + messageId == AttachmentTable.PREUPLOAD_MESSAGE_ID -> false + SignalDatabase.messages.isStory(messageId) -> false + else -> true + } + } + /** * Copies a thumbnail that has been uploaded to the transit cdn to the archive cdn. */ diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/DatabaseAttachmentArchiveUtil.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/DatabaseAttachmentArchiveUtil.kt index 0a3c628dcb..d0853a1156 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/DatabaseAttachmentArchiveUtil.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/DatabaseAttachmentArchiveUtil.kt @@ -59,7 +59,10 @@ object DatabaseAttachmentArchiveUtil { return MediaName.fromPlaintextHashAndRemoteKeyForThumbnail(attachment.dataHash!!.decodeBase64OrThrow(), attachment.remoteKey!!.decodeBase64OrThrow()) } - private fun hadIntegrityCheckPerformed(attachment: DatabaseAttachment): Boolean { + /** + * Returns whether an integrity check has been performed at some point by checking against its transfer state + */ + fun hadIntegrityCheckPerformed(attachment: DatabaseAttachment): Boolean { if (attachment.archiveTransferState == AttachmentTable.ArchiveTransferState.FINISHED) { return true } diff --git a/app/src/main/java/org/thoughtcrime/securesms/database/AttachmentTable.kt b/app/src/main/java/org/thoughtcrime/securesms/database/AttachmentTable.kt index 043be5f5c3..d2657db88d 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/database/AttachmentTable.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/database/AttachmentTable.kt @@ -629,10 +629,10 @@ class AttachmentTable( */ fun getAttachmentsThatNeedArchiveUpload(): List { return readableDatabase - .select(ID) - .from(TABLE_NAME) - .where("($ARCHIVE_TRANSFER_STATE = ? or $ARCHIVE_TRANSFER_STATE = ?) AND $DATA_FILE NOT NULL AND $TRANSFER_STATE = $TRANSFER_PROGRESS_DONE", ArchiveTransferState.NONE.value, ArchiveTransferState.TEMPORARY_FAILURE.value) - .orderBy("$ID DESC") + .select("$TABLE_NAME.$ID") + .from("$TABLE_NAME LEFT JOIN ${MessageTable.TABLE_NAME} ON $TABLE_NAME.$MESSAGE_ID = ${MessageTable.TABLE_NAME}.${MessageTable.ID}") + .where("($ARCHIVE_TRANSFER_STATE = ? or $ARCHIVE_TRANSFER_STATE = ?) AND $DATA_FILE NOT NULL AND $TRANSFER_STATE = $TRANSFER_PROGRESS_DONE AND (${MessageTable.STORY_TYPE} = 0 OR ${MessageTable.STORY_TYPE} IS NULL)", ArchiveTransferState.NONE.value, ArchiveTransferState.TEMPORARY_FAILURE.value) + .orderBy("$TABLE_NAME.$ID DESC") .run() .readToList { AttachmentId(it.requireLong(ID)) } } @@ -677,8 +677,8 @@ class AttachmentTable( */ fun doAnyAttachmentsNeedArchiveUpload(): Boolean { return readableDatabase - .exists(TABLE_NAME) - .where("($ARCHIVE_TRANSFER_STATE = ? OR $ARCHIVE_TRANSFER_STATE = ?) AND $DATA_FILE NOT NULL AND $TRANSFER_STATE = $TRANSFER_PROGRESS_DONE", ArchiveTransferState.NONE.value, ArchiveTransferState.TEMPORARY_FAILURE.value) + .exists("$TABLE_NAME LEFT JOIN ${MessageTable.TABLE_NAME} ON $TABLE_NAME.$MESSAGE_ID = ${MessageTable.TABLE_NAME}.${MessageTable.ID}") + .where("($ARCHIVE_TRANSFER_STATE = ? OR $ARCHIVE_TRANSFER_STATE = ?) AND $DATA_FILE NOT NULL AND $TRANSFER_STATE = $TRANSFER_PROGRESS_DONE AND (${MessageTable.STORY_TYPE} = 0 OR ${MessageTable.STORY_TYPE} IS NULL)", ArchiveTransferState.NONE.value, ArchiveTransferState.TEMPORARY_FAILURE.value) .run() } @@ -829,12 +829,13 @@ class AttachmentTable( SELECT SUM($DATA_SIZE) FROM ( SELECT DISTINCT $DATA_HASH_END, $REMOTE_KEY, $DATA_SIZE - FROM $TABLE_NAME + FROM $TABLE_NAME LEFT JOIN ${MessageTable.TABLE_NAME} ON $TABLE_NAME.$MESSAGE_ID = ${MessageTable.TABLE_NAME}.${MessageTable.ID} WHERE $DATA_FILE NOT NULL AND $DATA_HASH_END NOT NULL AND $REMOTE_KEY NOT NULL AND - $ARCHIVE_TRANSFER_STATE NOT IN (${ArchiveTransferState.FINISHED.value}, ${ArchiveTransferState.PERMANENT_FAILURE.value}) + $ARCHIVE_TRANSFER_STATE NOT IN (${ArchiveTransferState.FINISHED.value}, ${ArchiveTransferState.PERMANENT_FAILURE.value}) AND + (${MessageTable.STORY_TYPE} = 0 OR ${MessageTable.STORY_TYPE} IS NULL) ) """.trimIndent() ) @@ -1490,6 +1491,15 @@ class AttachmentTable( return getAttachment(result.values.iterator().next()) ?: throw MmsException("Failed to retrieve attachment we just inserted!") } + fun getMessageId(attachmentId: AttachmentId): Long { + return readableDatabase + .select(MESSAGE_ID) + .from(TABLE_NAME) + .where("$ID = ?", attachmentId.id) + .run() + .readToSingleLong() + } + fun updateMessageId(attachmentIds: Collection, mmsId: Long, isStory: Boolean) { writableDatabase.withinTransaction { db -> val values = ContentValues(2).apply { diff --git a/app/src/main/java/org/thoughtcrime/securesms/jobs/AttachmentDownloadJob.kt b/app/src/main/java/org/thoughtcrime/securesms/jobs/AttachmentDownloadJob.kt index 93b5130ed0..377525c103 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/jobs/AttachmentDownloadJob.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/jobs/AttachmentDownloadJob.kt @@ -220,6 +220,7 @@ class AttachmentDownloadJob private constructor( } if (SignalStore.backup.backsUpMedia) { + val isStory = SignalDatabase.messages.isStory(messageId) when { attachment.archiveTransferState == AttachmentTable.ArchiveTransferState.FINISHED -> { Log.i(TAG, "[$attachmentId] Already archived. Skipping.") @@ -230,6 +231,10 @@ class AttachmentDownloadJob private constructor( AppDependencies.jobManager.add(UploadAttachmentToArchiveJob(attachmentId)) } + isStory -> { + Log.i(TAG, "[$attachmentId] Attachment is a story. Skipping.") + } + else -> { Log.i(TAG, "[$attachmentId] Enqueuing job to copy to archive.") AppDependencies.jobManager.add(CopyAttachmentToArchiveJob(attachmentId)) diff --git a/app/src/main/java/org/thoughtcrime/securesms/jobs/AttachmentUploadJob.kt b/app/src/main/java/org/thoughtcrime/securesms/jobs/AttachmentUploadJob.kt index d477f89d5c..4f5685d886 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/jobs/AttachmentUploadJob.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/jobs/AttachmentUploadJob.kt @@ -17,6 +17,7 @@ import org.thoughtcrime.securesms.attachments.Attachment import org.thoughtcrime.securesms.attachments.AttachmentId import org.thoughtcrime.securesms.attachments.AttachmentUploadUtil import org.thoughtcrime.securesms.attachments.DatabaseAttachment +import org.thoughtcrime.securesms.backup.v2.BackupRepository import org.thoughtcrime.securesms.database.AttachmentTable import org.thoughtcrime.securesms.database.SignalDatabase import org.thoughtcrime.securesms.dependencies.AppDependencies @@ -139,6 +140,11 @@ class AttachmentUploadJob private constructor( val timeSinceUpload = System.currentTimeMillis() - databaseAttachment.uploadTimestamp if (timeSinceUpload < UPLOAD_REUSE_THRESHOLD && !TextUtils.isEmpty(databaseAttachment.remoteLocation)) { Log.i(TAG, "We can re-use an already-uploaded file. It was uploaded $timeSinceUpload ms (${timeSinceUpload.milliseconds.inRoundedDays()} days) ago. Skipping.") + SignalDatabase.attachments.setTransferState(databaseAttachment.mmsId, attachmentId, AttachmentTable.TRANSFER_PROGRESS_DONE) + if (BackupRepository.shouldCopyAttachmentToArchive(databaseAttachment.attachmentId, databaseAttachment.mmsId)) { + Log.i(TAG, "[$attachmentId] The re-used file was not copied to the archive. Copying now.") + AppDependencies.jobManager.add(CopyAttachmentToArchiveJob(attachmentId)) + } return } else if (databaseAttachment.uploadTimestamp > 0) { Log.i(TAG, "This file was previously-uploaded, but too long ago to be re-used. Age: $timeSinceUpload ms (${timeSinceUpload.milliseconds.inRoundedDays()} days)") @@ -173,10 +179,18 @@ class AttachmentUploadJob private constructor( val uploadResult: AttachmentUploadResult = SignalNetwork.attachments.uploadAttachmentV4(localAttachment).successOrThrow() SignalDatabase.attachments.finalizeAttachmentAfterUpload(databaseAttachment.attachmentId, uploadResult) if (SignalStore.backup.backsUpMedia) { + val messageId = SignalDatabase.attachments.getMessageId(databaseAttachment.attachmentId) + val isStory = SignalDatabase.messages.isStory(messageId) when { databaseAttachment.archiveTransferState == AttachmentTable.ArchiveTransferState.FINISHED -> { Log.i(TAG, "[$attachmentId] Already archived. Skipping.") } + isStory -> { + Log.i(TAG, "[$attachmentId] Attachment is a story. Skipping.") + } + messageId == AttachmentTable.PREUPLOAD_MESSAGE_ID -> { + Log.i(TAG, "[$attachmentId] Avoid uploading preuploaded attachments to archive. Skipping.") + } else -> { Log.i(TAG, "[$attachmentId] Enqueuing job to copy to archive.") AppDependencies.jobManager.add(CopyAttachmentToArchiveJob(attachmentId)) diff --git a/app/src/main/java/org/thoughtcrime/securesms/jobs/CopyAttachmentToArchiveJob.kt b/app/src/main/java/org/thoughtcrime/securesms/jobs/CopyAttachmentToArchiveJob.kt index 9782947858..55f6496306 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/jobs/CopyAttachmentToArchiveJob.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/jobs/CopyAttachmentToArchiveJob.kt @@ -20,7 +20,6 @@ import org.thoughtcrime.securesms.jobmanager.impl.NoRemoteArchiveGarbageCollecti import org.thoughtcrime.securesms.jobs.protos.CopyAttachmentToArchiveJobData import org.thoughtcrime.securesms.keyvalue.SignalStore import org.whispersystems.signalservice.api.NetworkResult -import java.lang.RuntimeException import java.util.concurrent.TimeUnit /** @@ -91,6 +90,12 @@ class CopyAttachmentToArchiveJob private constructor(private val attachmentId: A return Result.failure() } + if (SignalDatabase.messages.isStory(attachment.mmsId)) { + Log.i(TAG, "[$attachmentId] Attachment is a story. Resetting transfer state to none and skipping.") + SignalDatabase.attachments.setArchiveTransferState(attachmentId, AttachmentTable.ArchiveTransferState.NONE) + return Result.success() + } + if (isCanceled) { Log.w(TAG, "[$attachmentId] Canceled. Refusing to proceed.") return Result.failure() diff --git a/app/src/main/java/org/thoughtcrime/securesms/jobs/UploadAttachmentToArchiveJob.kt b/app/src/main/java/org/thoughtcrime/securesms/jobs/UploadAttachmentToArchiveJob.kt index e9ee4182c4..3c1f541def 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/jobs/UploadAttachmentToArchiveJob.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/jobs/UploadAttachmentToArchiveJob.kt @@ -125,6 +125,12 @@ class UploadAttachmentToArchiveJob private constructor( return Result.success() } + if (SignalDatabase.messages.isStory(attachment.mmsId)) { + Log.i(TAG, "[$attachmentId] Attachment is a story. Resetting transfer state to none and skipping.") + SignalDatabase.attachments.setArchiveTransferState(attachmentId, AttachmentTable.ArchiveTransferState.NONE) + return Result.success() + } + if (attachment.remoteKey == null) { Log.w(TAG, "[$attachmentId] Attachment is missing remote key! Cannot upload.") return Result.failure() diff --git a/app/src/main/java/org/thoughtcrime/securesms/sms/MessageSender.java b/app/src/main/java/org/thoughtcrime/securesms/sms/MessageSender.java index 8789ce585a..e0de88cfd7 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/sms/MessageSender.java +++ b/app/src/main/java/org/thoughtcrime/securesms/sms/MessageSender.java @@ -31,6 +31,7 @@ import org.signal.core.util.logging.Log; import org.thoughtcrime.securesms.attachments.Attachment; import org.thoughtcrime.securesms.attachments.AttachmentId; import org.thoughtcrime.securesms.attachments.DatabaseAttachment; +import org.thoughtcrime.securesms.backup.v2.BackupRepository; import org.thoughtcrime.securesms.contacts.sync.ContactDiscovery; import org.thoughtcrime.securesms.contactshare.Contact; import org.thoughtcrime.securesms.database.AttachmentTable; @@ -51,6 +52,7 @@ import org.thoughtcrime.securesms.jobmanager.JobManager; import org.thoughtcrime.securesms.jobs.AttachmentCompressionJob; import org.thoughtcrime.securesms.jobs.AttachmentCopyJob; import org.thoughtcrime.securesms.jobs.AttachmentUploadJob; +import org.thoughtcrime.securesms.jobs.CopyAttachmentToArchiveJob; import org.thoughtcrime.securesms.jobs.IndividualSendJob; import org.thoughtcrime.securesms.jobs.ProfileKeySendJob; import org.thoughtcrime.securesms.jobs.PushDistributionListSendJob; @@ -274,6 +276,14 @@ public class MessageSender { false, insertListener); + for (AttachmentId attachmentId: attachmentIds) { + boolean wasPreuploaded = SignalDatabase.attachments().getMessageId(attachmentId) == AttachmentTable.PREUPLOAD_MESSAGE_ID; + if (wasPreuploaded && BackupRepository.shouldCopyAttachmentToArchive(attachmentId, messageId)) { + Log.i(TAG, "[" + attachmentId + "] Was previously preuploaded and should now be copied to the archive."); + AppDependencies.getJobManager().add(new CopyAttachmentToArchiveJob(attachmentId)); + } + } + attachmentDatabase.updateMessageId(attachmentIds, messageId, message.getStoryType().isStory()); sendMessageInternal(context, recipient, SendType.SIGNAL, messageId, jobIds); @@ -296,15 +306,16 @@ public class MessageSender { Preconditions.checkArgument(messages.size() > 0, "No messages!"); Preconditions.checkArgument(Stream.of(messages).allMatch(m -> m.getAttachments().isEmpty()), "Messages can't have attachments! They should be pre-uploaded."); - JobManager jobManager = AppDependencies.getJobManager(); - AttachmentTable attachmentDatabase = SignalDatabase.attachments(); - MessageTable mmsDatabase = SignalDatabase.messages(); - ThreadTable threadTable = SignalDatabase.threads(); - List preUploadAttachmentIds = Stream.of(preUploadResults).map(PreUploadResult::getAttachmentId).toList(); - List preUploadJobIds = Stream.of(preUploadResults).map(PreUploadResult::getJobIds).flatMap(Stream::of).toList(); - List messageIds = new ArrayList<>(messages.size()); - List messageDependsOnIds = new ArrayList<>(preUploadJobIds); - OutgoingMessage primaryMessage = messages.get(0); + JobManager jobManager = AppDependencies.getJobManager(); + AttachmentTable attachmentDatabase = SignalDatabase.attachments(); + MessageTable mmsDatabase = SignalDatabase.messages(); + ThreadTable threadTable = SignalDatabase.threads(); + List preUploadAttachmentIds = Stream.of(preUploadResults).map(PreUploadResult::getAttachmentId).toList(); + List preUploadJobIds = Stream.of(preUploadResults).map(PreUploadResult::getJobIds).flatMap(Stream::of).toList(); + List messageIds = new ArrayList<>(messages.size()); + List messageDependsOnIds = new ArrayList<>(preUploadJobIds); + OutgoingMessage primaryMessage = messages.get(0); + List attachmentsWithPreuploadId = preUploadAttachmentIds.stream().filter(id -> SignalDatabase.attachments().getMessageId(id) == AttachmentTable.PREUPLOAD_MESSAGE_ID).collect(Collectors.toList()); mmsDatabase.beginTransaction(); try { @@ -379,6 +390,14 @@ public class MessageSender { } } + for (AttachmentId attachmentId : attachmentsWithPreuploadId) { + long messageId = SignalDatabase.attachments().getMessageId(attachmentId); + if (BackupRepository.shouldCopyAttachmentToArchive(attachmentId, messageId)) { + Log.i(TAG, "[" + attachmentId + "] Was previously preuploaded and should now be copied to the archive."); + jobManager.add(new CopyAttachmentToArchiveJob(attachmentId)); + } + } + onMessageSent(); mmsDatabase.setTransactionSuccessful(); } catch (MmsException e) {