Signal/Android
1
0
Fork 0
mirror of https://github.com/signalapp/Signal-Android.git synced 2025-12-25 05:27:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

Avoid uploading story-only media to backups.

Browse Source
This commit is contained in:
Michelle Tang
2025-07-16 13:10:32 -04:00
committed by GitHub
parent 559539dc3b
commit 141faf3fb6
8 changed files with 104 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
app/src/main/java/org/thoughtcrime/securesms/backup/v2/BackupRepository.kt
View File

@@ -49,6 +49,7 @@ import org.signal.libsignal.zkgroup.backups.BackupLevel
import org.signal.libsignal.zkgroup.profiles.ProfileKey
import org.thoughtcrime.securesms.R
import org.thoughtcrime.securesms.attachments.Attachment
import org.thoughtcrime.securesms.attachments.AttachmentId
import org.thoughtcrime.securesms.attachments.Cdn
import org.thoughtcrime.securesms.attachments.DatabaseAttachment
import org.thoughtcrime.securesms.backup.ArchiveUploadProgress
@@ -1370,6 +1371,28 @@ object BackupRepository {
}
}
/**
* Returns if an attachment should be copied to the archive if it meets certain requirements eg
* not a story, not already uploaded to the archive cdn, not a preuploaded attachment, etc.
*/
@JvmStatic
fun shouldCopyAttachmentToArchive(attachmentId: AttachmentId, messageId: Long): Boolean {
if (!SignalStore.backup.backsUpMedia) {
return false
}
val attachment = SignalDatabase.attachments.getAttachment(attachmentId)
return when {
attachment == null -> false
attachment.archiveTransferState == AttachmentTable.ArchiveTransferState.FINISHED -> false
!DatabaseAttachmentArchiveUtil.hadIntegrityCheckPerformed(attachment) -> false
messageId == AttachmentTable.PREUPLOAD_MESSAGE_ID -> false
SignalDatabase.messages.isStory(messageId) -> false
else -> true
}
}
/**
* Copies a thumbnail that has been uploaded to the transit cdn to the archive cdn.
*/

5
app/src/main/java/org/thoughtcrime/securesms/backup/v2/DatabaseAttachmentArchiveUtil.kt
View File

@@ -59,7 +59,10 @@ object DatabaseAttachmentArchiveUtil {
return MediaName.fromPlaintextHashAndRemoteKeyForThumbnail(attachment.dataHash!!.decodeBase64OrThrow(), attachment.remoteKey!!.decodeBase64OrThrow())
}
private fun hadIntegrityCheckPerformed(attachment: DatabaseAttachment): Boolean {
/**
* Returns whether an integrity check has been performed at some point by checking against its transfer state
*/
fun hadIntegrityCheckPerformed(attachment: DatabaseAttachment): Boolean {
if (attachment.archiveTransferState == AttachmentTable.ArchiveTransferState.FINISHED) {
return true
}

26
app/src/main/java/org/thoughtcrime/securesms/database/AttachmentTable.kt
View File

@@ -629,10 +629,10 @@ class AttachmentTable(
*/
fun getAttachmentsThatNeedArchiveUpload(): List<AttachmentId> {
return readableDatabase
.select(ID)
.from(TABLE_NAME)
.where("($ARCHIVE_TRANSFER_STATE = ? or $ARCHIVE_TRANSFER_STATE = ?) AND $DATA_FILE NOT NULL AND $TRANSFER_STATE = $TRANSFER_PROGRESS_DONE", ArchiveTransferState.NONE.value, ArchiveTransferState.TEMPORARY_FAILURE.value)
.orderBy("$ID DESC")
.select("$TABLE_NAME.$ID")
.from("$TABLE_NAME LEFT JOIN ${MessageTable.TABLE_NAME} ON $TABLE_NAME.$MESSAGE_ID = ${MessageTable.TABLE_NAME}.${MessageTable.ID}")
.where("($ARCHIVE_TRANSFER_STATE = ? or $ARCHIVE_TRANSFER_STATE = ?) AND $DATA_FILE NOT NULL AND $TRANSFER_STATE = $TRANSFER_PROGRESS_DONE AND (${MessageTable.STORY_TYPE} = 0 OR ${MessageTable.STORY_TYPE} IS NULL)", ArchiveTransferState.NONE.value, ArchiveTransferState.TEMPORARY_FAILURE.value)
.orderBy("$TABLE_NAME.$ID DESC")
.run()
.readToList { AttachmentId(it.requireLong(ID)) }
}
@@ -677,8 +677,8 @@ class AttachmentTable(
*/
fun doAnyAttachmentsNeedArchiveUpload(): Boolean {
return readableDatabase
.exists(TABLE_NAME)
.where("($ARCHIVE_TRANSFER_STATE = ? OR $ARCHIVE_TRANSFER_STATE = ?) AND $DATA_FILE NOT NULL AND $TRANSFER_STATE = $TRANSFER_PROGRESS_DONE", ArchiveTransferState.NONE.value, ArchiveTransferState.TEMPORARY_FAILURE.value)
.exists("$TABLE_NAME LEFT JOIN ${MessageTable.TABLE_NAME} ON $TABLE_NAME.$MESSAGE_ID = ${MessageTable.TABLE_NAME}.${MessageTable.ID}")
.where("($ARCHIVE_TRANSFER_STATE = ? OR $ARCHIVE_TRANSFER_STATE = ?) AND $DATA_FILE NOT NULL AND $TRANSFER_STATE = $TRANSFER_PROGRESS_DONE AND (${MessageTable.STORY_TYPE} = 0 OR ${MessageTable.STORY_TYPE} IS NULL)", ArchiveTransferState.NONE.value, ArchiveTransferState.TEMPORARY_FAILURE.value)
.run()
}
@@ -829,12 +829,13 @@ class AttachmentTable(
SELECT SUM($DATA_SIZE)
FROM (
SELECT DISTINCT $DATA_HASH_END, $REMOTE_KEY, $DATA_SIZE
FROM $TABLE_NAME
FROM $TABLE_NAME LEFT JOIN ${MessageTable.TABLE_NAME} ON $TABLE_NAME.$MESSAGE_ID = ${MessageTable.TABLE_NAME}.${MessageTable.ID}
WHERE
$DATA_FILE NOT NULL AND
$DATA_HASH_END NOT NULL AND
$REMOTE_KEY NOT NULL AND
$ARCHIVE_TRANSFER_STATE NOT IN (${ArchiveTransferState.FINISHED.value}, ${ArchiveTransferState.PERMANENT_FAILURE.value})
$ARCHIVE_TRANSFER_STATE NOT IN (${ArchiveTransferState.FINISHED.value}, ${ArchiveTransferState.PERMANENT_FAILURE.value}) AND
(${MessageTable.STORY_TYPE} = 0 OR ${MessageTable.STORY_TYPE} IS NULL)
)
""".trimIndent()
)
@@ -1490,6 +1491,15 @@ class AttachmentTable(
return getAttachment(result.values.iterator().next()) ?: throw MmsException("Failed to retrieve attachment we just inserted!")
}
fun getMessageId(attachmentId: AttachmentId): Long {
return readableDatabase
.select(MESSAGE_ID)
.from(TABLE_NAME)
.where("$ID = ?", attachmentId.id)
.run()
.readToSingleLong()
}
fun updateMessageId(attachmentIds: Collection<AttachmentId>, mmsId: Long, isStory: Boolean) {
writableDatabase.withinTransaction { db ->
val values = ContentValues(2).apply {

5
app/src/main/java/org/thoughtcrime/securesms/jobs/AttachmentDownloadJob.kt
View File

@@ -220,6 +220,7 @@ class AttachmentDownloadJob private constructor(
}
if (SignalStore.backup.backsUpMedia) {
val isStory = SignalDatabase.messages.isStory(messageId)
when {
attachment.archiveTransferState == AttachmentTable.ArchiveTransferState.FINISHED -> {
Log.i(TAG, "[$attachmentId] Already archived. Skipping.")
@@ -230,6 +231,10 @@ class AttachmentDownloadJob private constructor(
AppDependencies.jobManager.add(UploadAttachmentToArchiveJob(attachmentId))
}
isStory -> {
Log.i(TAG, "[$attachmentId] Attachment is a story. Skipping.")
}
else -> {
Log.i(TAG, "[$attachmentId] Enqueuing job to copy to archive.")
AppDependencies.jobManager.add(CopyAttachmentToArchiveJob(attachmentId))

14
app/src/main/java/org/thoughtcrime/securesms/jobs/AttachmentUploadJob.kt
View File

@@ -17,6 +17,7 @@ import org.thoughtcrime.securesms.attachments.Attachment
import org.thoughtcrime.securesms.attachments.AttachmentId
import org.thoughtcrime.securesms.attachments.AttachmentUploadUtil
import org.thoughtcrime.securesms.attachments.DatabaseAttachment
import org.thoughtcrime.securesms.backup.v2.BackupRepository
import org.thoughtcrime.securesms.database.AttachmentTable
import org.thoughtcrime.securesms.database.SignalDatabase
import org.thoughtcrime.securesms.dependencies.AppDependencies
@@ -139,6 +140,11 @@ class AttachmentUploadJob private constructor(
val timeSinceUpload = System.currentTimeMillis() - databaseAttachment.uploadTimestamp
if (timeSinceUpload < UPLOAD_REUSE_THRESHOLD && !TextUtils.isEmpty(databaseAttachment.remoteLocation)) {
Log.i(TAG, "We can re-use an already-uploaded file. It was uploaded $timeSinceUpload ms (${timeSinceUpload.milliseconds.inRoundedDays()} days) ago. Skipping.")
SignalDatabase.attachments.setTransferState(databaseAttachment.mmsId, attachmentId, AttachmentTable.TRANSFER_PROGRESS_DONE)
if (BackupRepository.shouldCopyAttachmentToArchive(databaseAttachment.attachmentId, databaseAttachment.mmsId)) {
Log.i(TAG, "[$attachmentId] The re-used file was not copied to the archive. Copying now.")
AppDependencies.jobManager.add(CopyAttachmentToArchiveJob(attachmentId))
}
return
} else if (databaseAttachment.uploadTimestamp > 0) {
Log.i(TAG, "This file was previously-uploaded, but too long ago to be re-used. Age: $timeSinceUpload ms (${timeSinceUpload.milliseconds.inRoundedDays()} days)")
@@ -173,10 +179,18 @@ class AttachmentUploadJob private constructor(
val uploadResult: AttachmentUploadResult = SignalNetwork.attachments.uploadAttachmentV4(localAttachment).successOrThrow()
SignalDatabase.attachments.finalizeAttachmentAfterUpload(databaseAttachment.attachmentId, uploadResult)
if (SignalStore.backup.backsUpMedia) {
val messageId = SignalDatabase.attachments.getMessageId(databaseAttachment.attachmentId)
val isStory = SignalDatabase.messages.isStory(messageId)
when {
databaseAttachment.archiveTransferState == AttachmentTable.ArchiveTransferState.FINISHED -> {
Log.i(TAG, "[$attachmentId] Already archived. Skipping.")
}
isStory -> {
Log.i(TAG, "[$attachmentId] Attachment is a story. Skipping.")
}
messageId == AttachmentTable.PREUPLOAD_MESSAGE_ID -> {
Log.i(TAG, "[$attachmentId] Avoid uploading preuploaded attachments to archive. Skipping.")
}
else -> {
Log.i(TAG, "[$attachmentId] Enqueuing job to copy to archive.")
AppDependencies.jobManager.add(CopyAttachmentToArchiveJob(attachmentId))

7
app/src/main/java/org/thoughtcrime/securesms/jobs/CopyAttachmentToArchiveJob.kt
View File

@@ -20,7 +20,6 @@ import org.thoughtcrime.securesms.jobmanager.impl.NoRemoteArchiveGarbageCollecti
import org.thoughtcrime.securesms.jobs.protos.CopyAttachmentToArchiveJobData
import org.thoughtcrime.securesms.keyvalue.SignalStore
import org.whispersystems.signalservice.api.NetworkResult
import java.lang.RuntimeException
import java.util.concurrent.TimeUnit
/**
@@ -91,6 +90,12 @@ class CopyAttachmentToArchiveJob private constructor(private val attachmentId: A
return Result.failure()
}
if (SignalDatabase.messages.isStory(attachment.mmsId)) {
Log.i(TAG, "[$attachmentId] Attachment is a story. Resetting transfer state to none and skipping.")
SignalDatabase.attachments.setArchiveTransferState(attachmentId, AttachmentTable.ArchiveTransferState.NONE)
return Result.success()
}
if (isCanceled) {
Log.w(TAG, "[$attachmentId] Canceled. Refusing to proceed.")
return Result.failure()

6
app/src/main/java/org/thoughtcrime/securesms/jobs/UploadAttachmentToArchiveJob.kt
View File

@@ -125,6 +125,12 @@ class UploadAttachmentToArchiveJob private constructor(
return Result.success()
}
if (SignalDatabase.messages.isStory(attachment.mmsId)) {
Log.i(TAG, "[$attachmentId] Attachment is a story. Resetting transfer state to none and skipping.")
SignalDatabase.attachments.setArchiveTransferState(attachmentId, AttachmentTable.ArchiveTransferState.NONE)
return Result.success()
}
if (attachment.remoteKey == null) {
Log.w(TAG, "[$attachmentId] Attachment is missing remote key! Cannot upload.")
return Result.failure()

37
app/src/main/java/org/thoughtcrime/securesms/sms/MessageSender.java
View File

@@ -31,6 +31,7 @@ import org.signal.core.util.logging.Log;
import org.thoughtcrime.securesms.attachments.Attachment;
import org.thoughtcrime.securesms.attachments.AttachmentId;
import org.thoughtcrime.securesms.attachments.DatabaseAttachment;
import org.thoughtcrime.securesms.backup.v2.BackupRepository;
import org.thoughtcrime.securesms.contacts.sync.ContactDiscovery;
import org.thoughtcrime.securesms.contactshare.Contact;
import org.thoughtcrime.securesms.database.AttachmentTable;
@@ -51,6 +52,7 @@ import org.thoughtcrime.securesms.jobmanager.JobManager;
import org.thoughtcrime.securesms.jobs.AttachmentCompressionJob;
import org.thoughtcrime.securesms.jobs.AttachmentCopyJob;
import org.thoughtcrime.securesms.jobs.AttachmentUploadJob;
import org.thoughtcrime.securesms.jobs.CopyAttachmentToArchiveJob;
import org.thoughtcrime.securesms.jobs.IndividualSendJob;
import org.thoughtcrime.securesms.jobs.ProfileKeySendJob;
import org.thoughtcrime.securesms.jobs.PushDistributionListSendJob;
@@ -274,6 +276,14 @@ public class MessageSender {
false,
insertListener);
for (AttachmentId attachmentId: attachmentIds) {
boolean wasPreuploaded = SignalDatabase.attachments().getMessageId(attachmentId) == AttachmentTable.PREUPLOAD_MESSAGE_ID;
if (wasPreuploaded && BackupRepository.shouldCopyAttachmentToArchive(attachmentId, messageId)) {
Log.i(TAG, "[" + attachmentId + "] Was previously preuploaded and should now be copied to the archive.");
AppDependencies.getJobManager().add(new CopyAttachmentToArchiveJob(attachmentId));
}
}
attachmentDatabase.updateMessageId(attachmentIds, messageId, message.getStoryType().isStory());
sendMessageInternal(context, recipient, SendType.SIGNAL, messageId, jobIds);
@@ -296,15 +306,16 @@ public class MessageSender {
Preconditions.checkArgument(messages.size() > 0, "No messages!");
Preconditions.checkArgument(Stream.of(messages).allMatch(m -> m.getAttachments().isEmpty()), "Messages can't have attachments! They should be pre-uploaded.");
JobManager jobManager = AppDependencies.getJobManager();
AttachmentTable attachmentDatabase = SignalDatabase.attachments();
MessageTable mmsDatabase = SignalDatabase.messages();
ThreadTable threadTable = SignalDatabase.threads();
List<AttachmentId> preUploadAttachmentIds = Stream.of(preUploadResults).map(PreUploadResult::getAttachmentId).toList();
List<String> preUploadJobIds = Stream.of(preUploadResults).map(PreUploadResult::getJobIds).flatMap(Stream::of).toList();
List<Long> messageIds = new ArrayList<>(messages.size());
List<String> messageDependsOnIds = new ArrayList<>(preUploadJobIds);
OutgoingMessage primaryMessage = messages.get(0);
JobManager jobManager = AppDependencies.getJobManager();
AttachmentTable attachmentDatabase = SignalDatabase.attachments();
MessageTable mmsDatabase = SignalDatabase.messages();
ThreadTable threadTable = SignalDatabase.threads();
List<AttachmentId> preUploadAttachmentIds = Stream.of(preUploadResults).map(PreUploadResult::getAttachmentId).toList();
List<String> preUploadJobIds = Stream.of(preUploadResults).map(PreUploadResult::getJobIds).flatMap(Stream::of).toList();
List<Long> messageIds = new ArrayList<>(messages.size());
List<String> messageDependsOnIds = new ArrayList<>(preUploadJobIds);
OutgoingMessage primaryMessage = messages.get(0);
List<AttachmentId> attachmentsWithPreuploadId = preUploadAttachmentIds.stream().filter(id -> SignalDatabase.attachments().getMessageId(id) == AttachmentTable.PREUPLOAD_MESSAGE_ID).collect(Collectors.toList());
mmsDatabase.beginTransaction();
try {
@@ -379,6 +390,14 @@ public class MessageSender {
}
}
for (AttachmentId attachmentId : attachmentsWithPreuploadId) {
long messageId = SignalDatabase.attachments().getMessageId(attachmentId);
if (BackupRepository.shouldCopyAttachmentToArchive(attachmentId, messageId)) {
Log.i(TAG, "[" + attachmentId + "] Was previously preuploaded and should now be copied to the archive.");
jobManager.add(new CopyAttachmentToArchiveJob(attachmentId));
}
}
onMessageSent();
mmsDatabase.setTransactionSuccessful();
} catch (MmsException e) {