diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/BackupRepository.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/BackupRepository.kt index 59dc9a18a3..03be8caeb2 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/BackupRepository.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/BackupRepository.kt @@ -73,8 +73,8 @@ import org.whispersystems.signalservice.api.messages.SignalServiceAttachment.Pro import org.whispersystems.signalservice.api.push.ServiceId.ACI import org.whispersystems.signalservice.api.push.ServiceId.PNI import org.whispersystems.signalservice.internal.crypto.PaddingInputStream +import org.whispersystems.signalservice.internal.push.AttachmentUploadForm import org.whispersystems.signalservice.internal.push.SubscriptionsConfiguration -import org.whispersystems.signalservice.internal.push.http.ResumableUploadSpec import java.io.ByteArrayOutputStream import java.io.File import java.io.IOException @@ -606,27 +606,30 @@ object BackupRepository { } /** - * Retrieves an upload spec that can be used to upload attachment media. + * Retrieves an [AttachmentUploadForm] that can be used to upload an attachment to the transit cdn. + * To continue the upload, use [org.whispersystems.signalservice.api.attachment.AttachmentApi.getResumableUploadSpec]. + * + * It's important to note that in order to get this to the archive cdn, you still need to use [copyAttachmentToArchive]. */ - fun getMediaUploadSpec(secretKey: ByteArray? = null): NetworkResult { + fun getAttachmentUploadForm(): NetworkResult { val backupKey = SignalStore.svr.getOrCreateMasterKey().deriveBackupKey() return initBackupAndFetchAuth(backupKey) .then { credential -> SignalNetwork.archive.getMediaUploadForm(backupKey, SignalStore.account.requireAci(), credential) } - .then { form -> - SignalNetwork.archive.getResumableUploadSpec(form, secretKey) - } } - fun archiveThumbnail(thumbnailAttachment: Attachment, parentAttachment: DatabaseAttachment): NetworkResult { + /** + * Copies a thumbnail that has been uploaded to the transit cdn to the archive cdn. + */ + fun copyThumbnailToArchive(thumbnailAttachment: Attachment, parentAttachment: DatabaseAttachment): NetworkResult { val backupKey = SignalStore.svr.getOrCreateMasterKey().deriveBackupKey() val request = thumbnailAttachment.toArchiveMediaRequest(parentAttachment.getThumbnailMediaName(), backupKey) return initBackupAndFetchAuth(backupKey) .then { credential -> - SignalNetwork.archive.archiveAttachmentMedia( + SignalNetwork.archive.copyAttachmentToArchive( backupKey = backupKey, aci = SignalStore.account.requireAci(), serviceCredential = credential, @@ -635,7 +638,10 @@ object BackupRepository { } } - fun archiveMedia(attachment: DatabaseAttachment): NetworkResult { + /** + * Copies an attachment that has been uploaded to the transit cdn to the archive cdn. + */ + fun copyAttachmentToArchive(attachment: DatabaseAttachment): NetworkResult { val backupKey = SignalStore.svr.getOrCreateMasterKey().deriveBackupKey() return initBackupAndFetchAuth(backupKey) @@ -643,7 +649,7 @@ object BackupRepository { val mediaName = attachment.getMediaName() val request = attachment.toArchiveMediaRequest(mediaName, backupKey) SignalNetwork.archive - .archiveAttachmentMedia( + .copyAttachmentToArchive( backupKey = backupKey, aci = SignalStore.account.requireAci(), serviceCredential = credential, @@ -658,7 +664,7 @@ object BackupRepository { .also { Log.i(TAG, "archiveMediaResult: $it") } } - fun archiveMedia(databaseAttachments: List): NetworkResult { + fun copyAttachmentToArchive(databaseAttachments: List): NetworkResult { val backupKey = SignalStore.svr.getOrCreateMasterKey().deriveBackupKey() return initBackupAndFetchAuth(backupKey) @@ -676,7 +682,7 @@ object BackupRepository { } SignalNetwork.archive - .archiveAttachmentMedia( + .copyAttachmentToArchive( backupKey = backupKey, aci = SignalStore.account.requireAci(), serviceCredential = credential, diff --git a/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/backup/InternalBackupPlaygroundViewModel.kt b/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/backup/InternalBackupPlaygroundViewModel.kt index 320b4fcaa4..c1296a9a64 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/backup/InternalBackupPlaygroundViewModel.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/backup/InternalBackupPlaygroundViewModel.kt @@ -244,7 +244,7 @@ class InternalBackupPlaygroundViewModel : ViewModel() { .filter { attachments.contains(it.dbAttachment.attachmentId) } .map { it.dbAttachment } - BackupRepository.archiveMedia(toArchive) + BackupRepository.copyAttachmentToArchive(toArchive) } .subscribeOn(Schedulers.io()) .observeOn(Schedulers.single()) @@ -268,7 +268,7 @@ class InternalBackupPlaygroundViewModel : ViewModel() { } fun archiveAttachmentMedia(attachment: BackupAttachment) { - disposables += Single.fromCallable { BackupRepository.archiveMedia(attachment.dbAttachment) } + disposables += Single.fromCallable { BackupRepository.copyAttachmentToArchive(attachment.dbAttachment) } .subscribeOn(Schedulers.io()) .observeOn(Schedulers.single()) .doOnSubscribe { _mediaState.set { update(inProgress = inProgressMediaIds + attachment.dbAttachment.attachmentId) } } diff --git a/app/src/main/java/org/thoughtcrime/securesms/database/AttachmentTable.kt b/app/src/main/java/org/thoughtcrime/securesms/database/AttachmentTable.kt index b0c7aa7571..6bac2ce1d8 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/database/AttachmentTable.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/database/AttachmentTable.kt @@ -539,6 +539,54 @@ class AttachmentTable( .readToList { AttachmentId(it.requireLong(ID)) } } + /** + * At archive creation time, we need to ensure that all relevant attachments have populated (key, iv, digest) tuples. + * This does that. + */ + fun createKeyIvDigestForAttachmentsThatNeedArchiveUpload(): Int { + var count = 0 + + writableDatabase.select(ID, REMOTE_KEY, REMOTE_IV, REMOTE_DIGEST, DATA_FILE, DATA_RANDOM) + .from(TABLE_NAME) + .where( + """ + $ARCHIVE_TRANSFER_STATE = ${ArchiveTransferState.NONE.value} AND + $DATA_FILE NOT NULL AND + $TRANSFER_STATE = $TRANSFER_PROGRESS_DONE AND + ( + $REMOTE_KEY IS NULL OR + $REMOTE_IV IS NULL OR + $REMOTE_DIGEST IS NULL + ) + """ + ) + .run() + .forEach { cursor -> + val attachmentId = AttachmentId(cursor.requireLong(ID)) + Log.w(TAG, "[createKeyIvDigestForAttachmentsThatNeedArchiveUpload][$attachmentId] Missing key, iv, or digest. Generating.") + + val key = cursor.requireString(REMOTE_KEY)?.let { Base64.decode(it) } ?: Util.getSecretBytes(64) + val iv = cursor.requireBlob(REMOTE_IV) ?: Util.getSecretBytes(16) + val digest = run { + val fileInfo = getDataFileInfo(attachmentId)!! + calculateDigest(fileInfo, key, iv) + } + + writableDatabase.update(TABLE_NAME) + .values( + REMOTE_KEY to Base64.encodeWithPadding(key), + REMOTE_IV to iv, + REMOTE_DIGEST to digest + ) + .where("$ID = ?", attachmentId.id) + .run() + + count++ + } + + return count + } + /** * Similar to [getAttachmentsThatNeedArchiveUpload], but returns if the list would be non-null in a more efficient way. */ @@ -928,7 +976,7 @@ class AttachmentTable( * @return True if we had to change the digest as part of saving the file, otherwise false. */ @Throws(MmsException::class) - fun finalizeAttachmentAfterDownload(mmsId: Long, attachmentId: AttachmentId, inputStream: LimitedInputStream, iv: ByteArray?): Boolean { + fun finalizeAttachmentAfterDownload(mmsId: Long, attachmentId: AttachmentId, inputStream: LimitedInputStream, iv: ByteArray): Boolean { Log.i(TAG, "[finalizeAttachmentAfterDownload] Finalizing downloaded data for $attachmentId. (MessageId: $mmsId, $attachmentId)") val existingPlaceholder: DatabaseAttachment = getAttachment(attachmentId) ?: throw MmsException("No attachment found for id: $attachmentId") @@ -947,12 +995,8 @@ class AttachmentTable( } else { Log.w(TAG, "[finalizeAttachmentAfterDownload] $attachmentId has non-zero padding bytes. Recomputing digest.") - val stream = PaddingInputStream(getDataStream(fileWriteResult.file, fileWriteResult.random, 0), fileWriteResult.length) val key = Base64.decode(existingPlaceholder.remoteKey!!) - val cipherOutputStream = AttachmentCipherOutputStream(key, iv, NullOutputStream) - - StreamUtil.copy(stream, cipherOutputStream) - cipherOutputStream.transmittedDigest + calculateDigest(fileWriteResult, key, iv) } val digestChanged = !digest.contentEquals(existingPlaceholder.remoteDigest) @@ -1606,6 +1650,37 @@ class AttachmentTable( notifyConversationListeners(threadId) } + /** + * This will ensure that a (key/iv/digest) tuple exists for an attachment, filling each one if necessary. + */ + @Throws(IOException::class) + fun createKeyIvDigestIfNecessary(attachment: DatabaseAttachment) { + if (attachment.remoteKey != null && attachment.remoteIv != null && attachment.remoteDigest != null) { + return + } + + val attachmentId = attachment.attachmentId + + Log.w(TAG, "[createKeyIvDigestIfNecessary][$attachmentId] Missing one of (key, iv, digest). Filling in the gaps.") + + val key = attachment.remoteKey?.let { Base64.decode(it) } ?: Util.getSecretBytes(64) + val iv = attachment.remoteIv ?: Util.getSecretBytes(16) + val digest: ByteArray = run { + val fileInfo = getDataFileInfo(attachmentId) ?: throw IOException("No data file found for $attachmentId!") + calculateDigest(fileInfo, key, iv) + } + + writableDatabase + .update(TABLE_NAME) + .values( + REMOTE_KEY to Base64.encodeWithPadding(key), + REMOTE_IV to iv, + REMOTE_DIGEST to digest + ) + .where("$ID = ?", attachmentId.id) + .run() + } + fun getAttachments(cursor: Cursor): List { return try { if (cursor.getColumnIndex(ATTACHMENT_JSON_ALIAS) != -1) { @@ -1775,6 +1850,22 @@ class AttachmentTable( .run() } + private fun calculateDigest(fileInfo: DataFileWriteResult, key: ByteArray, iv: ByteArray): ByteArray { + return calculateDigest(file = fileInfo.file, random = fileInfo.random, length = fileInfo.length, key = key, iv = iv) + } + + private fun calculateDigest(fileInfo: DataFileInfo, key: ByteArray, iv: ByteArray): ByteArray { + return calculateDigest(file = fileInfo.file, random = fileInfo.random, length = fileInfo.length, key = key, iv = iv) + } + + private fun calculateDigest(file: File, random: ByteArray, length: Long, key: ByteArray, iv: ByteArray): ByteArray { + val stream = PaddingInputStream(getDataStream(file, random, 0), length) + val cipherOutputStream = AttachmentCipherOutputStream(key, iv, NullOutputStream) + + StreamUtil.copy(stream, cipherOutputStream) + return cipherOutputStream.transmittedDigest + } + /** * Deletes the data file if there's no strong references to other attachments. * If deleted, it will also clear all weak references (i.e. quotes) of the attachment. diff --git a/app/src/main/java/org/thoughtcrime/securesms/jobs/ArchiveAttachmentBackfillJob.kt b/app/src/main/java/org/thoughtcrime/securesms/jobs/ArchiveAttachmentBackfillJob.kt index 79a38245f8..9f1d1347a8 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/jobs/ArchiveAttachmentBackfillJob.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/jobs/ArchiveAttachmentBackfillJob.kt @@ -42,6 +42,8 @@ class ArchiveAttachmentBackfillJob private constructor(parameters: Parameters) : val jobs = SignalDatabase.attachments.getAttachmentsThatNeedArchiveUpload() .map { attachmentId -> UploadAttachmentToArchiveJob(attachmentId, forBackfill = true) } + SignalDatabase.attachments.createKeyIvDigestForAttachmentsThatNeedArchiveUpload() + SignalStore.backup.totalAttachmentUploadCount = jobs.size.toLong() SignalStore.backup.currentAttachmentUploadCount = 0 diff --git a/app/src/main/java/org/thoughtcrime/securesms/jobs/ArchiveThumbnailUploadJob.kt b/app/src/main/java/org/thoughtcrime/securesms/jobs/ArchiveThumbnailUploadJob.kt index 34182981e4..db4c2553ca 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/jobs/ArchiveThumbnailUploadJob.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/jobs/ArchiveThumbnailUploadJob.kt @@ -22,6 +22,7 @@ import org.thoughtcrime.securesms.jobmanager.impl.NetworkConstraint import org.thoughtcrime.securesms.jobs.protos.ArchiveThumbnailUploadJobData import org.thoughtcrime.securesms.keyvalue.SignalStore import org.thoughtcrime.securesms.mms.DecryptableStreamUriLoader.DecryptableUri +import org.thoughtcrime.securesms.net.SignalNetwork import org.thoughtcrime.securesms.util.ImageCompressionUtil import org.thoughtcrime.securesms.util.MediaUtil import org.whispersystems.signalservice.api.NetworkResult @@ -93,13 +94,23 @@ class ArchiveThumbnailUploadJob private constructor( val backupKey = SignalStore.svr.getOrCreateMasterKey().deriveBackupKey() - val resumableUpload = when (val result = BackupRepository.getMediaUploadSpec(secretKey = backupKey.deriveThumbnailTransitKey(attachment.getThumbnailMediaName()))) { + val specResult = BackupRepository + .getAttachmentUploadForm() + .then { form -> + SignalNetwork.attachments.getResumableUploadSpec( + key = backupKey.deriveThumbnailTransitKey(attachment.getThumbnailMediaName()), + iv = attachment.remoteIv!!, + uploadForm = form + ) + } + + val resumableUpload = when (specResult) { is NetworkResult.Success -> { Log.d(TAG, "Got an upload spec!") - result.result.toProto() + specResult.result.toProto() } is NetworkResult.ApplicationError -> { - Log.w(TAG, "Failed to get an upload spec due to an application error. Retrying.", result.throwable) + Log.w(TAG, "Failed to get an upload spec due to an application error. Retrying.", specResult.throwable) return Result.retry(defaultBackoff()) } is NetworkResult.NetworkError -> { @@ -107,7 +118,7 @@ class ArchiveThumbnailUploadJob private constructor( return Result.retry(defaultBackoff()) } is NetworkResult.StatusCodeError -> { - Log.w(TAG, "Failed to get an upload spec with status code ${result.code}") + Log.w(TAG, "Failed to get an upload spec with status code ${specResult.code}") return Result.retry(defaultBackoff()) } } @@ -125,7 +136,7 @@ class ArchiveThumbnailUploadJob private constructor( val backupDirectories = BackupRepository.getCdnBackupDirectories().successOrThrow() val mediaSecrets = backupKey.deriveMediaSecrets(attachment.getThumbnailMediaName()) - return when (val result = BackupRepository.archiveThumbnail(attachmentPointer, attachment)) { + return when (val result = BackupRepository.copyThumbnailToArchive(attachmentPointer, attachment)) { is NetworkResult.Success -> { // save attachment thumbnail val archiveMediaId = attachment.archiveMediaId ?: backupKey.deriveMediaId(attachment.getMediaName()).encode() diff --git a/app/src/main/java/org/thoughtcrime/securesms/jobs/AttachmentDownloadJob.kt b/app/src/main/java/org/thoughtcrime/securesms/jobs/AttachmentDownloadJob.kt index a83fa4983f..741f32c48e 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/jobs/AttachmentDownloadJob.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/jobs/AttachmentDownloadJob.kt @@ -408,7 +408,7 @@ class AttachmentDownloadJob private constructor( messageId, attachmentId, LimitedInputStream.withoutLimits((body.source() as Source).buffer().inputStream()), - iv = updatedAttachment.remoteIv + iv = updatedAttachment.remoteIv!! ) } } diff --git a/app/src/main/java/org/thoughtcrime/securesms/jobs/BackupMessagesJob.kt b/app/src/main/java/org/thoughtcrime/securesms/jobs/BackupMessagesJob.kt index 4d6fa60a4f..75c7e87f2a 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/jobs/BackupMessagesJob.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/jobs/BackupMessagesJob.kt @@ -6,6 +6,7 @@ package org.thoughtcrime.securesms.jobs import org.greenrobot.eventbus.EventBus +import org.signal.core.util.Stopwatch import org.signal.core.util.logging.Log import org.thoughtcrime.securesms.backup.v2.BackupRepository import org.thoughtcrime.securesms.backup.v2.BackupV2Event @@ -64,11 +65,17 @@ class BackupMessagesJob private constructor(parameters: Parameters) : Job(parame override fun onFailure() = Unit override fun run(): Result { + val stopwatch = Stopwatch("BackupMessagesJob") + + SignalDatabase.attachments.createKeyIvDigestForAttachmentsThatNeedArchiveUpload().takeIf { it > 0 }?.let { count -> Log.w(TAG, "Needed to create $count key/iv/digests.") } + stopwatch.split("key-iv-digest") + EventBus.getDefault().postSticky(BackupV2Event(type = BackupV2Event.Type.PROGRESS_MESSAGES, count = 0, estimatedTotalCount = 0)) val tempBackupFile = BlobProvider.getInstance().forNonAutoEncryptingSingleSessionOnDisk(AppDependencies.application) val outputStream = FileOutputStream(tempBackupFile) BackupRepository.export(outputStream = outputStream, append = { tempBackupFile.appendBytes(it) }, plaintext = false) + stopwatch.split("export") FileInputStream(tempBackupFile).use { when (val result = BackupRepository.uploadBackupFile(it, tempBackupFile.length())) { @@ -78,6 +85,7 @@ class BackupMessagesJob private constructor(parameters: Parameters) : Job(parame is NetworkResult.ApplicationError -> throw result.throwable } } + stopwatch.split("upload") SignalStore.backup.lastBackupProtoSize = tempBackupFile.length() if (!tempBackupFile.delete()) { @@ -94,6 +102,8 @@ class BackupMessagesJob private constructor(parameters: Parameters) : Job(parame } is NetworkResult.ApplicationError -> throw result.throwable } + stopwatch.split("used-space") + stopwatch.stop(TAG) if (SignalDatabase.attachments.doAnyAttachmentsNeedArchiveUpload()) { Log.i(TAG, "Enqueuing attachment backfill job.") diff --git a/app/src/main/java/org/thoughtcrime/securesms/jobs/CopyAttachmentToArchiveJob.kt b/app/src/main/java/org/thoughtcrime/securesms/jobs/CopyAttachmentToArchiveJob.kt index fe02a6c45b..4efa00e779 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/jobs/CopyAttachmentToArchiveJob.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/jobs/CopyAttachmentToArchiveJob.kt @@ -91,7 +91,7 @@ class CopyAttachmentToArchiveJob private constructor(private val attachmentId: A return Result.success() } - val result = when (val archiveResult = BackupRepository.archiveMedia(attachment)) { + val result = when (val archiveResult = BackupRepository.copyAttachmentToArchive(attachment)) { is NetworkResult.Success -> { Log.i(TAG, "[$attachmentId] Successfully copied the archive tier.") Result.success() diff --git a/app/src/main/java/org/thoughtcrime/securesms/jobs/JobManagerFactories.java b/app/src/main/java/org/thoughtcrime/securesms/jobs/JobManagerFactories.java index 31e3f8e344..a0ffef57ff 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/jobs/JobManagerFactories.java +++ b/app/src/main/java/org/thoughtcrime/securesms/jobs/JobManagerFactories.java @@ -113,7 +113,7 @@ public final class JobManagerFactories { put(AttachmentCopyJob.KEY, new AttachmentCopyJob.Factory()); put(AttachmentDownloadJob.KEY, new AttachmentDownloadJob.Factory()); put(AttachmentHashBackfillJob.KEY, new AttachmentHashBackfillJob.Factory()); - put(AttachmentMarkUploadedJob.KEY, new AttachmentMarkUploadedJob.Factory()); + put(MarkNoteToSelfAttachmentUploadedJob.KEY, new MarkNoteToSelfAttachmentUploadedJob.Factory()); put(AttachmentUploadJob.KEY, new AttachmentUploadJob.Factory()); put(AutomaticSessionResetJob.KEY, new AutomaticSessionResetJob.Factory()); put(AvatarGroupsV1DownloadJob.KEY, new AvatarGroupsV1DownloadJob.Factory()); diff --git a/app/src/main/java/org/thoughtcrime/securesms/jobs/AttachmentMarkUploadedJob.java b/app/src/main/java/org/thoughtcrime/securesms/jobs/MarkNoteToSelfAttachmentUploadedJob.java similarity index 60% rename from app/src/main/java/org/thoughtcrime/securesms/jobs/AttachmentMarkUploadedJob.java rename to app/src/main/java/org/thoughtcrime/securesms/jobs/MarkNoteToSelfAttachmentUploadedJob.java index 1c3b206857..4293f7bcf0 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/jobs/AttachmentMarkUploadedJob.java +++ b/app/src/main/java/org/thoughtcrime/securesms/jobs/MarkNoteToSelfAttachmentUploadedJob.java @@ -11,18 +11,18 @@ import org.thoughtcrime.securesms.database.SignalDatabase; import org.thoughtcrime.securesms.jobmanager.JsonJobData; import org.thoughtcrime.securesms.jobmanager.Job; -import java.io.IOException; import java.util.concurrent.TimeUnit; /** - * Only marks an attachment as uploaded. + * Marks a note to self attachment (that didn't need to be uploaded, because there's no linked devices) as being uploaded for UX purposes. + * Also generates a key/iv/digest that otherwise wouldn't exist due to the lack of upload. */ -public final class AttachmentMarkUploadedJob extends BaseJob { +public final class MarkNoteToSelfAttachmentUploadedJob extends BaseJob { public static final String KEY = "AttachmentMarkUploadedJob"; @SuppressWarnings("unused") - private static final String TAG = Log.tag(AttachmentMarkUploadedJob.class); + private static final String TAG = Log.tag(MarkNoteToSelfAttachmentUploadedJob.class); private static final String KEY_ATTACHMENT_ID = "row_id"; private static final String KEY_MESSAGE_ID = "message_id"; @@ -30,7 +30,7 @@ public final class AttachmentMarkUploadedJob extends BaseJob { private final AttachmentId attachmentId; private final long messageId; - public AttachmentMarkUploadedJob(long messageId, @NonNull AttachmentId attachmentId) { + public MarkNoteToSelfAttachmentUploadedJob(long messageId, @NonNull AttachmentId attachmentId) { this(new Parameters.Builder() .setLifespan(TimeUnit.DAYS.toMillis(1)) .setMaxAttempts(Parameters.UNLIMITED) @@ -39,7 +39,7 @@ public final class AttachmentMarkUploadedJob extends BaseJob { attachmentId); } - private AttachmentMarkUploadedJob(@NonNull Parameters parameters, long messageId, @NonNull AttachmentId attachmentId) { + private MarkNoteToSelfAttachmentUploadedJob(@NonNull Parameters parameters, long messageId, @NonNull AttachmentId attachmentId) { super(parameters); this.attachmentId = attachmentId; this.messageId = messageId; @@ -59,14 +59,14 @@ public final class AttachmentMarkUploadedJob extends BaseJob { @Override public void onRun() throws Exception { - AttachmentTable database = SignalDatabase.attachments(); - DatabaseAttachment databaseAttachment = database.getAttachment(attachmentId); + DatabaseAttachment databaseAttachment = SignalDatabase.attachments().getAttachment(attachmentId); if (databaseAttachment == null) { throw new InvalidAttachmentException("Cannot find the specified attachment."); } - database.markAttachmentUploaded(messageId, databaseAttachment); + SignalDatabase.attachments().markAttachmentUploaded(messageId, databaseAttachment); + SignalDatabase.attachments().createKeyIvDigestIfNecessary(databaseAttachment); } @Override @@ -75,7 +75,7 @@ public final class AttachmentMarkUploadedJob extends BaseJob { @Override protected boolean onShouldRetry(@NonNull Exception exception) { - return exception instanceof IOException; + return false; } private class InvalidAttachmentException extends Exception { @@ -84,14 +84,14 @@ public final class AttachmentMarkUploadedJob extends BaseJob { } } - public static final class Factory implements Job.Factory { + public static final class Factory implements Job.Factory { @Override - public @NonNull AttachmentMarkUploadedJob create(@NonNull Parameters parameters, @Nullable byte[] serializedData) { + public @NonNull MarkNoteToSelfAttachmentUploadedJob create(@NonNull Parameters parameters, @Nullable byte[] serializedData) { JsonJobData data = JsonJobData.deserialize(serializedData); - return new AttachmentMarkUploadedJob(parameters, - data.getLong(KEY_MESSAGE_ID), - new AttachmentId(data.getLong(KEY_ATTACHMENT_ID))); + return new MarkNoteToSelfAttachmentUploadedJob(parameters, + data.getLong(KEY_MESSAGE_ID), + new AttachmentId(data.getLong(KEY_ATTACHMENT_ID))); } } } diff --git a/app/src/main/java/org/thoughtcrime/securesms/jobs/UploadAttachmentToArchiveJob.kt b/app/src/main/java/org/thoughtcrime/securesms/jobs/UploadAttachmentToArchiveJob.kt index ba135a1d5d..aebc7cd8ef 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/jobs/UploadAttachmentToArchiveJob.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/jobs/UploadAttachmentToArchiveJob.kt @@ -5,6 +5,7 @@ package org.thoughtcrime.securesms.jobs +import org.signal.core.util.Base64 import org.signal.core.util.logging.Log import org.signal.protos.resumableuploads.ResumableUpload import org.thoughtcrime.securesms.attachments.AttachmentId @@ -100,6 +101,12 @@ class UploadAttachmentToArchiveJob private constructor( return Result.success() } + if (attachment.remoteKey == null || attachment.remoteIv == null) { + Log.w(TAG, "[$attachmentId] Attachment is missing remote key or IV! Cannot upload.") + SignalDatabase.attachments.setArchiveTransferState(attachmentId, AttachmentTable.ArchiveTransferState.NONE) + return Result.failure() + } + if (uploadSpec != null && System.currentTimeMillis() > uploadSpec!!.timeout) { Log.w(TAG, "[$attachmentId] Upload spec expired! Clearing.") uploadSpec = null @@ -108,7 +115,7 @@ class UploadAttachmentToArchiveJob private constructor( if (uploadSpec == null) { Log.d(TAG, "[$attachmentId] Need an upload spec. Fetching...") - val (spec, result) = fetchResumableUploadSpec() + val (spec, result) = fetchResumableUploadSpec(key = Base64.decode(attachment.remoteKey), iv = attachment.remoteIv) if (result != null) { return result } @@ -154,15 +161,19 @@ class UploadAttachmentToArchiveJob private constructor( override fun onFailure() = Unit - private fun fetchResumableUploadSpec(): Pair { - return when (val spec = BackupRepository.getMediaUploadSpec()) { + private fun fetchResumableUploadSpec(key: ByteArray, iv: ByteArray): Pair { + val uploadSpec = BackupRepository + .getAttachmentUploadForm() + .then { form -> SignalNetwork.attachments.getResumableUploadSpec(key, iv, form) } + + return when (uploadSpec) { is NetworkResult.Success -> { Log.d(TAG, "[$attachmentId] Got an upload spec!") - spec.result.toProto() to null + uploadSpec.result.toProto() to null } is NetworkResult.ApplicationError -> { - Log.w(TAG, "[$attachmentId] Failed to get an upload spec due to an application error. Retrying.", spec.throwable) + Log.w(TAG, "[$attachmentId] Failed to get an upload spec due to an application error. Retrying.", uploadSpec.throwable) return null to Result.retry(defaultBackoff()) } @@ -172,9 +183,9 @@ class UploadAttachmentToArchiveJob private constructor( } is NetworkResult.StatusCodeError -> { - Log.w(TAG, "[$attachmentId] Failed request with status code ${spec.code}") + Log.w(TAG, "[$attachmentId] Failed request with status code ${uploadSpec.code}") - when (ArchiveMediaUploadFormStatusCodes.from(spec.code)) { + when (ArchiveMediaUploadFormStatusCodes.from(uploadSpec.code)) { ArchiveMediaUploadFormStatusCodes.BadArguments, ArchiveMediaUploadFormStatusCodes.InvalidPresentationOrSignature, ArchiveMediaUploadFormStatusCodes.InsufficientPermissions, diff --git a/app/src/main/java/org/thoughtcrime/securesms/sms/MessageSender.java b/app/src/main/java/org/thoughtcrime/securesms/sms/MessageSender.java index 92bf90b530..94c6755a99 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/sms/MessageSender.java +++ b/app/src/main/java/org/thoughtcrime/securesms/sms/MessageSender.java @@ -50,7 +50,7 @@ import org.thoughtcrime.securesms.jobmanager.Job; import org.thoughtcrime.securesms.jobmanager.JobManager; import org.thoughtcrime.securesms.jobs.AttachmentCompressionJob; import org.thoughtcrime.securesms.jobs.AttachmentCopyJob; -import org.thoughtcrime.securesms.jobs.AttachmentMarkUploadedJob; +import org.thoughtcrime.securesms.jobs.MarkNoteToSelfAttachmentUploadedJob; import org.thoughtcrime.securesms.jobs.AttachmentUploadJob; import org.thoughtcrime.securesms.jobs.IndividualSendJob; import org.thoughtcrime.securesms.jobs.ProfileKeySendJob; @@ -643,9 +643,9 @@ public class MessageSender { .map(a -> AttachmentCompressionJob.fromAttachment((DatabaseAttachment) a, false, -1)) .toList(); - List fakeUploadJobs = Stream.of(attachments) - .map(a -> new AttachmentMarkUploadedJob(messageId, ((DatabaseAttachment) a).attachmentId)) - .toList(); + List fakeUploadJobs = Stream.of(attachments) + .map(a -> new MarkNoteToSelfAttachmentUploadedJob(messageId, ((DatabaseAttachment) a).attachmentId)) + .toList(); AppDependencies.getJobManager().startChain(compressionJobs) .then(fakeUploadJobs) diff --git a/libsignal-service/src/main/java/org/whispersystems/signalservice/api/archive/ArchiveApi.kt b/libsignal-service/src/main/java/org/whispersystems/signalservice/api/archive/ArchiveApi.kt index f033894861..867ebd8bf6 100644 --- a/libsignal-service/src/main/java/org/whispersystems/signalservice/api/archive/ArchiveApi.kt +++ b/libsignal-service/src/main/java/org/whispersystems/signalservice/api/archive/ArchiveApi.kt @@ -17,7 +17,6 @@ import org.whispersystems.signalservice.api.backup.BackupKey import org.whispersystems.signalservice.api.push.ServiceId.ACI import org.whispersystems.signalservice.internal.push.AttachmentUploadForm import org.whispersystems.signalservice.internal.push.PushServiceSocket -import org.whispersystems.signalservice.internal.push.http.ResumableUploadSpec import java.io.InputStream import java.time.Instant @@ -146,7 +145,11 @@ class ArchiveApi(private val pushServiceSocket: PushServiceSocket) { /** * Retrieves an [AttachmentUploadForm] that can be used to upload pre-existing media to the archive. - * After uploading, the media still needs to be copied via [archiveAttachmentMedia]. + * + * This is basically the same as [org.whispersystems.signalservice.api.attachment.AttachmentApi.getAttachmentV4UploadForm], but with a relaxed rate limit + * so we can request them more often (which is required for backfilling). + * + * After uploading, the media still needs to be copied via [copyAttachmentToArchive]. */ fun getMediaUploadForm(backupKey: BackupKey, aci: ACI, serviceCredential: ArchiveServiceCredential): NetworkResult { return NetworkResult.fromFetch { @@ -156,16 +159,6 @@ class ArchiveApi(private val pushServiceSocket: PushServiceSocket) { } } - fun getResumableUploadSpec(uploadForm: AttachmentUploadForm, secretKey: ByteArray?): NetworkResult { - return NetworkResult.fromFetch { - if (secretKey == null) { - pushServiceSocket.getResumableUploadSpec(uploadForm) - } else { - pushServiceSocket.getResumableUploadSpecWithKey(uploadForm, secretKey) - } - } - } - /** * Retrieves all media items in the user's archive. Note that this could be a very large number of items, making this only suitable for debugging. * Use [getArchiveMediaItemsPage] in production. @@ -210,7 +203,7 @@ class ArchiveApi(private val pushServiceSocket: PushServiceSocket) { * 413: No media space remaining * 429: Rate-limited */ - fun archiveAttachmentMedia( + fun copyAttachmentToArchive( backupKey: BackupKey, aci: ACI, serviceCredential: ArchiveServiceCredential, @@ -227,7 +220,7 @@ class ArchiveApi(private val pushServiceSocket: PushServiceSocket) { /** * Copy and re-encrypt media from the attachments cdn into the backup cdn. */ - fun archiveAttachmentMedia( + fun copyAttachmentToArchive( backupKey: BackupKey, aci: ACI, serviceCredential: ArchiveServiceCredential,