From 4b47d38d78425e2f5b7075302db6d012faa3e707 Mon Sep 17 00:00:00 2001 From: Greyson Parrelli Date: Fri, 30 Aug 2024 12:11:22 -0400 Subject: [PATCH] Add IV to the attachment table. --- .../database/AttachmentTableTest_deduping.kt | 45 +++---- ...ageProcessorTest_synchronizeDeleteForMe.kt | 43 +++++-- .../attachments/ArchivedAttachment.kt | 2 + .../securesms/attachments/Attachment.kt | 3 + .../attachments/DatabaseAttachment.kt | 2 + .../attachments/PointerAttachment.kt | 17 ++- .../attachments/TombstoneAttachment.kt | 2 + .../securesms/attachments/UriAttachment.kt | 1 + .../v2/database/ChatItemImportInserter.kt | 1 + .../securesms/database/AttachmentTable.kt | 48 +++++-- .../helpers/SignalDatabaseMigrations.kt | 6 +- .../migration/V244_AttachmentRemoteIv.kt | 18 +++ .../jobs/ArchiveAttachmentBackfillJob.kt | 19 ++- .../securesms/jobs/AttachmentUploadJob.kt | 24 +++- .../sms/UploadDependencyGraphTest.kt | 1 + .../securesms/database/FakeMessageRecords.kt | 74 +++++------ .../signalservice/api/NetworkResult.kt | 81 ++++++++++++ .../api/SignalServiceMessageSender.java | 13 +- .../api/attachment/AttachmentApi.kt | 120 +++++++++++++++++ .../api/attachment/AttachmentUploadResult.kt | 23 ++++ .../internal/crypto/AttachmentDigest.kt | 6 +- .../internal/push/PushAttachmentData.java | 74 ----------- .../internal/push/PushAttachmentData.kt | 26 ++++ .../internal/push/PushServiceSocket.java | 2 +- .../push/http/ResumableUploadSpec.java | 121 ------------------ .../internal/push/http/ResumableUploadSpec.kt | 71 ++++++++++ 26 files changed, 534 insertions(+), 309 deletions(-) create mode 100644 app/src/main/java/org/thoughtcrime/securesms/database/helpers/migration/V244_AttachmentRemoteIv.kt create mode 100644 libsignal-service/src/main/java/org/whispersystems/signalservice/api/attachment/AttachmentApi.kt create mode 100644 libsignal-service/src/main/java/org/whispersystems/signalservice/api/attachment/AttachmentUploadResult.kt delete mode 100644 libsignal-service/src/main/java/org/whispersystems/signalservice/internal/push/PushAttachmentData.java create mode 100644 libsignal-service/src/main/java/org/whispersystems/signalservice/internal/push/PushAttachmentData.kt delete mode 100644 libsignal-service/src/main/java/org/whispersystems/signalservice/internal/push/http/ResumableUploadSpec.java create mode 100644 libsignal-service/src/main/java/org/whispersystems/signalservice/internal/push/http/ResumableUploadSpec.kt diff --git a/app/src/androidTest/java/org/thoughtcrime/securesms/database/AttachmentTableTest_deduping.kt b/app/src/androidTest/java/org/thoughtcrime/securesms/database/AttachmentTableTest_deduping.kt index 2940cc0204..9f6212a81c 100644 --- a/app/src/androidTest/java/org/thoughtcrime/securesms/database/AttachmentTableTest_deduping.kt +++ b/app/src/androidTest/java/org/thoughtcrime/securesms/database/AttachmentTableTest_deduping.kt @@ -15,7 +15,6 @@ import org.signal.core.util.Base64 import org.signal.core.util.update import org.thoughtcrime.securesms.attachments.AttachmentId import org.thoughtcrime.securesms.attachments.Cdn -import org.thoughtcrime.securesms.attachments.PointerAttachment import org.thoughtcrime.securesms.backup.v2.BackupRepository.getMediaName import org.thoughtcrime.securesms.database.AttachmentTable.TransformProperties import org.thoughtcrime.securesms.keyvalue.SignalStore @@ -27,7 +26,9 @@ import org.thoughtcrime.securesms.providers.BlobProvider import org.thoughtcrime.securesms.recipients.Recipient import org.thoughtcrime.securesms.util.MediaUtil import org.thoughtcrime.securesms.util.Util +import org.whispersystems.signalservice.api.attachment.AttachmentUploadResult import org.whispersystems.signalservice.api.backup.MediaId +import org.whispersystems.signalservice.api.messages.SignalServiceAttachmentRemoteId import org.whispersystems.signalservice.api.push.ServiceId import java.io.File import java.util.UUID @@ -661,7 +662,7 @@ class AttachmentTableTest_deduping { } fun upload(attachmentId: AttachmentId, uploadTimestamp: Long = System.currentTimeMillis()) { - SignalDatabase.attachments.finalizeAttachmentAfterUpload(attachmentId, createPointerAttachment(attachmentId, uploadTimestamp), uploadTimestamp) + SignalDatabase.attachments.finalizeAttachmentAfterUpload(attachmentId, createUploadResult(attachmentId, uploadTimestamp)) val attachment = SignalDatabase.attachments.getAttachment(attachmentId)!! SignalDatabase.attachments.setArchiveData( @@ -763,6 +764,7 @@ class AttachmentTableTest_deduping { assertEquals(lhsAttachment.remoteLocation, rhsAttachment.remoteLocation) assertEquals(lhsAttachment.remoteKey, rhsAttachment.remoteKey) + assertArrayEquals(lhsAttachment.remoteIv, rhsAttachment.remoteIv) assertArrayEquals(lhsAttachment.remoteDigest, rhsAttachment.remoteDigest) assertArrayEquals(lhsAttachment.incrementalDigest, rhsAttachment.incrementalDigest) assertEquals(lhsAttachment.incrementalMacChunkSize, rhsAttachment.incrementalMacChunkSize) @@ -796,36 +798,19 @@ class AttachmentTableTest_deduping { return MediaStream(this.inputStream(), MediaUtil.IMAGE_JPEG, 2, 2) } - private fun createPointerAttachment(attachmentId: AttachmentId, uploadTimestamp: Long = System.currentTimeMillis()): PointerAttachment { - val location = "somewhere-${Random.nextLong()}" - val key = "somekey-${Random.nextLong()}" - val digest = Random.nextBytes(32) - val incrementalDigest = Random.nextBytes(16) - + private fun createUploadResult(attachmentId: AttachmentId, uploadTimestamp: Long = System.currentTimeMillis()): AttachmentUploadResult { val databaseAttachment = SignalDatabase.attachments.getAttachment(attachmentId)!! - return PointerAttachment( - "image/jpeg", - AttachmentTable.TRANSFER_PROGRESS_DONE, - databaseAttachment.size, // size - null, - Cdn.CDN_3, // cdnNumber - location, - key, - digest, - incrementalDigest, - 5, // incrementalMacChunkSize - null, - databaseAttachment.voiceNote, - databaseAttachment.borderless, - databaseAttachment.videoGif, - databaseAttachment.width, - databaseAttachment.height, - uploadTimestamp, - databaseAttachment.caption, - databaseAttachment.stickerLocator, - databaseAttachment.blurHash, - databaseAttachment.uuid + return AttachmentUploadResult( + remoteId = SignalServiceAttachmentRemoteId.V4("somewhere-${Random.nextLong()}"), + cdnNumber = Cdn.CDN_3.cdnNumber, + key = databaseAttachment.remoteKey?.let { Base64.decode(it) } ?: Util.getSecretBytes(64), + iv = databaseAttachment.remoteIv ?: Util.getSecretBytes(16), + digest = Random.nextBytes(32), + incrementalDigest = Random.nextBytes(16), + incrementalDigestChunkSize = 5, + uploadTimestamp = uploadTimestamp, + dataSize = databaseAttachment.size ) } } diff --git a/app/src/androidTest/java/org/thoughtcrime/securesms/messages/SyncMessageProcessorTest_synchronizeDeleteForMe.kt b/app/src/androidTest/java/org/thoughtcrime/securesms/messages/SyncMessageProcessorTest_synchronizeDeleteForMe.kt index f0a3abccfc..5231d50b2e 100644 --- a/app/src/androidTest/java/org/thoughtcrime/securesms/messages/SyncMessageProcessorTest_synchronizeDeleteForMe.kt +++ b/app/src/androidTest/java/org/thoughtcrime/securesms/messages/SyncMessageProcessorTest_synchronizeDeleteForMe.kt @@ -13,6 +13,7 @@ import org.junit.Ignore import org.junit.Rule import org.junit.Test import org.junit.runner.RunWith +import org.signal.core.util.Base64 import org.signal.core.util.logging.Log import org.signal.core.util.update import org.signal.core.util.withinTransaction @@ -33,6 +34,9 @@ import org.thoughtcrime.securesms.testing.assertIsNot import org.thoughtcrime.securesms.testing.assertIsNotNull import org.thoughtcrime.securesms.testing.assertIsSize import org.thoughtcrime.securesms.util.IdentityUtil +import org.thoughtcrime.securesms.util.Util +import org.whispersystems.signalservice.api.attachment.AttachmentUploadResult +import org.whispersystems.signalservice.api.messages.SignalServiceAttachmentRemoteId import java.util.UUID @Suppress("ClassName") @@ -574,30 +578,35 @@ class SyncMessageProcessorTest_synchronizeDeleteForMe { // Has all three SignalDatabase.attachments.finalizeAttachmentAfterUpload( id = attachments[0].attachmentId, - attachment = attachments[0].copy(digest = byteArrayOf(attachments[0].attachmentId.id.toByte())), - uploadTimestamp = message1.timestamp + 1 + uploadResult = attachments[0].toUploadResult( + digest = byteArrayOf(attachments[0].attachmentId.id.toByte()), + uploadTimestamp = message1.timestamp + 1 + ) ) // Missing uuid and digest SignalDatabase.attachments.finalizeAttachmentAfterUpload( id = attachments[1].attachmentId, - attachment = attachments[1], - uploadTimestamp = message1.timestamp + 1 + uploadResult = attachments[1].toUploadResult(uploadTimestamp = message1.timestamp + 1) ) // Missing uuid and plain text SignalDatabase.attachments.finalizeAttachmentAfterUpload( id = attachments[2].attachmentId, - attachment = attachments[2].copy(digest = byteArrayOf(attachments[2].attachmentId.id.toByte())), - uploadTimestamp = message1.timestamp + 1 + uploadResult = attachments[2].toUploadResult( + digest = byteArrayOf(attachments[2].attachmentId.id.toByte()), + uploadTimestamp = message1.timestamp + 1 + ) ) SignalDatabase.rawDatabase.update(AttachmentTable.TABLE_NAME).values(AttachmentTable.DATA_HASH_END to null).where("${AttachmentTable.ID} = ?", attachments[2].attachmentId).run() // Different has all three SignalDatabase.attachments.finalizeAttachmentAfterUpload( id = attachments[3].attachmentId, - attachment = attachments[3].copy(digest = byteArrayOf(attachments[3].attachmentId.id.toByte())), - uploadTimestamp = message1.timestamp + 1 + uploadResult = attachments[3].toUploadResult( + digest = byteArrayOf(attachments[3].attachmentId.id.toByte()), + uploadTimestamp = message1.timestamp + 1 + ) ) attachments = SignalDatabase.attachments.getAttachmentsForMessage(message1.messageId) @@ -674,6 +683,7 @@ class SyncMessageProcessorTest_synchronizeDeleteForMe { cdn = this.cdn, location = this.remoteLocation, key = this.remoteKey, + iv = this.remoteIv, digest = digest, incrementalDigest = this.incrementalDigest, incrementalMacChunkSize = this.incrementalMacChunkSize, @@ -700,4 +710,21 @@ class SyncMessageProcessorTest_synchronizeDeleteForMe { uuid = uuid ) } + + private fun Attachment.toUploadResult( + digest: ByteArray = this.remoteDigest ?: byteArrayOf(), + uploadTimestamp: Long = this.uploadTimestamp + ): AttachmentUploadResult { + return AttachmentUploadResult( + remoteId = SignalServiceAttachmentRemoteId.V4(this.remoteLocation ?: "some-location"), + cdnNumber = this.cdn.cdnNumber, + key = this.remoteKey?.let { Base64.decode(it) } ?: Util.getSecretBytes(64), + iv = this.remoteIv ?: Util.getSecretBytes(16), + digest = digest, + incrementalDigest = this.incrementalDigest, + incrementalDigestChunkSize = this.incrementalMacChunkSize, + dataSize = this.size, + uploadTimestamp = uploadTimestamp + ) + } } diff --git a/app/src/main/java/org/thoughtcrime/securesms/attachments/ArchivedAttachment.kt b/app/src/main/java/org/thoughtcrime/securesms/attachments/ArchivedAttachment.kt index fc9038f0c0..b227f3ee16 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/attachments/ArchivedAttachment.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/attachments/ArchivedAttachment.kt @@ -32,6 +32,7 @@ class ArchivedAttachment : Attachment { size: Long, cdn: Int, key: ByteArray, + iv: ByteArray?, cdnKey: String?, archiveCdn: Int?, archiveMediaName: String, @@ -60,6 +61,7 @@ class ArchivedAttachment : Attachment { cdn = Cdn.fromCdnNumber(cdn), remoteLocation = cdnKey, remoteKey = Base64.encodeWithoutPadding(key), + remoteIv = iv, remoteDigest = digest, incrementalDigest = incrementalMac, fastPreflightId = null, diff --git a/app/src/main/java/org/thoughtcrime/securesms/attachments/Attachment.kt b/app/src/main/java/org/thoughtcrime/securesms/attachments/Attachment.kt index 32f20a866b..27a54c5c07 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/attachments/Attachment.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/attachments/Attachment.kt @@ -37,6 +37,8 @@ abstract class Attachment( @JvmField val remoteKey: String?, @JvmField + val remoteIv: ByteArray?, + @JvmField val remoteDigest: ByteArray?, @JvmField val incrementalDigest: ByteArray?, @@ -86,6 +88,7 @@ abstract class Attachment( cdn = Cdn.deserialize(parcel.readInt()), remoteLocation = parcel.readString(), remoteKey = parcel.readString(), + remoteIv = ParcelUtil.readByteArray(parcel), remoteDigest = ParcelUtil.readByteArray(parcel), incrementalDigest = ParcelUtil.readByteArray(parcel), fastPreflightId = parcel.readString(), diff --git a/app/src/main/java/org/thoughtcrime/securesms/attachments/DatabaseAttachment.kt b/app/src/main/java/org/thoughtcrime/securesms/attachments/DatabaseAttachment.kt index 0e1a16c992..e3c54c82c5 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/attachments/DatabaseAttachment.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/attachments/DatabaseAttachment.kt @@ -58,6 +58,7 @@ class DatabaseAttachment : Attachment { cdn: Cdn, location: String?, key: String?, + iv: ByteArray?, digest: ByteArray?, incrementalDigest: ByteArray?, incrementalMacChunkSize: Int, @@ -90,6 +91,7 @@ class DatabaseAttachment : Attachment { cdn = cdn, remoteLocation = location, remoteKey = key, + remoteIv = iv, remoteDigest = digest, incrementalDigest = incrementalDigest, fastPreflightId = fastPreflightId, diff --git a/app/src/main/java/org/thoughtcrime/securesms/attachments/PointerAttachment.kt b/app/src/main/java/org/thoughtcrime/securesms/attachments/PointerAttachment.kt index c98468740f..0ddef132e8 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/attachments/PointerAttachment.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/attachments/PointerAttachment.kt @@ -3,7 +3,7 @@ package org.thoughtcrime.securesms.attachments import android.net.Uri import android.os.Parcel import androidx.annotation.VisibleForTesting -import org.signal.core.util.Base64.encodeWithPadding +import org.signal.core.util.Base64 import org.thoughtcrime.securesms.blurhash.BlurHash import org.thoughtcrime.securesms.database.AttachmentTable import org.thoughtcrime.securesms.stickers.StickerLocator @@ -24,6 +24,7 @@ class PointerAttachment : Attachment { cdn: Cdn, location: String, key: String?, + iv: ByteArray?, digest: ByteArray?, incrementalDigest: ByteArray?, incrementalMacChunkSize: Int, @@ -46,6 +47,7 @@ class PointerAttachment : Attachment { cdn = cdn, remoteLocation = location, remoteKey = key, + remoteIv = iv, remoteDigest = digest, incrementalDigest = incrementalDigest, fastPreflightId = fastPreflightId, @@ -86,12 +88,17 @@ class PointerAttachment : Attachment { @JvmStatic @JvmOverloads - fun forPointer(pointer: Optional, stickerLocator: StickerLocator? = null, fastPreflightId: String? = null, transferState: Int = AttachmentTable.TRANSFER_PROGRESS_PENDING): Optional { + fun forPointer( + pointer: Optional, + stickerLocator: StickerLocator? = null, + fastPreflightId: String? = null, + transferState: Int = AttachmentTable.TRANSFER_PROGRESS_PENDING + ): Optional { if (!pointer.isPresent || !pointer.get().isPointer()) { return Optional.empty() } - val encodedKey: String? = pointer.get().asPointer().key?.let { encodeWithPadding(it) } + val encodedKey: String? = pointer.get().asPointer().key?.let { Base64.encodeWithPadding(it) } return Optional.of( PointerAttachment( @@ -102,6 +109,7 @@ class PointerAttachment : Attachment { cdn = Cdn.fromCdnNumber(pointer.get().asPointer().cdnNumber), location = pointer.get().asPointer().remoteId.toString(), key = encodedKey, + iv = null, digest = pointer.get().asPointer().digest.orElse(null), incrementalDigest = pointer.get().asPointer().incrementalDigest.orElse(null), incrementalMacChunkSize = pointer.get().asPointer().incrementalMacChunkSize, @@ -139,7 +147,8 @@ class PointerAttachment : Attachment { fileName = quotedAttachment.fileName, cdn = Cdn.fromCdnNumber(thumbnail?.asPointer()?.cdnNumber ?: 0), location = thumbnail?.asPointer()?.remoteId?.toString() ?: "0", - key = thumbnail?.asPointer()?.key?.let { encodeWithPadding(it) }, + key = thumbnail?.asPointer()?.key?.let { Base64.encodeWithPadding(it) }, + iv = null, digest = thumbnail?.asPointer()?.digest?.orElse(null), incrementalDigest = thumbnail?.asPointer()?.incrementalDigest?.orElse(null), incrementalMacChunkSize = thumbnail?.asPointer()?.incrementalMacChunkSize ?: 0, diff --git a/app/src/main/java/org/thoughtcrime/securesms/attachments/TombstoneAttachment.kt b/app/src/main/java/org/thoughtcrime/securesms/attachments/TombstoneAttachment.kt index cb3430f86f..3dd2898617 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/attachments/TombstoneAttachment.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/attachments/TombstoneAttachment.kt @@ -22,6 +22,7 @@ class TombstoneAttachment : Attachment { cdn = Cdn.CDN_0, remoteLocation = null, remoteKey = null, + remoteIv = null, remoteDigest = null, incrementalDigest = null, fastPreflightId = null, @@ -62,6 +63,7 @@ class TombstoneAttachment : Attachment { cdn = Cdn.CDN_0, remoteLocation = null, remoteKey = null, + remoteIv = null, remoteDigest = null, incrementalDigest = incrementalMac, fastPreflightId = null, diff --git a/app/src/main/java/org/thoughtcrime/securesms/attachments/UriAttachment.kt b/app/src/main/java/org/thoughtcrime/securesms/attachments/UriAttachment.kt index 18439eb2e1..e6182fc95a 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/attachments/UriAttachment.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/attachments/UriAttachment.kt @@ -75,6 +75,7 @@ class UriAttachment : Attachment { cdn = Cdn.CDN_0, remoteLocation = null, remoteKey = null, + remoteIv = null, remoteDigest = null, incrementalDigest = null, fastPreflightId = fastPreflightId, diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/ChatItemImportInserter.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/ChatItemImportInserter.kt index 2922164555..e7b71a3801 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/ChatItemImportInserter.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/ChatItemImportInserter.kt @@ -1030,6 +1030,7 @@ class ChatItemImportInserter( size = this.backupLocator.size.toLong(), cdn = this.backupLocator.transitCdnNumber ?: Cdn.CDN_0.cdnNumber, key = this.backupLocator.key.toByteArray(), + iv = null, cdnKey = this.backupLocator.transitCdnKey, archiveCdn = this.backupLocator.cdnNumber, archiveMediaName = this.backupLocator.mediaName, diff --git a/app/src/main/java/org/thoughtcrime/securesms/database/AttachmentTable.kt b/app/src/main/java/org/thoughtcrime/securesms/database/AttachmentTable.kt index e7ad7290a0..46c28cc8af 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/database/AttachmentTable.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/database/AttachmentTable.kt @@ -90,7 +90,9 @@ import org.thoughtcrime.securesms.util.FileUtils import org.thoughtcrime.securesms.util.JsonUtils.SaneJSONObject import org.thoughtcrime.securesms.util.MediaUtil import org.thoughtcrime.securesms.util.StorageUtil +import org.thoughtcrime.securesms.util.Util import org.thoughtcrime.securesms.video.EncryptedMediaDataSource +import org.whispersystems.signalservice.api.attachment.AttachmentUploadResult import org.whispersystems.signalservice.api.util.UuidUtil import org.whispersystems.signalservice.internal.util.JsonUtil import java.io.File @@ -118,6 +120,7 @@ class AttachmentTable( const val MESSAGE_ID = "message_id" const val CONTENT_TYPE = "content_type" const val REMOTE_KEY = "remote_key" + const val REMOTE_IV = "remote_iv" const val REMOTE_LOCATION = "remote_location" const val REMOTE_DIGEST = "remote_digest" const val REMOTE_INCREMENTAL_DIGEST = "remote_incremental_digest" @@ -178,6 +181,7 @@ class AttachmentTable( MESSAGE_ID, CONTENT_TYPE, REMOTE_KEY, + REMOTE_IV, REMOTE_LOCATION, REMOTE_DIGEST, REMOTE_INCREMENTAL_DIGEST, @@ -263,7 +267,8 @@ class AttachmentTable( $THUMBNAIL_FILE TEXT DEFAULT NULL, $THUMBNAIL_RANDOM BLOB DEFAULT NULL, $THUMBNAIL_RESTORE_STATE INTEGER DEFAULT ${ThumbnailRestoreState.NONE.value}, - $ATTACHMENT_UUID TEXT DEFAULT NULL + $ATTACHMENT_UUID TEXT DEFAULT NULL, + $REMOTE_IV BLOB DEFAULT NULL ) """ @@ -1026,7 +1031,7 @@ class AttachmentTable( * it's ending hash, which is critical for backups. */ @Throws(IOException::class) - fun finalizeAttachmentAfterUpload(id: AttachmentId, attachment: Attachment, uploadTimestamp: Long) { + fun finalizeAttachmentAfterUpload(id: AttachmentId, uploadResult: AttachmentUploadResult) { Log.i(TAG, "[finalizeAttachmentAfterUpload] Finalizing upload for $id.") val dataStream = getAttachmentStream(id, 0) @@ -1040,17 +1045,14 @@ class AttachmentTable( val values = contentValuesOf( TRANSFER_STATE to TRANSFER_PROGRESS_DONE, - CDN_NUMBER to attachment.cdn.serialize(), - REMOTE_LOCATION to attachment.remoteLocation, - REMOTE_DIGEST to attachment.remoteDigest, - REMOTE_INCREMENTAL_DIGEST to attachment.incrementalDigest, - REMOTE_INCREMENTAL_DIGEST_CHUNK_SIZE to attachment.incrementalMacChunkSize, - REMOTE_KEY to attachment.remoteKey, - DATA_SIZE to attachment.size, + CDN_NUMBER to uploadResult.cdnNumber, + REMOTE_LOCATION to uploadResult.remoteId.toString(), + REMOTE_DIGEST to uploadResult.digest, + REMOTE_INCREMENTAL_DIGEST to uploadResult.incrementalDigest, + REMOTE_INCREMENTAL_DIGEST_CHUNK_SIZE to uploadResult.incrementalDigestChunkSize, + DATA_SIZE to uploadResult.dataSize, DATA_HASH_END to dataHashEnd, - FAST_PREFLIGHT_ID to attachment.fastPreflightId, - BLUR_HASH to attachment.getVisualHashStringOrNull(), - UPLOAD_TIMESTAMP to uploadTimestamp + UPLOAD_TIMESTAMP to uploadResult.uploadTimestamp ) val dataFilePath = getDataFilePath(id) ?: throw IOException("No data file found for attachment!") @@ -1153,6 +1155,25 @@ class AttachmentTable( } } + fun createKeyIvIfNecessary(attachmentId: AttachmentId) { + val key = Util.getSecretBytes(64) + val iv = Util.getSecretBytes(16) + + writableDatabase.withinTransaction { + writableDatabase + .update(TABLE_NAME) + .values(REMOTE_KEY to Base64.encodeWithPadding(key)) + .where("$ID = ? AND $REMOTE_KEY IS NULL", attachmentId.id) + .run() + + writableDatabase + .update(TABLE_NAME) + .values(REMOTE_IV to iv) + .where("$ID = ? AND $REMOTE_IV IS NULL", attachmentId.id) + .run() + } + } + /** * Inserts new attachments in the table. The [Attachment]s may or may not have data, depending on whether it's an attachment we created locally or some * inbound attachment that we haven't fetched yet. @@ -1507,6 +1528,7 @@ class AttachmentTable( cdn = Cdn.deserialize(jsonObject.getInt(CDN_NUMBER)), location = jsonObject.getString(REMOTE_LOCATION), key = jsonObject.getString(REMOTE_KEY), + iv = null, digest = null, incrementalDigest = null, incrementalMacChunkSize = 0, @@ -2040,6 +2062,7 @@ class AttachmentTable( contentValues.put(REMOTE_INCREMENTAL_DIGEST, uploadTemplate?.incrementalDigest) contentValues.put(REMOTE_INCREMENTAL_DIGEST_CHUNK_SIZE, uploadTemplate?.incrementalMacChunkSize ?: 0) contentValues.put(REMOTE_KEY, uploadTemplate?.remoteKey) + contentValues.put(REMOTE_IV, uploadTemplate?.remoteIv) contentValues.put(FILE_NAME, StorageUtil.getCleanFileName(attachment.fileName)) contentValues.put(FAST_PREFLIGHT_ID, attachment.fastPreflightId) contentValues.put(VOICE_NOTE, if (attachment.voiceNote) 1 else 0) @@ -2120,6 +2143,7 @@ class AttachmentTable( cdn = cursor.requireObject(CDN_NUMBER, Cdn.Serializer), location = cursor.requireString(REMOTE_LOCATION), key = cursor.requireString(REMOTE_KEY), + iv = cursor.requireBlob(REMOTE_IV), digest = cursor.requireBlob(REMOTE_DIGEST), incrementalDigest = cursor.requireBlob(REMOTE_INCREMENTAL_DIGEST), incrementalMacChunkSize = cursor.requireInt(REMOTE_INCREMENTAL_DIGEST_CHUNK_SIZE), diff --git a/app/src/main/java/org/thoughtcrime/securesms/database/helpers/SignalDatabaseMigrations.kt b/app/src/main/java/org/thoughtcrime/securesms/database/helpers/SignalDatabaseMigrations.kt index 21e71679fe..6ce97bae95 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/database/helpers/SignalDatabaseMigrations.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/database/helpers/SignalDatabaseMigrations.kt @@ -101,6 +101,7 @@ import org.thoughtcrime.securesms.database.helpers.migration.V240_MessageFullTex import org.thoughtcrime.securesms.database.helpers.migration.V241_ExpireTimerVersion import org.thoughtcrime.securesms.database.helpers.migration.V242_MessageFullTextSearchEmojiSupportV2 import org.thoughtcrime.securesms.database.helpers.migration.V243_MessageFullTextSearchDisableSecureDelete +import org.thoughtcrime.securesms.database.helpers.migration.V244_AttachmentRemoteIv /** * Contains all of the database migrations for [SignalDatabase]. Broken into a separate file for cleanliness. @@ -204,10 +205,11 @@ object SignalDatabaseMigrations { 240 to V240_MessageFullTextSearchSecureDelete, 241 to V241_ExpireTimerVersion, 242 to V242_MessageFullTextSearchEmojiSupportV2, - 243 to V243_MessageFullTextSearchDisableSecureDelete + 243 to V243_MessageFullTextSearchDisableSecureDelete, + 244 to V244_AttachmentRemoteIv ) - const val DATABASE_VERSION = 243 + const val DATABASE_VERSION = 244 @JvmStatic fun migrate(context: Application, db: SQLiteDatabase, oldVersion: Int, newVersion: Int) { diff --git a/app/src/main/java/org/thoughtcrime/securesms/database/helpers/migration/V244_AttachmentRemoteIv.kt b/app/src/main/java/org/thoughtcrime/securesms/database/helpers/migration/V244_AttachmentRemoteIv.kt new file mode 100644 index 0000000000..153bd7e883 --- /dev/null +++ b/app/src/main/java/org/thoughtcrime/securesms/database/helpers/migration/V244_AttachmentRemoteIv.kt @@ -0,0 +1,18 @@ +/* + * Copyright 2024 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ + +package org.thoughtcrime.securesms.database.helpers.migration + +import android.app.Application +import net.zetetic.database.sqlcipher.SQLiteDatabase + +/** + * Adds the remoteIv column to attachments. + */ +object V244_AttachmentRemoteIv : SignalDatabaseMigration { + override fun migrate(context: Application, db: SQLiteDatabase, oldVersion: Int, newVersion: Int) { + db.execSQL("ALTER TABLE attachment ADD COLUMN remote_iv BLOB DEFAULT NULL;") + } +} diff --git a/app/src/main/java/org/thoughtcrime/securesms/jobs/ArchiveAttachmentBackfillJob.kt b/app/src/main/java/org/thoughtcrime/securesms/jobs/ArchiveAttachmentBackfillJob.kt index 30766d8e36..50aa176445 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/jobs/ArchiveAttachmentBackfillJob.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/jobs/ArchiveAttachmentBackfillJob.kt @@ -8,11 +8,9 @@ package org.thoughtcrime.securesms.jobs import org.greenrobot.eventbus.EventBus import org.signal.core.util.logging.Log import org.signal.protos.resumableuploads.ResumableUpload -import org.thoughtcrime.securesms.attachments.Attachment import org.thoughtcrime.securesms.attachments.AttachmentId import org.thoughtcrime.securesms.attachments.AttachmentUploadUtil import org.thoughtcrime.securesms.attachments.DatabaseAttachment -import org.thoughtcrime.securesms.attachments.PointerAttachment import org.thoughtcrime.securesms.backup.v2.BackupRepository import org.thoughtcrime.securesms.backup.v2.BackupV2Event import org.thoughtcrime.securesms.database.AttachmentTable @@ -24,9 +22,8 @@ import org.thoughtcrime.securesms.jobs.protos.ArchiveAttachmentBackfillJobData import org.whispersystems.signalservice.api.NetworkResult import org.whispersystems.signalservice.api.archive.ArchiveMediaResponse import org.whispersystems.signalservice.api.archive.ArchiveMediaUploadFormStatusCodes -import org.whispersystems.signalservice.api.messages.SignalServiceAttachmentPointer +import org.whispersystems.signalservice.api.attachment.AttachmentUploadResult import java.io.IOException -import java.util.Optional import kotlin.time.Duration.Companion.days /** @@ -159,16 +156,16 @@ class ArchiveAttachmentBackfillJob private constructor( } Log.d(TAG, "Beginning upload...") - val remoteAttachment: SignalServiceAttachmentPointer = try { - AppDependencies.signalServiceMessageSender.uploadAttachment(attachmentStream) - } catch (e: IOException) { - Log.w(TAG, "Failed to upload $attachmentId", e) - return Result.retry(defaultBackoff()) + val attachmentApi = AppDependencies.signalServiceMessageSender.attachmentApi + val uploadResult: AttachmentUploadResult = when (val result = attachmentApi.uploadAttachmentV4(attachmentStream)) { + is NetworkResult.Success -> result.result + is NetworkResult.ApplicationError -> throw result.throwable + is NetworkResult.NetworkError -> return Result.retry(defaultBackoff()) + is NetworkResult.StatusCodeError -> return Result.retry(defaultBackoff()) } Log.d(TAG, "Upload complete!") - val pointerAttachment: Attachment = PointerAttachment.forPointer(Optional.of(remoteAttachment), null, attachmentRecord.fastPreflightId).get() - SignalDatabase.attachments.finalizeAttachmentAfterUpload(attachmentRecord.attachmentId, pointerAttachment, remoteAttachment.uploadTimestamp) + SignalDatabase.attachments.finalizeAttachmentAfterUpload(attachmentRecord.attachmentId, uploadResult) SignalDatabase.attachments.setArchiveTransferState(attachmentRecord.attachmentId, AttachmentTable.ArchiveTransferState.BACKFILL_UPLOADED) attachmentRecord = SignalDatabase.attachments.getAttachment(attachmentRecord.attachmentId) diff --git a/app/src/main/java/org/thoughtcrime/securesms/jobs/AttachmentUploadJob.kt b/app/src/main/java/org/thoughtcrime/securesms/jobs/AttachmentUploadJob.kt index db1fef790f..78587cef86 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/jobs/AttachmentUploadJob.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/jobs/AttachmentUploadJob.kt @@ -7,6 +7,7 @@ package org.thoughtcrime.securesms.jobs import android.text.TextUtils import okhttp3.internal.http2.StreamResetException import org.greenrobot.eventbus.EventBus +import org.signal.core.util.Base64 import org.signal.core.util.concurrent.SignalExecutors import org.signal.core.util.inRoundedDays import org.signal.core.util.logging.Log @@ -17,7 +18,6 @@ import org.thoughtcrime.securesms.attachments.Attachment import org.thoughtcrime.securesms.attachments.AttachmentId import org.thoughtcrime.securesms.attachments.AttachmentUploadUtil import org.thoughtcrime.securesms.attachments.DatabaseAttachment -import org.thoughtcrime.securesms.attachments.PointerAttachment import org.thoughtcrime.securesms.database.AttachmentTable import org.thoughtcrime.securesms.database.SignalDatabase import org.thoughtcrime.securesms.dependencies.AppDependencies @@ -32,6 +32,7 @@ import org.thoughtcrime.securesms.net.NotPushRegisteredException import org.thoughtcrime.securesms.recipients.Recipient import org.thoughtcrime.securesms.service.AttachmentProgressService import org.thoughtcrime.securesms.util.RemoteConfig +import org.whispersystems.signalservice.api.attachment.AttachmentUploadResult import org.whispersystems.signalservice.api.crypto.AttachmentCipherStreamUtil import org.whispersystems.signalservice.api.messages.SignalServiceAttachment import org.whispersystems.signalservice.api.messages.SignalServiceAttachmentStream @@ -39,7 +40,6 @@ import org.whispersystems.signalservice.api.push.exceptions.NonSuccessfulResumab import org.whispersystems.signalservice.api.push.exceptions.ResumeLocationInvalidException import org.whispersystems.signalservice.internal.crypto.PaddingInputStream import java.io.IOException -import java.util.Optional import java.util.concurrent.TimeUnit import kotlin.time.Duration.Companion.days import kotlin.time.Duration.Companion.milliseconds @@ -136,7 +136,10 @@ class AttachmentUploadJob private constructor( throw NotPushRegisteredException() } + SignalDatabase.attachments.createKeyIvIfNecessary(attachmentId) + val messageSender = AppDependencies.signalServiceMessageSender + val attachmentApi = messageSender.attachmentApi val databaseAttachment = SignalDatabase.attachments.getAttachment(attachmentId) ?: throw InvalidAttachmentException("Cannot find the specified attachment.") val timeSinceUpload = System.currentTimeMillis() - databaseAttachment.uploadTimestamp @@ -154,7 +157,17 @@ class AttachmentUploadJob private constructor( if (uploadSpec == null) { Log.d(TAG, "Need an upload spec. Fetching...") - uploadSpec = AppDependencies.signalServiceMessageSender.getResumableUploadSpec().toProto() + uploadSpec = attachmentApi + .getAttachmentV4UploadForm() + .then { form -> + attachmentApi.getResumableUploadSpec( + key = Base64.decode(databaseAttachment.remoteKey!!), + iv = databaseAttachment.remoteIv!!, + uploadForm = form + ) + } + .successOrThrow() + .toProto() } else { Log.d(TAG, "Re-using existing upload spec.") } @@ -163,9 +176,8 @@ class AttachmentUploadJob private constructor( try { getAttachmentNotificationIfNeeded(databaseAttachment).use { notification -> buildAttachmentStream(databaseAttachment, notification, uploadSpec!!).use { localAttachment -> - val remoteAttachment = messageSender.uploadAttachment(localAttachment) - val attachment = PointerAttachment.forPointer(Optional.of(remoteAttachment), null, databaseAttachment.fastPreflightId).get() - SignalDatabase.attachments.finalizeAttachmentAfterUpload(databaseAttachment.attachmentId, attachment, remoteAttachment.uploadTimestamp) + val uploadResult: AttachmentUploadResult = attachmentApi.uploadAttachmentV4(localAttachment).successOrThrow() + SignalDatabase.attachments.finalizeAttachmentAfterUpload(databaseAttachment.attachmentId, uploadResult) ArchiveThumbnailUploadJob.enqueueIfNecessary(databaseAttachment.attachmentId) } } diff --git a/app/src/test/java/org/thoughtcrime/securesms/sms/UploadDependencyGraphTest.kt b/app/src/test/java/org/thoughtcrime/securesms/sms/UploadDependencyGraphTest.kt index 0bf8f48a82..9a93a15968 100644 --- a/app/src/test/java/org/thoughtcrime/securesms/sms/UploadDependencyGraphTest.kt +++ b/app/src/test/java/org/thoughtcrime/securesms/sms/UploadDependencyGraphTest.kt @@ -240,6 +240,7 @@ class UploadDependencyGraphTest { cdn = attachment.cdn, location = attachment.remoteLocation, key = attachment.remoteKey, + iv = attachment.remoteIv, digest = attachment.remoteDigest, incrementalDigest = attachment.incrementalDigest, incrementalMacChunkSize = attachment.incrementalMacChunkSize, diff --git a/app/src/testShared/org/thoughtcrime/securesms/database/FakeMessageRecords.kt b/app/src/testShared/org/thoughtcrime/securesms/database/FakeMessageRecords.kt index 12611faef3..a88c90501a 100644 --- a/app/src/testShared/org/thoughtcrime/securesms/database/FakeMessageRecords.kt +++ b/app/src/testShared/org/thoughtcrime/securesms/database/FakeMessageRecords.kt @@ -40,6 +40,7 @@ object FakeMessageRecords { cdnNumber: Int = 3, location: String = "", key: String = "", + iv: ByteArray = byteArrayOf(), relay: String = "", digest: ByteArray = byteArrayOf(), incrementalDigest: ByteArray = byteArrayOf(), @@ -67,42 +68,43 @@ object FakeMessageRecords { thumbnailRestoreState: AttachmentTable.ThumbnailRestoreState = AttachmentTable.ThumbnailRestoreState.NONE ): DatabaseAttachment { return DatabaseAttachment( - attachmentId, - mmsId, - hasData, - hasThumbnail, - hasArchiveThumbnail, - contentType, - transferProgress, - size, - fileName, - Cdn.fromCdnNumber(cdnNumber), - location, - key, - digest, - incrementalDigest, - incrementalMacChunkSize, - fastPreflightId, - voiceNote, - borderless, - videoGif, - width, - height, - quote, - caption, - stickerLocator, - blurHash, - audioHash, - transformProperties, - displayOrder, - uploadTimestamp, - dataHash, - archiveCdn, - archiveThumbnailCdn, - archiveMediaId, - archiveMediaName, - thumbnailRestoreState, - null + attachmentId = attachmentId, + mmsId = mmsId, + hasData = hasData, + hasThumbnail = hasThumbnail, + hasArchiveThumbnail = hasArchiveThumbnail, + contentType = contentType, + transferProgress = transferProgress, + size = size, + fileName = fileName, + cdn = Cdn.fromCdnNumber(cdnNumber), + location = location, + key = key, + iv = iv, + digest = digest, + incrementalDigest = incrementalDigest, + incrementalMacChunkSize = incrementalMacChunkSize, + fastPreflightId = fastPreflightId, + voiceNote = voiceNote, + borderless = borderless, + videoGif = videoGif, + width = width, + height = height, + quote = quote, + caption = caption, + stickerLocator = stickerLocator, + blurHash = blurHash, + audioHash = audioHash, + transformProperties = transformProperties, + displayOrder = displayOrder, + uploadTimestamp = uploadTimestamp, + dataHash = dataHash, + archiveCdn = archiveCdn, + archiveThumbnailCdn = archiveThumbnailCdn, + archiveMediaName = archiveMediaId, + archiveMediaId = archiveMediaName, + thumbnailRestoreState = thumbnailRestoreState, + uuid = null ) } diff --git a/libsignal-service/src/main/java/org/whispersystems/signalservice/api/NetworkResult.kt b/libsignal-service/src/main/java/org/whispersystems/signalservice/api/NetworkResult.kt index 8b5940a957..cd97d3a185 100644 --- a/libsignal-service/src/main/java/org/whispersystems/signalservice/api/NetworkResult.kt +++ b/libsignal-service/src/main/java/org/whispersystems/signalservice/api/NetworkResult.kt @@ -6,7 +6,11 @@ package org.whispersystems.signalservice.api import org.whispersystems.signalservice.api.push.exceptions.NonSuccessfulResponseCodeException +import org.whispersystems.signalservice.internal.util.JsonUtil +import org.whispersystems.signalservice.internal.websocket.WebSocketRequestMessage +import org.whispersystems.signalservice.internal.websocket.WebsocketResponse import java.io.IOException +import kotlin.reflect.KClass typealias StatusCodeErrorAction = (NetworkResult.StatusCodeError<*>) -> Unit @@ -43,6 +47,50 @@ sealed class NetworkResult( } catch (e: Throwable) { ApplicationError(e) } + + /** + * A convenience method to convert a websocket request into a network result with simple conversion of the response body to the desired class. + * Common exceptions will be caught and translated to errors. + */ + @JvmStatic + fun fromWebSocketRequest( + signalWebSocket: SignalWebSocket, + request: WebSocketRequestMessage, + clazz: KClass + ): NetworkResult = try { + val result = signalWebSocket.request(request) + .map { response: WebsocketResponse -> JsonUtil.fromJson(response.body, clazz.java) } + .blockingGet() + Success(result) + } catch (e: NonSuccessfulResponseCodeException) { + StatusCodeError(e) + } catch (e: IOException) { + NetworkError(e) + } catch (e: Throwable) { + ApplicationError(e) + } + + /** + * A convenience method to convert a websocket request into a network result with the ability to convert the response to your target class. + * Common exceptions will be caught and translated to errors. + */ + @JvmStatic + fun fromWebSocketRequest( + signalWebSocket: SignalWebSocket, + request: WebSocketRequestMessage, + webSocketResponseConverter: WebSocketResponseConverter + ): NetworkResult = try { + val result = signalWebSocket.request(request) + .map { response: WebsocketResponse -> webSocketResponseConverter.convert(response) } + .blockingGet() + Success(result) + } catch (e: NonSuccessfulResponseCodeException) { + StatusCodeError(e) + } catch (e: IOException) { + NetworkError(e) + } catch (e: Throwable) { + ApplicationError(e) + } } /** Indicates the request was successful */ @@ -105,6 +153,34 @@ sealed class NetworkResult( } } + /** + * Provides the ability to fallback to [fromFetch] if the current [NetworkResult] is non-successful. + * + * The [fallback] will only be triggered on non-[Success] results. You can provide a [unless] to limit what kinds of errors you fallback on + * (the default is to fallback on every error). + * + * This primary usecase of this is to make a websocket request (see [fromWebSocketRequest]) and fallback to rest upon failure. + * + * ```kotlin + * val user: NetworkResult = NetworkResult + * .fromWebSocketRequest(websocket, request, LocalUserMode.class.java) + * .fallbackTo { result -> NetworkResult.fromFetch { http.getUser() } } + * ``` + * + * @param unless If this lamba returns true, the fallback will not be triggered. + */ + fun fallbackToFetch(unless: (NetworkResult) -> Boolean = { false }, fallback: Fetcher): NetworkResult { + if (this is Success) { + return this + } + + return if (unless(this)) { + fromFetch(fallback) + } else { + this + } + } + /** * Takes the output of one [NetworkResult] and passes it as the input to another if the operation is successful. * If it's non-successful, the [result] lambda is not run, and instead the original failure will be propagated. @@ -183,4 +259,9 @@ sealed class NetworkResult( @Throws(Exception::class) fun fetch(): T } + + fun interface WebSocketResponseConverter { + @Throws(Exception::class) + fun convert(response: WebsocketResponse): T + } } diff --git a/libsignal-service/src/main/java/org/whispersystems/signalservice/api/SignalServiceMessageSender.java b/libsignal-service/src/main/java/org/whispersystems/signalservice/api/SignalServiceMessageSender.java index 95ac92370f..69c8b96305 100644 --- a/libsignal-service/src/main/java/org/whispersystems/signalservice/api/SignalServiceMessageSender.java +++ b/libsignal-service/src/main/java/org/whispersystems/signalservice/api/SignalServiceMessageSender.java @@ -23,6 +23,7 @@ import org.signal.libsignal.protocol.state.SessionRecord; import org.signal.libsignal.protocol.util.Pair; import org.signal.libsignal.zkgroup.groupsend.GroupSendFullToken; import org.signal.libsignal.zkgroup.profiles.ClientZkProfileOperations; +import org.whispersystems.signalservice.api.attachment.AttachmentApi; import org.whispersystems.signalservice.api.crypto.AttachmentCipherStreamUtil; import org.whispersystems.signalservice.api.crypto.ContentHint; import org.whispersystems.signalservice.api.crypto.EnvelopeContent; @@ -170,6 +171,7 @@ public class SignalServiceMessageSender { private static final int RETRY_COUNT = 4; private final PushServiceSocket socket; + private final SignalWebSocket webSocket; private final SignalServiceAccountDataStore aciStore; private final SignalSessionLock sessionLock; private final SignalServiceAddress localAddress; @@ -198,6 +200,7 @@ public class SignalServiceMessageSender { boolean automaticNetworkRetry) { this.socket = new PushServiceSocket(urls, credentialsProvider, signalAgent, clientZkProfileOperations, automaticNetworkRetry); + this.webSocket = signalWebSocket; this.aciStore = store.aci(); this.sessionLock = sessionLock; this.localAddress = new SignalServiceAddress(credentialsProvider.getAci(), credentialsProvider.getE164()); @@ -212,6 +215,10 @@ public class SignalServiceMessageSender { this.scheduler = Schedulers.from(executor, false, false); } + public AttachmentApi getAttachmentApi() { + return AttachmentApi.create(webSocket, socket); + } + /** * Send a read receipt for a received message. * @@ -799,8 +806,8 @@ public class SignalServiceMessageSender { } public SignalServiceAttachmentPointer uploadAttachment(SignalServiceAttachmentStream attachment) throws IOException { - byte[] attachmentKey = attachment.getResumableUploadSpec().map(ResumableUploadSpec::getSecretKey).orElseGet(() -> Util.getSecretBytes(64)); - byte[] attachmentIV = attachment.getResumableUploadSpec().map(ResumableUploadSpec::getIV).orElseGet(() -> Util.getSecretBytes(16)); + byte[] attachmentKey = attachment.getResumableUploadSpec().map(ResumableUploadSpec::getAttachmentKey).orElseGet(() -> Util.getSecretBytes(64)); + byte[] attachmentIV = attachment.getResumableUploadSpec().map(ResumableUploadSpec::getAttachmentIv).orElseGet(() -> Util.getSecretBytes(16)); long paddedLength = PaddingInputStream.getPaddedSize(attachment.getLength()); InputStream dataStream = new PaddingInputStream(attachment.getInputStream(), attachment.getLength()); long ciphertextLength = AttachmentCipherStreamUtil.getCiphertextLength(paddedLength); @@ -811,7 +818,7 @@ public class SignalServiceMessageSender { new AttachmentCipherOutputStreamFactory(attachmentKey, attachmentIV), attachment.getListener(), attachment.getCancelationSignal(), - attachment.getResumableUploadSpec().orElse(null)); + attachment.getResumableUploadSpec().get()); if (attachment.getResumableUploadSpec().isEmpty()) { throw new IllegalStateException("Attachment must have a resumable upload spec."); diff --git a/libsignal-service/src/main/java/org/whispersystems/signalservice/api/attachment/AttachmentApi.kt b/libsignal-service/src/main/java/org/whispersystems/signalservice/api/attachment/AttachmentApi.kt new file mode 100644 index 0000000000..a6b3279339 --- /dev/null +++ b/libsignal-service/src/main/java/org/whispersystems/signalservice/api/attachment/AttachmentApi.kt @@ -0,0 +1,120 @@ +/* + * Copyright 2024 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ + +package org.whispersystems.signalservice.api.attachment + +import org.whispersystems.signalservice.api.NetworkResult +import org.whispersystems.signalservice.api.SignalWebSocket +import org.whispersystems.signalservice.api.crypto.AttachmentCipherStreamUtil +import org.whispersystems.signalservice.api.messages.SignalServiceAttachmentRemoteId +import org.whispersystems.signalservice.api.messages.SignalServiceAttachmentStream +import org.whispersystems.signalservice.internal.crypto.PaddingInputStream +import org.whispersystems.signalservice.internal.push.AttachmentUploadForm +import org.whispersystems.signalservice.internal.push.PushAttachmentData +import org.whispersystems.signalservice.internal.push.PushServiceSocket +import org.whispersystems.signalservice.internal.push.http.AttachmentCipherOutputStreamFactory +import org.whispersystems.signalservice.internal.push.http.ResumableUploadSpec +import org.whispersystems.signalservice.internal.websocket.WebSocketRequestMessage +import java.io.InputStream +import java.security.SecureRandom + +/** + * Class to interact with various attachment-related endpoints. + */ +class AttachmentApi( + private val signalWebSocket: SignalWebSocket, + private val pushServiceSocket: PushServiceSocket +) { + companion object { + @JvmStatic + fun create(signalWebSocket: SignalWebSocket, pushServiceSocket: PushServiceSocket): AttachmentApi { + return AttachmentApi(signalWebSocket, pushServiceSocket) + } + } + + /** + * Gets a v4 attachment upload form, which provides the necessary information to upload an attachment. + */ + fun getAttachmentV4UploadForm(): NetworkResult { + val request = WebSocketRequestMessage( + id = SecureRandom().nextLong(), + verb = "GET", + path = "/v4/attachments/form/upload" + ) + + return NetworkResult + .fromWebSocketRequest(signalWebSocket, request, AttachmentUploadForm::class) + .fallbackToFetch( + unless = { it is NetworkResult.StatusCodeError && it.code == 209 }, + fallback = { pushServiceSocket.attachmentV4UploadAttributes } + ) + } + + /** + * Gets a resumable upload spec, which can be saved and re-used across upload attempts to resume upload progress. + */ + fun getResumableUploadSpec(key: ByteArray, iv: ByteArray, uploadForm: AttachmentUploadForm): NetworkResult { + return getResumableUploadUrl(uploadForm) + .map { url -> + ResumableUploadSpec( + attachmentKey = key, + attachmentIv = iv, + cdnKey = uploadForm.key, + cdnNumber = uploadForm.cdn, + resumeLocation = url, + expirationTimestamp = System.currentTimeMillis() + PushServiceSocket.CDN2_RESUMABLE_LINK_LIFETIME_MILLIS, + headers = uploadForm.headers + ) + } + } + + /** + * Uploads an attachment using the v4 upload scheme. + */ + fun uploadAttachmentV4(attachmentStream: SignalServiceAttachmentStream): NetworkResult { + if (attachmentStream.resumableUploadSpec.isEmpty) { + throw IllegalStateException("Attachment must have a resumable upload spec!") + } + + return NetworkResult.fromFetch { + val resumableUploadSpec = attachmentStream.resumableUploadSpec.get() + + val paddedLength = PaddingInputStream.getPaddedSize(attachmentStream.length) + val dataStream: InputStream = PaddingInputStream(attachmentStream.inputStream, attachmentStream.length) + val ciphertextLength = AttachmentCipherStreamUtil.getCiphertextLength(paddedLength) + + val attachmentData = PushAttachmentData( + contentType = attachmentStream.contentType, + data = dataStream, + dataSize = ciphertextLength, + incremental = attachmentStream.isFaststart, + outputStreamFactory = AttachmentCipherOutputStreamFactory(resumableUploadSpec.attachmentKey, resumableUploadSpec.attachmentIv), + listener = attachmentStream.listener, + cancelationSignal = attachmentStream.cancelationSignal, + resumableUploadSpec = attachmentStream.resumableUploadSpec.get() + ) + + val digestInfo = pushServiceSocket.uploadAttachment(attachmentData) + + AttachmentUploadResult( + remoteId = SignalServiceAttachmentRemoteId.V4(attachmentData.resumableUploadSpec.cdnKey), + cdnNumber = attachmentData.resumableUploadSpec.cdnNumber, + key = resumableUploadSpec.attachmentKey, + iv = resumableUploadSpec.attachmentIv, + digest = digestInfo.digest, + incrementalDigest = digestInfo.incrementalDigest, + incrementalDigestChunkSize = digestInfo.incrementalMacChunkSize, + uploadTimestamp = attachmentStream.uploadTimestamp, + dataSize = attachmentData.dataSize + ) + } + } + + private fun getResumableUploadUrl(uploadForm: AttachmentUploadForm): NetworkResult { + return NetworkResult.fromFetch { + pushServiceSocket.getResumableUploadUrl(uploadForm) + } + } +} diff --git a/libsignal-service/src/main/java/org/whispersystems/signalservice/api/attachment/AttachmentUploadResult.kt b/libsignal-service/src/main/java/org/whispersystems/signalservice/api/attachment/AttachmentUploadResult.kt new file mode 100644 index 0000000000..7f358b48ed --- /dev/null +++ b/libsignal-service/src/main/java/org/whispersystems/signalservice/api/attachment/AttachmentUploadResult.kt @@ -0,0 +1,23 @@ +/* + * Copyright 2024 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ + +package org.whispersystems.signalservice.api.attachment + +import org.whispersystems.signalservice.api.messages.SignalServiceAttachmentRemoteId + +/** + * The result of uploading an attachment. Just the additional metadata related to the upload itself. + */ +class AttachmentUploadResult( + val remoteId: SignalServiceAttachmentRemoteId, + val cdnNumber: Int, + val key: ByteArray, + val iv: ByteArray, + val digest: ByteArray, + val incrementalDigest: ByteArray?, + val incrementalDigestChunkSize: Int, + val dataSize: Long, + val uploadTimestamp: Long +) diff --git a/libsignal-service/src/main/java/org/whispersystems/signalservice/internal/crypto/AttachmentDigest.kt b/libsignal-service/src/main/java/org/whispersystems/signalservice/internal/crypto/AttachmentDigest.kt index debd703b0c..2d0d647e9e 100644 --- a/libsignal-service/src/main/java/org/whispersystems/signalservice/internal/crypto/AttachmentDigest.kt +++ b/libsignal-service/src/main/java/org/whispersystems/signalservice/internal/crypto/AttachmentDigest.kt @@ -5,4 +5,8 @@ package org.whispersystems.signalservice.internal.crypto -data class AttachmentDigest(val digest: ByteArray, val incrementalDigest: ByteArray?, val incrementalMacChunkSize: Int) +data class AttachmentDigest( + val digest: ByteArray, + val incrementalDigest: ByteArray?, + val incrementalMacChunkSize: Int +) diff --git a/libsignal-service/src/main/java/org/whispersystems/signalservice/internal/push/PushAttachmentData.java b/libsignal-service/src/main/java/org/whispersystems/signalservice/internal/push/PushAttachmentData.java deleted file mode 100644 index 384528effd..0000000000 --- a/libsignal-service/src/main/java/org/whispersystems/signalservice/internal/push/PushAttachmentData.java +++ /dev/null @@ -1,74 +0,0 @@ -/** - * Copyright (C) 2014-2016 Open Whisper Systems - * - * Licensed according to the LICENSE file in this repository. - */ - -package org.whispersystems.signalservice.internal.push; - -import org.whispersystems.signalservice.api.messages.SignalServiceAttachment.ProgressListener; -import org.whispersystems.signalservice.internal.push.http.CancelationSignal; -import org.whispersystems.signalservice.internal.push.http.OutputStreamFactory; -import org.whispersystems.signalservice.internal.push.http.ResumableUploadSpec; - -import java.io.InputStream; - -public class PushAttachmentData { - - private final String contentType; - private final InputStream data; - private final long dataSize; - private final boolean incremental; - private final OutputStreamFactory outputStreamFactory; - private final ProgressListener listener; - private final CancelationSignal cancelationSignal; - private final ResumableUploadSpec resumableUploadSpec; - - public PushAttachmentData(String contentType, InputStream data, long dataSize, - boolean incremental, OutputStreamFactory outputStreamFactory, - ProgressListener listener, CancelationSignal cancelationSignal, - ResumableUploadSpec resumableUploadSpec) - { - this.contentType = contentType; - this.data = data; - this.dataSize = dataSize; - this.incremental = incremental; - this.outputStreamFactory = outputStreamFactory; - this.resumableUploadSpec = resumableUploadSpec; - this.listener = listener; - this.cancelationSignal = cancelationSignal; - } - - public String getContentType() { - return contentType; - } - - public InputStream getData() { - return data; - } - - public long getDataSize() { - return dataSize; - } - - public boolean getIncremental() { - return incremental; - } - - public OutputStreamFactory getOutputStreamFactory() { - return outputStreamFactory; - } - - public ProgressListener getListener() { - return listener; - } - - public CancelationSignal getCancelationSignal() { - return cancelationSignal; - } - - public ResumableUploadSpec getResumableUploadSpec() { - return resumableUploadSpec; - } - -} diff --git a/libsignal-service/src/main/java/org/whispersystems/signalservice/internal/push/PushAttachmentData.kt b/libsignal-service/src/main/java/org/whispersystems/signalservice/internal/push/PushAttachmentData.kt new file mode 100644 index 0000000000..f914f31e37 --- /dev/null +++ b/libsignal-service/src/main/java/org/whispersystems/signalservice/internal/push/PushAttachmentData.kt @@ -0,0 +1,26 @@ +/** + * Copyright (C) 2014-2016 Open Whisper Systems + * + * Licensed according to the LICENSE file in this repository. + */ +package org.whispersystems.signalservice.internal.push + +import org.whispersystems.signalservice.api.messages.SignalServiceAttachment +import org.whispersystems.signalservice.internal.push.http.CancelationSignal +import org.whispersystems.signalservice.internal.push.http.OutputStreamFactory +import org.whispersystems.signalservice.internal.push.http.ResumableUploadSpec +import java.io.InputStream + +/** + * A bundle of data needed to start an attachment upload. + */ +data class PushAttachmentData( + val contentType: String?, + val data: InputStream, + val dataSize: Long, + val incremental: Boolean, + val outputStreamFactory: OutputStreamFactory, + val listener: SignalServiceAttachment.ProgressListener?, + val cancelationSignal: CancelationSignal?, + val resumableUploadSpec: ResumableUploadSpec +) diff --git a/libsignal-service/src/main/java/org/whispersystems/signalservice/internal/push/PushServiceSocket.java b/libsignal-service/src/main/java/org/whispersystems/signalservice/internal/push/PushServiceSocket.java index 22e4d4669c..20d216e4a3 100644 --- a/libsignal-service/src/main/java/org/whispersystems/signalservice/internal/push/PushServiceSocket.java +++ b/libsignal-service/src/main/java/org/whispersystems/signalservice/internal/push/PushServiceSocket.java @@ -343,7 +343,7 @@ public class PushServiceSocket { private static final ResponseCodeHandler NO_HANDLER = new EmptyResponseCodeHandler(); private static final ResponseCodeHandler UNOPINIONATED_HANDLER = new UnopinionatedResponseCodeHandler(); - private static final long CDN2_RESUMABLE_LINK_LIFETIME_MILLIS = TimeUnit.DAYS.toMillis(7); + public static final long CDN2_RESUMABLE_LINK_LIFETIME_MILLIS = TimeUnit.DAYS.toMillis(7); private static final int MAX_FOLLOW_UPS = 20; diff --git a/libsignal-service/src/main/java/org/whispersystems/signalservice/internal/push/http/ResumableUploadSpec.java b/libsignal-service/src/main/java/org/whispersystems/signalservice/internal/push/http/ResumableUploadSpec.java deleted file mode 100644 index d3fe2d2419..0000000000 --- a/libsignal-service/src/main/java/org/whispersystems/signalservice/internal/push/http/ResumableUploadSpec.java +++ /dev/null @@ -1,121 +0,0 @@ -package org.whispersystems.signalservice.internal.push.http; - -import org.signal.protos.resumableuploads.ResumableUpload; -import org.whispersystems.signalservice.api.push.exceptions.ResumeLocationInvalidException; -import org.signal.core.util.Base64; - -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; -import java.util.stream.Collectors; - -import okio.ByteString; - -public final class ResumableUploadSpec { - - private final byte[] secretKey; - private final byte[] iv; - - private final String cdnKey; - private final Integer cdnNumber; - private final String resumeLocation; - private final Long expirationTimestamp; - private final Map headers; - - public ResumableUploadSpec(byte[] secretKey, - byte[] iv, - String cdnKey, - int cdnNumber, - String resumeLocation, - long expirationTimestamp, - Map headers) - { - this.secretKey = secretKey; - this.iv = iv; - this.cdnKey = cdnKey; - this.cdnNumber = cdnNumber; - this.resumeLocation = resumeLocation; - this.expirationTimestamp = expirationTimestamp; - this.headers = headers; - } - - public byte[] getSecretKey() { - return secretKey; - } - - public byte[] getIV() { - return iv; - } - - public String getCdnKey() { - return cdnKey; - } - - public Integer getCdnNumber() { - return cdnNumber; - } - - public String getResumeLocation() { - return resumeLocation; - } - - public Long getExpirationTimestamp() { - return expirationTimestamp; - } - - public Map getHeaders() { - return headers; - } - - public ResumableUpload toProto() { - ResumableUpload.Builder builder = new ResumableUpload.Builder() - .secretKey(ByteString.of(getSecretKey())) - .iv(ByteString.of(getIV())) - .timeout(getExpirationTimestamp()) - .cdnNumber(getCdnNumber()) - .cdnKey(getCdnKey()) - .location(getResumeLocation()) - .timeout(getExpirationTimestamp()); - - builder.headers( - headers.entrySet() - .stream() - .map(e -> new ResumableUpload.Header.Builder().key(e.getKey()).value_(e.getValue()).build()) - .collect(Collectors.toList()) - ); - - return builder.build(); - } - - public String serialize() { - return Base64.encodeWithPadding(toProto().encode()); - } - - public static ResumableUploadSpec deserialize(String serializedSpec) throws ResumeLocationInvalidException { - try { - ResumableUpload resumableUpload = ResumableUpload.ADAPTER.decode(Base64.decode(serializedSpec)); - return from(resumableUpload); - } catch (IOException e) { - throw new ResumeLocationInvalidException(); - } - } - - public static ResumableUploadSpec from(ResumableUpload resumableUpload) throws ResumeLocationInvalidException { - if (resumableUpload == null) return null; - - Map headers = new HashMap<>(); - for (ResumableUpload.Header header : resumableUpload.headers) { - headers.put(header.key, header.value_); - } - - return new ResumableUploadSpec( - resumableUpload.secretKey.toByteArray(), - resumableUpload.iv.toByteArray(), - resumableUpload.cdnKey, - resumableUpload.cdnNumber, - resumableUpload.location, - resumableUpload.timeout, - headers - ); - } -} diff --git a/libsignal-service/src/main/java/org/whispersystems/signalservice/internal/push/http/ResumableUploadSpec.kt b/libsignal-service/src/main/java/org/whispersystems/signalservice/internal/push/http/ResumableUploadSpec.kt new file mode 100644 index 0000000000..0d1a087695 --- /dev/null +++ b/libsignal-service/src/main/java/org/whispersystems/signalservice/internal/push/http/ResumableUploadSpec.kt @@ -0,0 +1,71 @@ +package org.whispersystems.signalservice.internal.push.http + +import okio.ByteString.Companion.toByteString +import org.signal.core.util.Base64 +import org.signal.protos.resumableuploads.ResumableUpload +import org.whispersystems.signalservice.api.push.exceptions.ResumeLocationInvalidException +import java.io.IOException + +/** + * Contains data around how to begin or resume an upload. + * For given attachment, this data be saved and reused within the [expirationTimestamp] window. + */ +class ResumableUploadSpec( + val attachmentKey: ByteArray, + val attachmentIv: ByteArray, + val cdnKey: String, + val cdnNumber: Int, + val resumeLocation: String, + val expirationTimestamp: Long, + val headers: Map +) { + fun toProto(): ResumableUpload { + return ResumableUpload( + secretKey = attachmentKey.toByteString(), + iv = attachmentIv.toByteString(), + timeout = expirationTimestamp, + cdnNumber = cdnNumber, + cdnKey = cdnKey, + location = resumeLocation, + headers = headers.entries.map { ResumableUpload.Header(key = it.key, value_ = it.value) } + ) + } + + fun serialize(): String { + return Base64.encodeWithPadding(toProto().encode()) + } + + companion object { + @Throws(ResumeLocationInvalidException::class) + fun deserialize(serializedSpec: String?): ResumableUploadSpec? { + try { + val resumableUpload = ResumableUpload.ADAPTER.decode(Base64.decode(serializedSpec!!)) + return from(resumableUpload) + } catch (e: IOException) { + throw ResumeLocationInvalidException() + } + } + + @Throws(ResumeLocationInvalidException::class) + fun from(resumableUpload: ResumableUpload?): ResumableUploadSpec? { + if (resumableUpload == null) { + return null + } + + val headers: MutableMap = HashMap() + for (header in resumableUpload.headers) { + headers[header.key] = header.value_ + } + + return ResumableUploadSpec( + attachmentKey = resumableUpload.secretKey.toByteArray(), + attachmentIv = resumableUpload.iv.toByteArray(), + cdnKey = resumableUpload.cdnKey, + cdnNumber = resumableUpload.cdnNumber, + resumeLocation = resumableUpload.location, + expirationTimestamp = resumableUpload.timeout, + headers = headers + ) + } + } +}