diff --git a/app/src/androidTest/java/org/thoughtcrime/securesms/backup/v2/BackupTest.kt b/app/src/androidTest/java/org/thoughtcrime/securesms/backup/v2/BackupTest.kt index 9606456a5e..f8998ebeb2 100644 --- a/app/src/androidTest/java/org/thoughtcrime/securesms/backup/v2/BackupTest.kt +++ b/app/src/androidTest/java/org/thoughtcrime/securesms/backup/v2/BackupTest.kt @@ -19,6 +19,7 @@ import org.signal.core.util.requireBlob import org.signal.core.util.requireLong import org.signal.core.util.requireString import org.signal.core.util.select +import org.signal.core.util.toInt import org.signal.libsignal.zkgroup.profiles.ProfileKey import org.thoughtcrime.securesms.backup.v2.database.clearAllDataForBackupRestore import org.thoughtcrime.securesms.database.EmojiSearchTable @@ -40,6 +41,7 @@ import org.thoughtcrime.securesms.util.TextSecurePreferences import org.whispersystems.signalservice.api.push.ServiceId.ACI import org.whispersystems.signalservice.api.push.ServiceId.PNI import org.whispersystems.signalservice.api.subscriptions.SubscriberId +import java.io.ByteArrayInputStream import java.util.UUID import kotlin.random.Random @@ -176,7 +178,6 @@ class BackupTest { SignalStore.settings().setKeepMutedChatsArchived(true) SignalStore.storyValues().viewedReceiptsEnabled = false - SignalStore.storyValues().userHasReadOnboardingStory = true SignalStore.storyValues().userHasViewedOnboardingStory = true SignalStore.storyValues().isFeatureDisabled = false SignalStore.storyValues().userHasBeenNotifiedAboutStories = true @@ -227,7 +228,8 @@ class BackupTest { val startingMainData: DatabaseData = SignalDatabase.rawDatabase.readAllContents() val startingKeyValueData: DatabaseData = if (validateKeyValue) SignalDatabase.rawDatabase.readAllContents() else emptyMap() - BackupRepository.import(BackupRepository.export()) + val exported: ByteArray = BackupRepository.export() + BackupRepository.import(length = exported.size.toLong(), inputStreamFactory = { ByteArrayInputStream(exported) }, selfData = BackupRepository.SelfData(SELF_ACI, SELF_PNI, SELF_E164, SELF_PROFILE_KEY)) val endingData: DatabaseData = SignalDatabase.rawDatabase.readAllContents() val endingKeyValueData: DatabaseData = if (validateKeyValue) SignalDatabase.rawDatabase.readAllContents() else emptyMap() @@ -299,7 +301,7 @@ class BackupTest { fun standardMessage( outgoing: Boolean, sentTimestamp: Long = System.currentTimeMillis(), - receivedTimestamp: Long = sentTimestamp + 1, + receivedTimestamp: Long = if (outgoing) sentTimestamp else sentTimestamp + 1, serverTimestamp: Long = sentTimestamp, body: String? = null, read: Boolean = true, @@ -328,7 +330,7 @@ class BackupTest { fun remoteDeletedMessage( outgoing: Boolean, sentTimestamp: Long = System.currentTimeMillis(), - receivedTimestamp: Long = sentTimestamp + 1, + receivedTimestamp: Long = if (outgoing) sentTimestamp else sentTimestamp + 1, serverTimestamp: Long = sentTimestamp ): Long { return db.insertMessage( @@ -350,7 +352,7 @@ class BackupTest { outgoing: Boolean, threadId: Long, sentTimestamp: Long = System.currentTimeMillis(), - receivedTimestamp: Long = sentTimestamp + 1, + receivedTimestamp: Long = if (outgoing) sentTimestamp else sentTimestamp + 1, serverTimestamp: Long = sentTimestamp, body: String? = null, read: Boolean = true, @@ -390,12 +392,12 @@ class BackupTest { if (quotes != null) { val quoteDetails = this.getQuoteDetailsFor(quotes) - contentValues.put(MessageTable.QUOTE_ID, quoteDetails.quotedSentTimestamp) + contentValues.put(MessageTable.QUOTE_ID, if (quoteTargetMissing) MessageTable.QUOTE_TARGET_MISSING_ID else quoteDetails.quotedSentTimestamp) contentValues.put(MessageTable.QUOTE_AUTHOR, quoteDetails.authorId.serialize()) contentValues.put(MessageTable.QUOTE_BODY, quoteDetails.body) contentValues.put(MessageTable.QUOTE_BODY_RANGES, quoteDetails.bodyRanges) contentValues.put(MessageTable.QUOTE_TYPE, quoteDetails.type) - contentValues.put(MessageTable.QUOTE_MISSING, quoteTargetMissing) + contentValues.put(MessageTable.QUOTE_MISSING, quoteTargetMissing.toInt()) } if (body != null && (randomMention || randomStyling)) { @@ -493,7 +495,7 @@ class BackupTest { if (!contentEquals(expectedValue, actualValue)) { if (!describedRow) { - builder.append("-- ROW $i\n") + builder.append("-- ROW ${i + 1}\n") describedRow = true } builder.append("  [$key] Expected: ${expectedValue.prettyPrint()} || Actual: ${actualValue.prettyPrint()} \n") @@ -502,6 +504,8 @@ class BackupTest { if (describedRow) { builder.append("\n") + builder.append("Expected: $expectedRow\n") + builder.append("Actual: $actualRow\n") } } diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/BackupRepository.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/BackupRepository.kt index 8babf88e79..12694f2325 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/BackupRepository.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/BackupRepository.kt @@ -8,52 +8,68 @@ package org.thoughtcrime.securesms.backup.v2 import org.signal.core.util.EventTimer import org.signal.core.util.logging.Log import org.signal.core.util.withinTransaction +import org.signal.libsignal.zkgroup.profiles.ProfileKey import org.thoughtcrime.securesms.backup.v2.database.ChatItemImportInserter import org.thoughtcrime.securesms.backup.v2.database.clearAllDataForBackupRestore import org.thoughtcrime.securesms.backup.v2.processor.AccountDataProcessor import org.thoughtcrime.securesms.backup.v2.processor.ChatBackupProcessor import org.thoughtcrime.securesms.backup.v2.processor.ChatItemBackupProcessor import org.thoughtcrime.securesms.backup.v2.processor.RecipientBackupProcessor -import org.thoughtcrime.securesms.backup.v2.stream.BackupExportStream -import org.thoughtcrime.securesms.backup.v2.stream.PlainTextBackupExportStream -import org.thoughtcrime.securesms.backup.v2.stream.PlainTextBackupImportStream +import org.thoughtcrime.securesms.backup.v2.stream.BackupExportWriter +import org.thoughtcrime.securesms.backup.v2.stream.EncryptedBackupReader +import org.thoughtcrime.securesms.backup.v2.stream.EncryptedBackupWriter +import org.thoughtcrime.securesms.backup.v2.stream.PlainTextBackupReader +import org.thoughtcrime.securesms.backup.v2.stream.PlainTextBackupWriter import org.thoughtcrime.securesms.database.SignalDatabase import org.thoughtcrime.securesms.keyvalue.SignalStore import org.thoughtcrime.securesms.recipients.RecipientId -import java.io.ByteArrayInputStream +import org.whispersystems.signalservice.api.push.ServiceId.ACI +import org.whispersystems.signalservice.api.push.ServiceId.PNI import java.io.ByteArrayOutputStream +import java.io.InputStream object BackupRepository { private val TAG = Log.tag(BackupRepository::class.java) - fun export(): ByteArray { + fun export(plaintext: Boolean = false): ByteArray { val eventTimer = EventTimer() val outputStream = ByteArrayOutputStream() - val writer: BackupExportStream = PlainTextBackupExportStream(outputStream) + val writer: BackupExportWriter = if (plaintext) { + PlainTextBackupWriter(outputStream) + } else { + EncryptedBackupWriter( + key = SignalStore.svr().getOrCreateMasterKey().deriveBackupKey(), + aci = SignalStore.account().aci!!, + outputStream = outputStream, + append = { mac -> outputStream.write(mac) } + ) + } - // Note: Without a transaction, we may export inconsistent state. But because we have a transaction, - // writes from other threads are blocked. This is something to think more about. - SignalDatabase.rawDatabase.withinTransaction { - AccountDataProcessor.export { - writer.write(it) - eventTimer.emit("account") - } + writer.use { + // Note: Without a transaction, we may export inconsistent state. But because we have a transaction, + // writes from other threads are blocked. This is something to think more about. + SignalDatabase.rawDatabase.withinTransaction { + AccountDataProcessor.export { + writer.write(it) + eventTimer.emit("account") + } - RecipientBackupProcessor.export { - writer.write(it) - eventTimer.emit("recipient") - } + RecipientBackupProcessor.export { + writer.write(it) + eventTimer.emit("recipient") + } - ChatBackupProcessor.export { frame -> - writer.write(frame) - eventTimer.emit("thread") - } + ChatBackupProcessor.export { frame -> + writer.write(frame) + eventTimer.emit("thread") + } - ChatItemBackupProcessor.export { frame -> - writer.write(frame) - eventTimer.emit("message") + ChatItemBackupProcessor.export { frame -> + writer.write(frame) + eventTimer.emit("message") + } } } @@ -62,11 +78,19 @@ object BackupRepository { return outputStream.toByteArray() } - fun import(data: ByteArray) { + fun import(length: Long, inputStreamFactory: () -> InputStream, selfData: SelfData, plaintext: Boolean = false) { val eventTimer = EventTimer() - val stream = ByteArrayInputStream(data) - val frameReader = PlainTextBackupImportStream(stream) + val frameReader = if (plaintext) { + PlainTextBackupReader(inputStreamFactory()) + } else { + EncryptedBackupReader( + key = SignalStore.svr().getOrCreateMasterKey().deriveBackupKey(), + aci = selfData.aci, + streamLength = length, + dataStream = inputStreamFactory + ) + } // Note: Without a transaction, bad imports could lead to lost data. But because we have a transaction, // writes from other threads are blocked. This is something to think more about. @@ -76,6 +100,12 @@ object BackupRepository { SignalDatabase.distributionLists.clearAllDataForBackupRestore() SignalDatabase.threads.clearAllDataForBackupRestore() SignalDatabase.messages.clearAllDataForBackupRestore() + SignalDatabase.attachments.clearAllDataForBackupRestore() + + // Add back self after clearing data + val selfId: RecipientId = SignalDatabase.recipients.getAndPossiblyMerge(selfData.aci, selfData.pni, selfData.e164, pniVerified = true, changeSelf = true) + SignalDatabase.recipients.setProfileKey(selfId, selfData.profileKey) + SignalDatabase.recipients.setProfileSharing(selfId, true) val backupState = BackupState() val chatItemInserter: ChatItemImportInserter = ChatItemBackupProcessor.beginImport(backupState) @@ -83,7 +113,7 @@ object BackupRepository { for (frame in frameReader) { when { frame.account != null -> { - AccountDataProcessor.import(frame.account) + AccountDataProcessor.import(frame.account, selfId) eventTimer.emit("account") } @@ -118,6 +148,13 @@ object BackupRepository { Log.d(TAG, "import() ${eventTimer.stop().summary}") } + + data class SelfData( + val aci: ACI, + val pni: PNI, + val e164: String, + val profileKey: ProfileKey + ) } class BackupState { diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/AttachmentTableBackupExtensions.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/AttachmentTableBackupExtensions.kt new file mode 100644 index 0000000000..08934ef9c5 --- /dev/null +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/AttachmentTableBackupExtensions.kt @@ -0,0 +1,13 @@ +/* + * Copyright 2023 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ + +package org.thoughtcrime.securesms.backup.v2.database + +import org.signal.core.util.delete +import org.thoughtcrime.securesms.database.AttachmentTable + +fun AttachmentTable.clearAllDataForBackupRestore() { + writableDatabase.delete(AttachmentTable.TABLE_NAME).run() +} diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/ChatItemExportIterator.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/ChatItemExportIterator.kt index 4cdcfbe538..795c090a1a 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/ChatItemExportIterator.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/ChatItemExportIterator.kt @@ -6,6 +6,7 @@ package org.thoughtcrime.securesms.backup.v2.database import android.database.Cursor +import okio.ByteString.Companion.toByteString import org.signal.core.util.Base64 import org.signal.core.util.logging.Log import org.signal.core.util.requireBlob @@ -14,16 +15,16 @@ import org.signal.core.util.requireInt import org.signal.core.util.requireLong import org.signal.core.util.requireString import org.thoughtcrime.securesms.backup.v2.proto.ChatItem -import org.thoughtcrime.securesms.backup.v2.proto.ExpirationTimerChange -import org.thoughtcrime.securesms.backup.v2.proto.ProfileChange +import org.thoughtcrime.securesms.backup.v2.proto.ChatUpdateMessage +import org.thoughtcrime.securesms.backup.v2.proto.ExpirationTimerChatUpdate +import org.thoughtcrime.securesms.backup.v2.proto.ProfileChangeChatUpdate import org.thoughtcrime.securesms.backup.v2.proto.Quote import org.thoughtcrime.securesms.backup.v2.proto.Reaction import org.thoughtcrime.securesms.backup.v2.proto.RemoteDeletedMessage import org.thoughtcrime.securesms.backup.v2.proto.SendStatus -import org.thoughtcrime.securesms.backup.v2.proto.SimpleUpdate +import org.thoughtcrime.securesms.backup.v2.proto.SimpleChatUpdate import org.thoughtcrime.securesms.backup.v2.proto.StandardMessage import org.thoughtcrime.securesms.backup.v2.proto.Text -import org.thoughtcrime.securesms.backup.v2.proto.UpdateMessage import org.thoughtcrime.securesms.database.GroupReceiptTable import org.thoughtcrime.securesms.database.MessageTable import org.thoughtcrime.securesms.database.MessageTypes @@ -35,6 +36,8 @@ import org.thoughtcrime.securesms.database.model.databaseprotos.BodyRangeList import org.thoughtcrime.securesms.database.model.databaseprotos.ProfileChangeDetails import org.thoughtcrime.securesms.mms.QuoteModel import org.thoughtcrime.securesms.util.JsonUtils +import org.whispersystems.signalservice.api.util.UuidUtil +import org.whispersystems.signalservice.api.util.toByteArray import java.io.Closeable import java.io.IOException import java.util.LinkedList @@ -90,31 +93,31 @@ class ChatItemExportIterator(private val cursor: Cursor, private val batchSize: when { record.remoteDeleted -> builder.remoteDeletedMessage = RemoteDeletedMessage() - MessageTypes.isJoinedType(record.type) -> builder.updateMessage = UpdateMessage(simpleUpdate = SimpleUpdate(type = SimpleUpdate.Type.JOINED_SIGNAL)) - MessageTypes.isIdentityUpdate(record.type) -> builder.updateMessage = UpdateMessage(simpleUpdate = SimpleUpdate(type = SimpleUpdate.Type.IDENTITY_UPDATE)) - MessageTypes.isIdentityVerified(record.type) -> builder.updateMessage = UpdateMessage(simpleUpdate = SimpleUpdate(type = SimpleUpdate.Type.IDENTITY_VERIFIED)) - MessageTypes.isIdentityDefault(record.type) -> builder.updateMessage = UpdateMessage(simpleUpdate = SimpleUpdate(type = SimpleUpdate.Type.IDENTITY_DEFAULT)) - MessageTypes.isChangeNumber(record.type) -> builder.updateMessage = UpdateMessage(simpleUpdate = SimpleUpdate(type = SimpleUpdate.Type.CHANGE_NUMBER)) - MessageTypes.isBoostRequest(record.type) -> builder.updateMessage = UpdateMessage(simpleUpdate = SimpleUpdate(type = SimpleUpdate.Type.BOOST_REQUEST)) - MessageTypes.isEndSessionType(record.type) -> builder.updateMessage = UpdateMessage(simpleUpdate = SimpleUpdate(type = SimpleUpdate.Type.END_SESSION)) - MessageTypes.isChatSessionRefresh(record.type) -> builder.updateMessage = UpdateMessage(simpleUpdate = SimpleUpdate(type = SimpleUpdate.Type.CHAT_SESSION_REFRESH)) - MessageTypes.isBadDecryptType(record.type) -> builder.updateMessage = UpdateMessage(simpleUpdate = SimpleUpdate(type = SimpleUpdate.Type.BAD_DECRYPT)) - MessageTypes.isPaymentsActivated(record.type) -> builder.updateMessage = UpdateMessage(simpleUpdate = SimpleUpdate(type = SimpleUpdate.Type.PAYMENTS_ACTIVATED)) - MessageTypes.isPaymentsRequestToActivate(record.type) -> builder.updateMessage = UpdateMessage(simpleUpdate = SimpleUpdate(type = SimpleUpdate.Type.PAYMENT_ACTIVATION_REQUEST)) - MessageTypes.isExpirationTimerUpdate(record.type) -> builder.updateMessage = UpdateMessage(expirationTimerChange = ExpirationTimerChange((record.expiresIn / 1000).toInt())) + MessageTypes.isJoinedType(record.type) -> builder.updateMessage = ChatUpdateMessage(simpleUpdate = SimpleChatUpdate(type = SimpleChatUpdate.Type.JOINED_SIGNAL)) + MessageTypes.isIdentityUpdate(record.type) -> builder.updateMessage = ChatUpdateMessage(simpleUpdate = SimpleChatUpdate(type = SimpleChatUpdate.Type.IDENTITY_UPDATE)) + MessageTypes.isIdentityVerified(record.type) -> builder.updateMessage = ChatUpdateMessage(simpleUpdate = SimpleChatUpdate(type = SimpleChatUpdate.Type.IDENTITY_VERIFIED)) + MessageTypes.isIdentityDefault(record.type) -> builder.updateMessage = ChatUpdateMessage(simpleUpdate = SimpleChatUpdate(type = SimpleChatUpdate.Type.IDENTITY_DEFAULT)) + MessageTypes.isChangeNumber(record.type) -> builder.updateMessage = ChatUpdateMessage(simpleUpdate = SimpleChatUpdate(type = SimpleChatUpdate.Type.CHANGE_NUMBER)) + MessageTypes.isBoostRequest(record.type) -> builder.updateMessage = ChatUpdateMessage(simpleUpdate = SimpleChatUpdate(type = SimpleChatUpdate.Type.BOOST_REQUEST)) + MessageTypes.isEndSessionType(record.type) -> builder.updateMessage = ChatUpdateMessage(simpleUpdate = SimpleChatUpdate(type = SimpleChatUpdate.Type.END_SESSION)) + MessageTypes.isChatSessionRefresh(record.type) -> builder.updateMessage = ChatUpdateMessage(simpleUpdate = SimpleChatUpdate(type = SimpleChatUpdate.Type.CHAT_SESSION_REFRESH)) + MessageTypes.isBadDecryptType(record.type) -> builder.updateMessage = ChatUpdateMessage(simpleUpdate = SimpleChatUpdate(type = SimpleChatUpdate.Type.BAD_DECRYPT)) + MessageTypes.isPaymentsActivated(record.type) -> builder.updateMessage = ChatUpdateMessage(simpleUpdate = SimpleChatUpdate(type = SimpleChatUpdate.Type.PAYMENTS_ACTIVATED)) + MessageTypes.isPaymentsRequestToActivate(record.type) -> builder.updateMessage = ChatUpdateMessage(simpleUpdate = SimpleChatUpdate(type = SimpleChatUpdate.Type.PAYMENT_ACTIVATION_REQUEST)) + MessageTypes.isExpirationTimerUpdate(record.type) -> builder.updateMessage = ChatUpdateMessage(expirationTimerChange = ExpirationTimerChatUpdate((record.expiresIn / 1000).toInt())) MessageTypes.isProfileChange(record.type) -> { - builder.updateMessage = UpdateMessage( + builder.updateMessage = ChatUpdateMessage( profileChange = try { val decoded: ByteArray = Base64.decode(record.body!!) val profileChangeDetails = ProfileChangeDetails.ADAPTER.decode(decoded) if (profileChangeDetails.profileNameChange != null) { - ProfileChange(previousName = profileChangeDetails.profileNameChange.previous, newName = profileChangeDetails.profileNameChange.newValue) + ProfileChangeChatUpdate(previousName = profileChangeDetails.profileNameChange.previous, newName = profileChangeDetails.profileNameChange.newValue) } else { - ProfileChange() + ProfileChangeChatUpdate() } } catch (e: IOException) { Log.w(TAG, "Profile name change details could not be read", e) - ProfileChange() + ProfileChangeChatUpdate() } ) } @@ -142,9 +145,9 @@ class ChatItemExportIterator(private val cursor: Cursor, private val batchSize: chatId = record.threadId authorId = record.fromRecipientId dateSent = record.dateSent - dateReceived = record.dateReceived - expireStart = if (record.expireStarted > 0) record.expireStarted else null - expiresIn = if (record.expiresIn > 0) record.expiresIn else null + sealedSender = record.sealedSender + expireStartDate = if (record.expireStarted > 0) record.expireStarted else null + expiresInMs = if (record.expiresIn > 0) record.expiresIn else null revisions = emptyList() sms = !MessageTypes.isSecureType(record.type) @@ -155,7 +158,7 @@ class ChatItemExportIterator(private val cursor: Cursor, private val batchSize: } else { incoming = ChatItem.IncomingMessageDetails( dateServerSent = record.dateServer, - sealedSender = record.sealedSender, + dateReceived = record.dateReceived, read = record.read ) } @@ -169,21 +172,21 @@ class ChatItemExportIterator(private val cursor: Cursor, private val batchSize: body = this.body!!, bodyRanges = this.bodyRanges?.toBackupBodyRanges() ?: emptyList() ), - linkPreview = null, + // TODO Link previews! + linkPreview = emptyList(), longText = null, reactions = reactionRecords.toBackupReactions() ) } private fun BackupMessageRecord.toQuote(): Quote? { - return if (this.quoteTargetSentTimestamp > 0) { + return if (this.quoteTargetSentTimestamp != MessageTable.QUOTE_NOT_PRESENT_ID && this.quoteAuthor > 0) { // TODO Attachments! val type = QuoteModel.Type.fromCode(this.quoteType) Quote( - targetSentTimestamp = this.quoteTargetSentTimestamp, + targetSentTimestamp = this.quoteTargetSentTimestamp.takeIf { !this.quoteMissing && it != MessageTable.QUOTE_TARGET_MISSING_ID }, authorId = this.quoteAuthor, text = this.quoteBody, - originalMessageMissing = this.quoteMissing, bodyRanges = this.quoteBodyRanges?.toBackupBodyRanges() ?: emptyList(), type = when (type) { QuoteModel.Type.NORMAL -> Quote.Type.NORMAL @@ -207,7 +210,7 @@ class ChatItemExportIterator(private val cursor: Cursor, private val batchSize: BackupBodyRange( start = it.start, length = it.length, - mentionAci = it.mentionUuid, + mentionAci = it.mentionUuid?.let { UuidUtil.parseOrThrow(it) }?.toByteArray()?.toByteString(), style = it.style?.toBackupBodyRangeStyle() ) } @@ -245,9 +248,9 @@ class ChatItemExportIterator(private val cursor: Cursor, private val batchSize: } val status: SendStatus.Status = when { - this.viewedReceiptCount > 0 -> SendStatus.Status.VIEWED - this.readReceiptCount > 0 -> SendStatus.Status.READ - this.deliveryReceiptCount > 0 -> SendStatus.Status.DELIVERED + this.viewed -> SendStatus.Status.VIEWED + this.hasReadReceipt -> SendStatus.Status.READ + this.hasDeliveryReceipt -> SendStatus.Status.DELIVERED this.baseType == MessageTypes.BASE_SENT_TYPE -> SendStatus.Status.SENT MessageTypes.isFailedMessageType(this.type) -> SendStatus.Status.FAILED else -> SendStatus.Status.PENDING @@ -257,7 +260,7 @@ class ChatItemExportIterator(private val cursor: Cursor, private val batchSize: SendStatus( recipientId = this.toRecipientId, deliveryStatus = status, - timestamp = this.receiptTimestamp, + lastStatusUpdateTimestamp = this.receiptTimestamp, sealedSender = this.sealedSender, networkFailure = this.networkFailureRecipientIds.contains(this.toRecipientId), identityKeyMismatch = this.identityMismatchRecipientIds.contains(this.toRecipientId) @@ -271,7 +274,7 @@ class ChatItemExportIterator(private val cursor: Cursor, private val batchSize: recipientId = it.recipientId.toLong(), deliveryStatus = it.status.toBackupDeliveryStatus(), sealedSender = it.isUnidentified, - timestamp = it.timestamp, + lastStatusUpdateTimestamp = it.timestamp, networkFailure = networkFailureRecipientIds.contains(it.recipientId.toLong()), identityKeyMismatch = identityMismatchRecipientIds.contains(it.recipientId.toLong()) ) @@ -337,9 +340,9 @@ class ChatItemExportIterator(private val cursor: Cursor, private val batchSize: quoteType = this.requireInt(MessageTable.QUOTE_TYPE), originalMessageId = this.requireLong(MessageTable.ORIGINAL_MESSAGE_ID), latestRevisionId = this.requireLong(MessageTable.LATEST_REVISION_ID), - deliveryReceiptCount = this.requireInt(MessageTable.DELIVERY_RECEIPT_COUNT), - viewedReceiptCount = this.requireInt(MessageTable.VIEWED_RECEIPT_COUNT), - readReceiptCount = this.requireInt(MessageTable.READ_RECEIPT_COUNT), + hasDeliveryReceipt = this.requireBoolean(MessageTable.HAS_DELIVERY_RECEIPT), + viewed = this.requireBoolean(MessageTable.VIEWED_COLUMN), + hasReadReceipt = this.requireBoolean(MessageTable.HAS_READ_RECEIPT), read = this.requireBoolean(MessageTable.READ), receiptTimestamp = this.requireLong(MessageTable.RECEIPT_TIMESTAMP), networkFailureRecipientIds = this.requireString(MessageTable.NETWORK_FAILURES).parseNetworkFailures(), @@ -371,9 +374,9 @@ class ChatItemExportIterator(private val cursor: Cursor, private val batchSize: val quoteType: Int, val originalMessageId: Long, val latestRevisionId: Long, - val deliveryReceiptCount: Int, - val readReceiptCount: Int, - val viewedReceiptCount: Int, + val hasDeliveryReceipt: Boolean, + val hasReadReceipt: Boolean, + val viewed: Boolean, val receiptTimestamp: Long, val read: Boolean, val networkFailureRecipientIds: Set, diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/ChatItemImportInserter.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/ChatItemImportInserter.kt index 9fc5d759a8..152f18180f 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/ChatItemImportInserter.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/ChatItemImportInserter.kt @@ -14,12 +14,12 @@ import org.signal.core.util.toInt import org.thoughtcrime.securesms.backup.v2.BackupState import org.thoughtcrime.securesms.backup.v2.proto.BodyRange import org.thoughtcrime.securesms.backup.v2.proto.ChatItem +import org.thoughtcrime.securesms.backup.v2.proto.ChatUpdateMessage import org.thoughtcrime.securesms.backup.v2.proto.Quote import org.thoughtcrime.securesms.backup.v2.proto.Reaction import org.thoughtcrime.securesms.backup.v2.proto.SendStatus -import org.thoughtcrime.securesms.backup.v2.proto.SimpleUpdate +import org.thoughtcrime.securesms.backup.v2.proto.SimpleChatUpdate import org.thoughtcrime.securesms.backup.v2.proto.StandardMessage -import org.thoughtcrime.securesms.backup.v2.proto.UpdateMessage import org.thoughtcrime.securesms.database.GroupReceiptTable import org.thoughtcrime.securesms.database.MessageTable import org.thoughtcrime.securesms.database.MessageTypes @@ -35,6 +35,7 @@ import org.thoughtcrime.securesms.mms.QuoteModel import org.thoughtcrime.securesms.recipients.Recipient import org.thoughtcrime.securesms.recipients.RecipientId import org.thoughtcrime.securesms.util.JsonUtils +import org.whispersystems.signalservice.api.util.UuidUtil /** * An object that will ingest all fo the [ChatItem]s you want to write, buffer them until hitting a specified batch size, and then batch insert them @@ -58,9 +59,9 @@ class ChatItemImportInserter( MessageTable.BODY, MessageTable.FROM_RECIPIENT_ID, MessageTable.TO_RECIPIENT_ID, - MessageTable.DELIVERY_RECEIPT_COUNT, - MessageTable.READ_RECEIPT_COUNT, - MessageTable.VIEWED_RECEIPT_COUNT, + MessageTable.HAS_DELIVERY_RECEIPT, + MessageTable.HAS_READ_RECEIPT, + MessageTable.VIEWED_COLUMN, MessageTable.MISMATCHED_IDENTITIES, MessageTable.EXPIRES_IN, MessageTable.EXPIRE_STARTED, @@ -173,32 +174,32 @@ class ChatItemImportInserter( contentValues.put(MessageTable.FROM_RECIPIENT_ID, fromRecipientId.serialize()) contentValues.put(MessageTable.TO_RECIPIENT_ID, (if (this.outgoing != null) chatRecipientId else selfId).serialize()) contentValues.put(MessageTable.THREAD_ID, threadId) - contentValues.put(MessageTable.DATE_RECEIVED, this.dateReceived) - contentValues.put(MessageTable.RECEIPT_TIMESTAMP, this.outgoing?.sendStatus?.maxOf { it.timestamp } ?: 0) + contentValues.put(MessageTable.DATE_RECEIVED, this.incoming?.dateReceived ?: this.dateSent) + contentValues.put(MessageTable.RECEIPT_TIMESTAMP, this.outgoing?.sendStatus?.maxOf { it.lastStatusUpdateTimestamp } ?: 0) contentValues.putNull(MessageTable.LATEST_REVISION_ID) contentValues.putNull(MessageTable.ORIGINAL_MESSAGE_ID) contentValues.put(MessageTable.REVISION_NUMBER, 0) - contentValues.put(MessageTable.EXPIRES_IN, this.expiresIn ?: 0) - contentValues.put(MessageTable.EXPIRE_STARTED, this.expireStart ?: 0) + contentValues.put(MessageTable.EXPIRES_IN, this.expiresInMs ?: 0) + contentValues.put(MessageTable.EXPIRE_STARTED, this.expireStartDate ?: 0) if (this.outgoing != null) { - val viewReceiptCount = this.outgoing.sendStatus.count { it.deliveryStatus == SendStatus.Status.VIEWED } - val readReceiptCount = Integer.max(viewReceiptCount, this.outgoing.sendStatus.count { it.deliveryStatus == SendStatus.Status.READ }) - val deliveryReceiptCount = Integer.max(readReceiptCount, this.outgoing.sendStatus.count { it.deliveryStatus == SendStatus.Status.DELIVERED }) + val viewed = this.outgoing.sendStatus.any { it.deliveryStatus == SendStatus.Status.VIEWED } + val hasReadReceipt = viewed || this.outgoing.sendStatus.any { it.deliveryStatus == SendStatus.Status.READ } + val hasDeliveryReceipt = viewed || hasReadReceipt || this.outgoing.sendStatus.any { it.deliveryStatus == SendStatus.Status.DELIVERED } - contentValues.put(MessageTable.VIEWED_RECEIPT_COUNT, viewReceiptCount) - contentValues.put(MessageTable.READ_RECEIPT_COUNT, readReceiptCount) - contentValues.put(MessageTable.DELIVERY_RECEIPT_COUNT, deliveryReceiptCount) + contentValues.put(MessageTable.VIEWED_COLUMN, viewed.toInt()) + contentValues.put(MessageTable.HAS_READ_RECEIPT, hasReadReceipt.toInt()) + contentValues.put(MessageTable.HAS_DELIVERY_RECEIPT, hasDeliveryReceipt.toInt()) contentValues.put(MessageTable.UNIDENTIFIED, this.outgoing.sendStatus.count { it.sealedSender }) contentValues.put(MessageTable.READ, 1) contentValues.addNetworkFailures(this, backupState) contentValues.addIdentityKeyMismatches(this, backupState) } else { - contentValues.put(MessageTable.VIEWED_RECEIPT_COUNT, 0) - contentValues.put(MessageTable.READ_RECEIPT_COUNT, 0) - contentValues.put(MessageTable.DELIVERY_RECEIPT_COUNT, 0) - contentValues.put(MessageTable.UNIDENTIFIED, this.incoming?.sealedSender?.toInt() ?: 0) + contentValues.put(MessageTable.VIEWED_COLUMN, 0) + contentValues.put(MessageTable.HAS_READ_RECEIPT, 0) + contentValues.put(MessageTable.HAS_DELIVERY_RECEIPT, 0) + contentValues.put(MessageTable.UNIDENTIFIED, this.sealedSender?.toInt()) contentValues.put(MessageTable.READ, this.incoming?.read?.toInt() ?: 0) } @@ -264,7 +265,7 @@ class ChatItemImportInserter( GroupReceiptTable.MMS_ID to messageId, GroupReceiptTable.RECIPIENT_ID to recipientId.serialize(), GroupReceiptTable.STATUS to sendStatus.deliveryStatus.toLocalSendStatus(), - GroupReceiptTable.TIMESTAMP to sendStatus.timestamp, + GroupReceiptTable.TIMESTAMP to sendStatus.lastStatusUpdateTimestamp, GroupReceiptTable.UNIDENTIFIED to sendStatus.sealedSender ) } else { @@ -308,27 +309,28 @@ class ChatItemImportInserter( } } - private fun ContentValues.addUpdateMessage(updateMessage: UpdateMessage) { + private fun ContentValues.addUpdateMessage(updateMessage: ChatUpdateMessage) { var typeFlags: Long = 0 when { updateMessage.simpleUpdate != null -> { typeFlags = when (updateMessage.simpleUpdate.type) { - SimpleUpdate.Type.JOINED_SIGNAL -> MessageTypes.JOINED_TYPE - SimpleUpdate.Type.IDENTITY_UPDATE -> MessageTypes.KEY_EXCHANGE_IDENTITY_UPDATE_BIT - SimpleUpdate.Type.IDENTITY_VERIFIED -> MessageTypes.KEY_EXCHANGE_IDENTITY_VERIFIED_BIT - SimpleUpdate.Type.IDENTITY_DEFAULT -> MessageTypes.KEY_EXCHANGE_IDENTITY_DEFAULT_BIT - SimpleUpdate.Type.CHANGE_NUMBER -> MessageTypes.CHANGE_NUMBER_TYPE - SimpleUpdate.Type.BOOST_REQUEST -> MessageTypes.BOOST_REQUEST_TYPE - SimpleUpdate.Type.END_SESSION -> MessageTypes.END_SESSION_BIT - SimpleUpdate.Type.CHAT_SESSION_REFRESH -> MessageTypes.ENCRYPTION_REMOTE_FAILED_BIT - SimpleUpdate.Type.BAD_DECRYPT -> MessageTypes.BAD_DECRYPT_TYPE - SimpleUpdate.Type.PAYMENTS_ACTIVATED -> MessageTypes.SPECIAL_TYPE_PAYMENTS_ACTIVATED - SimpleUpdate.Type.PAYMENT_ACTIVATION_REQUEST -> MessageTypes.SPECIAL_TYPE_PAYMENTS_ACTIVATE_REQUEST + SimpleChatUpdate.Type.UNKNOWN -> 0 + SimpleChatUpdate.Type.JOINED_SIGNAL -> MessageTypes.JOINED_TYPE + SimpleChatUpdate.Type.IDENTITY_UPDATE -> MessageTypes.KEY_EXCHANGE_IDENTITY_UPDATE_BIT + SimpleChatUpdate.Type.IDENTITY_VERIFIED -> MessageTypes.KEY_EXCHANGE_IDENTITY_VERIFIED_BIT + SimpleChatUpdate.Type.IDENTITY_DEFAULT -> MessageTypes.KEY_EXCHANGE_IDENTITY_DEFAULT_BIT + SimpleChatUpdate.Type.CHANGE_NUMBER -> MessageTypes.CHANGE_NUMBER_TYPE + SimpleChatUpdate.Type.BOOST_REQUEST -> MessageTypes.BOOST_REQUEST_TYPE + SimpleChatUpdate.Type.END_SESSION -> MessageTypes.END_SESSION_BIT + SimpleChatUpdate.Type.CHAT_SESSION_REFRESH -> MessageTypes.ENCRYPTION_REMOTE_FAILED_BIT + SimpleChatUpdate.Type.BAD_DECRYPT -> MessageTypes.BAD_DECRYPT_TYPE + SimpleChatUpdate.Type.PAYMENTS_ACTIVATED -> MessageTypes.SPECIAL_TYPE_PAYMENTS_ACTIVATED + SimpleChatUpdate.Type.PAYMENT_ACTIVATION_REQUEST -> MessageTypes.SPECIAL_TYPE_PAYMENTS_ACTIVATE_REQUEST } } updateMessage.expirationTimerChange != null -> { typeFlags = MessageTypes.EXPIRATION_TIMER_UPDATE_BIT - put(MessageTable.EXPIRES_IN, updateMessage.expirationTimerChange.expiresIn.toLong() * 1000) + put(MessageTable.EXPIRES_IN, updateMessage.expirationTimerChange.expiresInMs.toLong()) } updateMessage.profileChange != null -> { typeFlags = MessageTypes.PROFILE_CHANGE_TYPE @@ -341,13 +343,13 @@ class ChatItemImportInserter( } private fun ContentValues.addQuote(quote: Quote) { - this.put(MessageTable.QUOTE_ID, quote.targetSentTimestamp) + this.put(MessageTable.QUOTE_ID, quote.targetSentTimestamp ?: MessageTable.QUOTE_TARGET_MISSING_ID) this.put(MessageTable.QUOTE_AUTHOR, backupState.backupToLocalRecipientId[quote.authorId]!!.serialize()) this.put(MessageTable.QUOTE_BODY, quote.text) this.put(MessageTable.QUOTE_TYPE, quote.type.toLocalQuoteType()) this.put(MessageTable.QUOTE_BODY_RANGES, quote.bodyRanges.toLocalBodyRanges()?.encode()) // TODO quote attachments - this.put(MessageTable.QUOTE_MISSING, quote.originalMessageMissing.toInt()) + this.put(MessageTable.QUOTE_MISSING, (quote.targetSentTimestamp == null).toInt()) } private fun Quote.Type.toLocalQuoteType(): Int { @@ -398,7 +400,7 @@ class ChatItemImportInserter( return BodyRangeList( ranges = this.map { bodyRange -> BodyRangeList.BodyRange( - mentionUuid = bodyRange.mentionAci, + mentionUuid = bodyRange.mentionAci?.let { UuidUtil.fromByteString(it) }?.toString(), style = bodyRange.style?.let { when (bodyRange.style) { BodyRange.Style.BOLD -> BodyRangeList.BodyRange.Style.BOLD @@ -418,6 +420,7 @@ class ChatItemImportInserter( private fun SendStatus.Status.toLocalSendStatus(): Int { return when (this) { + SendStatus.Status.UNKNOWN -> GroupReceiptTable.STATUS_UNKNOWN SendStatus.Status.FAILED -> GroupReceiptTable.STATUS_UNKNOWN SendStatus.Status.PENDING -> GroupReceiptTable.STATUS_UNDELIVERED SendStatus.Status.SENT -> GroupReceiptTable.STATUS_UNDELIVERED diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/DistributionListTablesBackupExtensions.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/DistributionListTablesBackupExtensions.kt index f132bfd34d..d8e1001d22 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/DistributionListTablesBackupExtensions.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/DistributionListTablesBackupExtensions.kt @@ -58,7 +58,6 @@ fun DistributionListTables.getAllForBackup(): List { distributionId = record.distributionId.asUuid().toByteArray().toByteString(), allowReplies = record.allowsReplies, deletionTimestamp = record.deletedAtTimestamp, - isUnknown = record.isUnknown, privacyMode = record.privacyMode.toBackupPrivacyMode(), memberRecipientIds = record.members.map { it.toLong() } ) @@ -81,7 +80,6 @@ fun DistributionListTables.restoreFromBackup(dlist: BackupDistributionList, back allowsReplies = dlist.allowReplies, deletionTimestamp = dlist.deletionTimestamp, storageId = null, - isUnknown = dlist.isUnknown, privacyMode = dlist.privacyMode.toLocalPrivacyMode() )!! @@ -108,6 +106,7 @@ private fun DistributionListPrivacyMode.toBackupPrivacyMode(): BackupDistributio private fun BackupDistributionList.PrivacyMode.toLocalPrivacyMode(): DistributionListPrivacyMode { return when (this) { + BackupDistributionList.PrivacyMode.UNKNOWN -> DistributionListPrivacyMode.ALL BackupDistributionList.PrivacyMode.ONLY_WITH -> DistributionListPrivacyMode.ONLY_WITH BackupDistributionList.PrivacyMode.ALL -> DistributionListPrivacyMode.ALL BackupDistributionList.PrivacyMode.ALL_EXCEPT -> DistributionListPrivacyMode.ALL_EXCEPT diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/MessageTableBackupExtensions.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/MessageTableBackupExtensions.kt index c1993befb2..bd707276c3 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/MessageTableBackupExtensions.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/MessageTableBackupExtensions.kt @@ -40,9 +40,9 @@ fun MessageTable.getMessagesForBackup(): ChatItemExportIterator { MessageTable.QUOTE_TYPE, MessageTable.ORIGINAL_MESSAGE_ID, MessageTable.LATEST_REVISION_ID, - MessageTable.DELIVERY_RECEIPT_COUNT, - MessageTable.READ_RECEIPT_COUNT, - MessageTable.VIEWED_RECEIPT_COUNT, + MessageTable.HAS_DELIVERY_RECEIPT, + MessageTable.HAS_READ_RECEIPT, + MessageTable.VIEWED_COLUMN, MessageTable.RECEIPT_TIMESTAMP, MessageTable.READ, MessageTable.NETWORK_FAILURES, diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/RecipientTableBackupExtensions.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/RecipientTableBackupExtensions.kt index 700e744bda..e6ac7e8348 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/RecipientTableBackupExtensions.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/RecipientTableBackupExtensions.kt @@ -127,9 +127,7 @@ fun RecipientTable.restoreRecipientFromBackup(recipient: BackupRecipient, backup /** * Given [AccountData], this will insert the necessary data for the local user into the [RecipientTable]. */ -fun RecipientTable.restoreSelfFromBackup(accountData: AccountData) { - val self = Recipient.trustedPush(ACI.parseOrThrow(accountData.aci.toByteArray()), PNI.parseOrNull(accountData.pni.toByteArray()), accountData.e164.toString()) - +fun RecipientTable.restoreSelfFromBackup(accountData: AccountData, selfId: RecipientId) { val values = ContentValues().apply { put(RecipientTable.PROFILE_GIVEN_NAME, accountData.givenName.nullIfBlank()) put(RecipientTable.PROFILE_FAMILY_NAME, accountData.familyName.nullIfBlank()) @@ -152,7 +150,7 @@ fun RecipientTable.restoreSelfFromBackup(accountData: AccountData) { writableDatabase .update(RecipientTable.TABLE_NAME) .values(values) - .where("${RecipientTable.ID} = ?", self.id) + .where("${RecipientTable.ID} = ?", selfId) .run() } @@ -181,7 +179,7 @@ private fun RecipientTable.restoreContactFromBackup(contact: Contact): Recipient RecipientTable.HIDDEN to contact.hidden, RecipientTable.PROFILE_FAMILY_NAME to contact.profileFamilyName.nullIfBlank(), RecipientTable.PROFILE_GIVEN_NAME to contact.profileGivenName.nullIfBlank(), - RecipientTable.PROFILE_JOINED_NAME to contact.profileJoinedName.nullIfBlank(), + RecipientTable.PROFILE_JOINED_NAME to ProfileName.fromParts(contact.profileGivenName.nullIfBlank(), contact.profileFamilyName.nullIfBlank()).toString().nullIfBlank(), RecipientTable.PROFILE_KEY to if (profileKey == null) null else Base64.encodeWithPadding(profileKey), RecipientTable.PROFILE_SHARING to contact.profileSharing.toInt(), RecipientTable.REGISTERED to contact.registered.toLocalRegisteredState().id, @@ -205,12 +203,12 @@ private fun Contact.toLocalExtras(): RecipientExtras { * Provides a nice iterable interface over a [RecipientTable] cursor, converting rows to [BackupRecipient]s. * Important: Because this is backed by a cursor, you must close it. It's recommended to use `.use()` or try-with-resources. */ -class BackupContactIterator(private val cursor: Cursor, private val selfId: Long) : Iterator, Closeable { +class BackupContactIterator(private val cursor: Cursor, private val selfId: Long) : Iterator, Closeable { override fun hasNext(): Boolean { return cursor.count > 0 && !cursor.isLast } - override fun next(): BackupRecipient { + override fun next(): BackupRecipient? { if (!cursor.moveToNext()) { throw NoSuchElementException() } @@ -225,10 +223,15 @@ class BackupContactIterator(private val cursor: Cursor, private val selfId: Long val aci = ACI.parseOrNull(cursor.requireString(RecipientTable.ACI_COLUMN)) val pni = PNI.parseOrNull(cursor.requireString(RecipientTable.PNI_COLUMN)) + val e164 = cursor.requireString(RecipientTable.E164)?.e164ToLong() val registeredState = RecipientTable.RegisteredState.fromId(cursor.requireInt(RecipientTable.REGISTERED)) val profileKey = cursor.requireString(RecipientTable.PROFILE_KEY) val extras = RecipientTableCursorUtil.getExtras(cursor) + if (aci == null && pni == null && e164 == null) { + return null + } + return BackupRecipient( id = id, contact = Contact( @@ -244,7 +247,6 @@ class BackupContactIterator(private val cursor: Cursor, private val selfId: Long profileSharing = cursor.requireBoolean(RecipientTable.PROFILE_SHARING), profileGivenName = cursor.requireString(RecipientTable.PROFILE_GIVEN_NAME).nullIfBlank(), profileFamilyName = cursor.requireString(RecipientTable.PROFILE_FAMILY_NAME).nullIfBlank(), - profileJoinedName = cursor.requireString(RecipientTable.PROFILE_JOINED_NAME).nullIfBlank(), hideStory = extras?.hideStory() ?: false ) ) diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/ThreadTableBackupExtensions.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/ThreadTableBackupExtensions.kt index 76f628ed12..8dc6c432d2 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/ThreadTableBackupExtensions.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/ThreadTableBackupExtensions.kt @@ -7,13 +7,20 @@ package org.thoughtcrime.securesms.backup.v2.database import android.database.Cursor import org.signal.core.util.SqlUtil +import org.signal.core.util.insertInto +import org.signal.core.util.logging.Log import org.signal.core.util.requireBoolean +import org.signal.core.util.requireInt import org.signal.core.util.requireLong import org.signal.core.util.select +import org.signal.core.util.toInt import org.thoughtcrime.securesms.backup.v2.proto.Chat import org.thoughtcrime.securesms.database.ThreadTable +import org.thoughtcrime.securesms.recipients.RecipientId import java.io.Closeable +private val TAG = Log.tag(ThreadTable::class.java) + fun ThreadTable.getThreadsForBackup(): ChatIterator { val cursor = readableDatabase .select( @@ -35,6 +42,17 @@ fun ThreadTable.clearAllDataForBackupRestore() { clearCache() } +fun ThreadTable.restoreFromBackup(chat: Chat, recipientId: RecipientId): Long? { + return writableDatabase + .insertInto(ThreadTable.TABLE_NAME) + .values( + ThreadTable.RECIPIENT_ID to recipientId.serialize(), + ThreadTable.PINNED to chat.pinnedOrder, + ThreadTable.ARCHIVED to chat.archived.toInt() + ) + .run() +} + class ChatIterator(private val cursor: Cursor) : Iterator, Closeable { override fun hasNext(): Boolean { return cursor.count > 0 && !cursor.isLast @@ -49,8 +67,8 @@ class ChatIterator(private val cursor: Cursor) : Iterator, Closeable { id = cursor.requireLong(ThreadTable.ID), recipientId = cursor.requireLong(ThreadTable.RECIPIENT_ID), archived = cursor.requireBoolean(ThreadTable.ARCHIVED), - pinned = cursor.requireBoolean(ThreadTable.PINNED), - expirationTimer = cursor.requireLong(ThreadTable.EXPIRES_IN) + pinnedOrder = cursor.requireInt(ThreadTable.PINNED), + expirationTimerMs = cursor.requireLong(ThreadTable.EXPIRES_IN) ) } diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/AccountDataProcessor.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/AccountDataProcessor.kt index 15a5c6420d..782bf48b54 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/AccountDataProcessor.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/AccountDataProcessor.kt @@ -19,6 +19,7 @@ import org.thoughtcrime.securesms.jobs.RetrieveProfileAvatarJob import org.thoughtcrime.securesms.keyvalue.PhoneNumberPrivacyValues import org.thoughtcrime.securesms.keyvalue.SignalStore import org.thoughtcrime.securesms.recipients.Recipient +import org.thoughtcrime.securesms.recipients.RecipientId import org.thoughtcrime.securesms.subscription.Subscriber import org.thoughtcrime.securesms.util.ProfileUtil import org.thoughtcrime.securesms.util.TextSecurePreferences @@ -35,21 +36,11 @@ object AccountDataProcessor { val self = Recipient.self().fresh() val record = recipients.getRecordForSync(self.id) - val pniIdentityKey = SignalStore.account().pniIdentityKey - val aciIdentityKey = SignalStore.account().aciIdentityKey - val subscriber: Subscriber? = SignalStore.donationsValues().getSubscriber() emitter.emit( Frame( account = AccountData( - aci = SignalStore.account().aci!!.toByteString(), - pni = SignalStore.account().pni!!.toByteString(), - e164 = SignalStore.account().e164!!.toLong(), - pniIdentityPrivateKey = pniIdentityKey.privateKey.serialize().toByteString(), - pniIdentityPublicKey = pniIdentityKey.publicKey.serialize().toByteString(), - aciIdentityPrivateKey = aciIdentityKey.privateKey.serialize().toByteString(), - aciIdentityPublicKey = aciIdentityKey.publicKey.serialize().toByteString(), profileKey = self.profileKey?.toByteString() ?: EMPTY, givenName = self.profileName.givenName, familyName = self.profileName.familyName, @@ -60,14 +51,12 @@ object AccountDataProcessor { subscriberCurrencyCode = subscriber?.currencyCode ?: defaultAccountRecord.subscriberCurrencyCode, accountSettings = AccountData.AccountSettings( storyViewReceiptsEnabled = SignalStore.storyValues().viewedReceiptsEnabled, - hasReadOnboardingStory = SignalStore.storyValues().userHasViewedOnboardingStory || SignalStore.storyValues().userHasReadOnboardingStory, - noteToSelfArchived = record != null && record.syncExtras.isArchived, noteToSelfMarkedUnread = record != null && record.syncExtras.isForcedUnread, typingIndicators = TextSecurePreferences.isTypingIndicatorsEnabled(context), readReceipts = TextSecurePreferences.isReadReceiptsEnabled(context), sealedSenderIndicators = TextSecurePreferences.isShowUnidentifiedDeliveryIndicatorsEnabled(context), linkPreviews = SignalStore.settings().isLinkPreviewsEnabled, - unlistedPhoneNumber = SignalStore.phoneNumberPrivacy().phoneNumberListingMode.isUnlisted, + notDiscoverableByPhoneNumber = SignalStore.phoneNumberPrivacy().phoneNumberListingMode.isUnlisted, phoneNumberSharingMode = SignalStore.phoneNumberPrivacy().phoneNumberSharingMode.toBackupPhoneNumberSharingMode(), preferContactAvatars = SignalStore.settings().isPreferSystemContactPhotos, universalExpireTimer = SignalStore.settings().universalExpireTimer, @@ -84,16 +73,12 @@ object AccountDataProcessor { ) } - fun import(accountData: AccountData) { - SignalStore.account().restoreAciIdentityKeyFromBackup(accountData.aciIdentityPublicKey.toByteArray(), accountData.aciIdentityPrivateKey.toByteArray()) - SignalStore.account().restorePniIdentityKeyFromBackup(accountData.pniIdentityPublicKey.toByteArray(), accountData.pniIdentityPrivateKey.toByteArray()) - - recipients.restoreSelfFromBackup(accountData) + fun import(accountData: AccountData, selfId: RecipientId) { + recipients.restoreSelfFromBackup(accountData, selfId) SignalStore.account().setRegistered(true) val context = ApplicationDependencies.getApplication() - val settings = accountData.accountSettings if (settings != null) { @@ -101,7 +86,7 @@ object AccountDataProcessor { TextSecurePreferences.setTypingIndicatorsEnabled(context, settings.typingIndicators) TextSecurePreferences.setShowUnidentifiedDeliveryIndicatorsEnabled(context, settings.sealedSenderIndicators) SignalStore.settings().isLinkPreviewsEnabled = settings.linkPreviews - SignalStore.phoneNumberPrivacy().phoneNumberListingMode = if (settings.unlistedPhoneNumber) PhoneNumberPrivacyValues.PhoneNumberListingMode.UNLISTED else PhoneNumberPrivacyValues.PhoneNumberListingMode.LISTED + SignalStore.phoneNumberPrivacy().phoneNumberListingMode = if (settings.notDiscoverableByPhoneNumber) PhoneNumberPrivacyValues.PhoneNumberListingMode.UNLISTED else PhoneNumberPrivacyValues.PhoneNumberListingMode.LISTED SignalStore.phoneNumberPrivacy().phoneNumberSharingMode = settings.phoneNumberSharingMode.toLocalPhoneNumberMode() SignalStore.settings().isPreferSystemContactPhotos = settings.preferContactAvatars SignalStore.settings().universalExpireTimer = settings.universalExpireTimer @@ -111,7 +96,6 @@ object AccountDataProcessor { SignalStore.storyValues().userHasBeenNotifiedAboutStories = settings.hasSetMyStoriesPrivacy SignalStore.storyValues().userHasViewedOnboardingStory = settings.hasViewedOnboardingStory SignalStore.storyValues().isFeatureDisabled = settings.storiesDisabled - SignalStore.storyValues().userHasReadOnboardingStory = settings.hasReadOnboardingStory SignalStore.storyValues().userHasSeenGroupStoryEducationSheet = settings.hasSeenGroupStoryEducationSheet SignalStore.storyValues().viewedReceiptsEnabled = settings.storyViewReceiptsEnabled ?: settings.readReceipts diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/ChatBackupProcessor.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/ChatBackupProcessor.kt index 41d7834193..e10d0921e2 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/ChatBackupProcessor.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/ChatBackupProcessor.kt @@ -8,12 +8,12 @@ package org.thoughtcrime.securesms.backup.v2.processor import org.signal.core.util.logging.Log import org.thoughtcrime.securesms.backup.v2.BackupState import org.thoughtcrime.securesms.backup.v2.database.getThreadsForBackup +import org.thoughtcrime.securesms.backup.v2.database.restoreFromBackup import org.thoughtcrime.securesms.backup.v2.proto.Chat import org.thoughtcrime.securesms.backup.v2.proto.Frame import org.thoughtcrime.securesms.backup.v2.stream.BackupFrameEmitter import org.thoughtcrime.securesms.database.SignalDatabase import org.thoughtcrime.securesms.recipients.RecipientId -import java.util.Collections object ChatBackupProcessor { val TAG = Log.tag(ChatBackupProcessor::class.java) @@ -27,26 +27,18 @@ object ChatBackupProcessor { } fun import(chat: Chat, backupState: BackupState) { - // TODO Perf can be improved here by doing a single insert instead of insert + multiple updates - val recipientId: RecipientId? = backupState.backupToLocalRecipientId[chat.recipientId] + if (recipientId == null) { + Log.w(TAG, "Missing recipient for chat ${chat.id}") + return + } - if (recipientId != null) { - val threadId = SignalDatabase.threads.getOrCreateThreadIdFor(org.thoughtcrime.securesms.recipients.Recipient.resolved(recipientId)) - - if (chat.archived) { - SignalDatabase.threads.archiveConversation(threadId) - } - - if (chat.pinned) { - SignalDatabase.threads.pinConversations(Collections.singleton(threadId)) - } - + SignalDatabase.threads.restoreFromBackup(chat, recipientId)?.let { threadId -> backupState.chatIdToLocalRecipientId[chat.id] = recipientId backupState.chatIdToLocalThreadId[chat.id] = threadId backupState.chatIdToBackupRecipientId[chat.id] = chat.recipientId - } else { - Log.w(TAG, "Recipient doesnt exist with id $recipientId") } + + // TODO there's several fields in the chat that actually need to be restored on the recipient table } } diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/RecipientBackupProcessor.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/RecipientBackupProcessor.kt index c3f2d26496..0c8db3753e 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/RecipientBackupProcessor.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/RecipientBackupProcessor.kt @@ -27,7 +27,9 @@ object RecipientBackupProcessor { SignalDatabase.recipients.getContactsForBackup(selfId).use { reader -> for (backupRecipient in reader) { - emitter.emit(Frame(recipient = backupRecipient)) + if (backupRecipient != null) { + emitter.emit(Frame(recipient = backupRecipient)) + } } } diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/stream/BackupEncryptedOutputStream.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/stream/BackupEncryptedOutputStream.kt new file mode 100644 index 0000000000..5a7c535d13 --- /dev/null +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/stream/BackupEncryptedOutputStream.kt @@ -0,0 +1,75 @@ +/* + * Copyright 2023 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ + +package org.thoughtcrime.securesms.backup.v2.stream + +import org.signal.libsignal.protocol.kdf.HKDF +import java.io.FilterOutputStream +import java.io.OutputStream +import javax.crypto.Cipher +import javax.crypto.Mac +import javax.crypto.spec.IvParameterSpec +import javax.crypto.spec.SecretKeySpec + +class BackupEncryptedOutputStream(key: ByteArray, backupId: ByteArray, wrapped: OutputStream) : FilterOutputStream(wrapped) { + + val cipher: Cipher = Cipher.getInstance("AES/CBC/PKCS5Padding") + val mac: Mac = Mac.getInstance("HmacSHA256") + + var finalMac: ByteArray? = null + + init { + if (key.size != 32) { + throw IllegalArgumentException("Key must be 32 bytes!") + } + + if (backupId.size != 16) { + throw IllegalArgumentException("BackupId must be 32 bytes!") + } + + val extendedKey = HKDF.deriveSecrets(key, backupId, "20231003_Signal_Backups_EncryptMessageBackup".toByteArray(), 80) + val macKey = extendedKey.copyOfRange(0, 32) + val cipherKey = extendedKey.copyOfRange(32, 64) + val iv = extendedKey.copyOfRange(64, 80) + + cipher.init(Cipher.ENCRYPT_MODE, SecretKeySpec(cipherKey, "AES"), IvParameterSpec(iv)) + mac.init(SecretKeySpec(macKey, "HmacSHA256")) + } + + override fun write(b: Int) { + throw UnsupportedOperationException() + } + + override fun write(data: ByteArray) { + write(data, 0, data.size) + } + + override fun write(data: ByteArray, off: Int, len: Int) { + cipher.update(data, off, len)?.let { ciphertext -> + mac.update(ciphertext) + super.write(ciphertext) + } + } + + override fun flush() { + cipher.doFinal()?.let { ciphertext -> + mac.update(ciphertext) + super.write(ciphertext) + } + + finalMac = mac.doFinal() + + super.flush() + } + + override fun close() { + flush() + super.close() + } + + fun getMac(): ByteArray { + return finalMac ?: throw IllegalStateException("Mac not yet available! You must call flush() before asking for the mac.") + } +} diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/stream/BackupExportStream.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/stream/BackupExportWriter.kt similarity index 82% rename from app/src/main/java/org/thoughtcrime/securesms/backup/v2/stream/BackupExportStream.kt rename to app/src/main/java/org/thoughtcrime/securesms/backup/v2/stream/BackupExportWriter.kt index 227f2dced3..24d3e34eb3 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/stream/BackupExportStream.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/stream/BackupExportWriter.kt @@ -7,6 +7,6 @@ package org.thoughtcrime.securesms.backup.v2.stream import org.thoughtcrime.securesms.backup.v2.proto.Frame -interface BackupExportStream { +interface BackupExportWriter : AutoCloseable { fun write(frame: Frame) } diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/stream/EncryptedBackupReader.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/stream/EncryptedBackupReader.kt new file mode 100644 index 0000000000..c32da6ab00 --- /dev/null +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/stream/EncryptedBackupReader.kt @@ -0,0 +1,112 @@ +/* + * Copyright 2023 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ + +package org.thoughtcrime.securesms.backup.v2.stream + +import org.signal.core.util.readFully +import org.signal.core.util.readNBytesOrThrow +import org.signal.core.util.readVarInt32 +import org.signal.core.util.stream.MacInputStream +import org.signal.core.util.stream.TruncatingInputStream +import org.thoughtcrime.securesms.backup.v2.proto.Frame +import org.whispersystems.signalservice.api.backup.BackupKey +import org.whispersystems.signalservice.api.push.ServiceId.ACI +import java.io.EOFException +import java.io.IOException +import java.io.InputStream +import java.util.zip.GZIPInputStream +import javax.crypto.Cipher +import javax.crypto.CipherInputStream +import javax.crypto.Mac +import javax.crypto.spec.IvParameterSpec +import javax.crypto.spec.SecretKeySpec + +/** + * Provides the ability to read backup frames in a streaming fashion from a target [InputStream]. + * As it's being read, it will be both decrypted and uncompressed. Specifically, the data is decrypted, + * that decrypted data is gunzipped, then that data is read as frames. + */ +class EncryptedBackupReader( + key: BackupKey, + aci: ACI, + streamLength: Long, + dataStream: () -> InputStream +) : Iterator, AutoCloseable { + + var next: Frame? = null + val stream: InputStream + + init { + val keyMaterial = key.deriveSecrets(aci) + + val cipher = Cipher.getInstance("AES/CBC/PKCS5Padding").apply { + init(Cipher.DECRYPT_MODE, SecretKeySpec(keyMaterial.cipherKey, "AES"), IvParameterSpec(keyMaterial.iv)) + } + + validateMac(keyMaterial.macKey, streamLength, dataStream()) + + stream = GZIPInputStream( + CipherInputStream( + TruncatingInputStream( + wrapped = dataStream(), + maxBytes = streamLength - MAC_SIZE + ), + cipher + ) + ) + + next = read() + } + + override fun hasNext(): Boolean { + return next != null + } + + override fun next(): Frame { + next?.let { out -> + next = read() + return out + } ?: throw NoSuchElementException() + } + + private fun read(): Frame? { + try { + val length = stream.readVarInt32().also { if (it < 0) return null } + val frameBytes: ByteArray = stream.readNBytesOrThrow(length) + + return Frame.ADAPTER.decode(frameBytes) + } catch (e: EOFException) { + return null + } + } + + override fun close() { + stream.close() + } + + companion object { + const val MAC_SIZE = 32 + + fun validateMac(macKey: ByteArray, streamLength: Long, dataStream: InputStream) { + val mac = Mac.getInstance("HmacSHA256").apply { + init(SecretKeySpec(macKey, "HmacSHA256")) + } + + val macStream = MacInputStream( + wrapped = TruncatingInputStream(dataStream, maxBytes = streamLength - MAC_SIZE), + mac = mac + ) + + macStream.readFully(false) + + val calculatedMac = macStream.mac.doFinal() + val expectedMac = dataStream.readNBytesOrThrow(MAC_SIZE) + + if (!calculatedMac.contentEquals(expectedMac)) { + throw IOException("Invalid MAC!") + } + } + } +} diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/stream/EncryptedBackupWriter.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/stream/EncryptedBackupWriter.kt new file mode 100644 index 0000000000..530f383195 --- /dev/null +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/stream/EncryptedBackupWriter.kt @@ -0,0 +1,76 @@ +/* + * Copyright 2023 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ + +package org.thoughtcrime.securesms.backup.v2.stream + +import org.signal.core.util.stream.MacOutputStream +import org.signal.core.util.writeVarInt32 +import org.thoughtcrime.securesms.backup.v2.proto.Frame +import org.whispersystems.signalservice.api.backup.BackupKey +import org.whispersystems.signalservice.api.push.ServiceId.ACI +import java.io.IOException +import java.io.OutputStream +import java.util.zip.GZIPOutputStream +import javax.crypto.Cipher +import javax.crypto.CipherOutputStream +import javax.crypto.Mac +import javax.crypto.spec.IvParameterSpec +import javax.crypto.spec.SecretKeySpec + +/** + * Provides the ability to write backup frames in a streaming fashion to a target [OutputStream]. + * As it's being written, it will be both encrypted and compressed. Specifically, the backup frames + * are gzipped, that gzipped data is encrypted, and then an HMAC of the encrypted data is appended + * to the end of the [outputStream]. + */ +class EncryptedBackupWriter( + key: BackupKey, + aci: ACI, + private val outputStream: OutputStream, + private val append: (ByteArray) -> Unit +) : BackupExportWriter { + + private val mainStream: GZIPOutputStream + private val macStream: MacOutputStream + + init { + val keyMaterial = key.deriveSecrets(aci) + + val cipher = Cipher.getInstance("AES/CBC/PKCS5Padding").apply { + init(Cipher.ENCRYPT_MODE, SecretKeySpec(keyMaterial.cipherKey, "AES"), IvParameterSpec(keyMaterial.iv)) + } + + val mac = Mac.getInstance("HmacSHA256").apply { + init(SecretKeySpec(keyMaterial.macKey, "HmacSHA256")) + } + + macStream = MacOutputStream(outputStream, mac) + + mainStream = GZIPOutputStream( + CipherOutputStream( + macStream, + cipher + ) + ) + } + + @Throws(IOException::class) + override fun write(frame: Frame) { + val frameBytes: ByteArray = frame.encode() + + mainStream.writeVarInt32(frameBytes.size) + mainStream.write(frameBytes) + } + + @Throws(IOException::class) + override fun close() { + // We need to close the main stream in order for the gzip and all the cipher operations to fully finish before + // we can calculate the MAC. Unfortunately flush()/finish() is not sufficient. So we have to defer to the + // caller to append the bytes to the end of the data however they see fit (like appending to a file). + mainStream.close() + val mac = macStream.mac.doFinal() + append(mac) + } +} diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/stream/PlainTextBackupImportStream.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/stream/PlainTextBackupReader.kt similarity index 67% rename from app/src/main/java/org/thoughtcrime/securesms/backup/v2/stream/PlainTextBackupImportStream.kt rename to app/src/main/java/org/thoughtcrime/securesms/backup/v2/stream/PlainTextBackupReader.kt index f3f2701a23..91c9945d9d 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/stream/PlainTextBackupImportStream.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/stream/PlainTextBackupReader.kt @@ -5,8 +5,8 @@ package org.thoughtcrime.securesms.backup.v2.stream -import org.signal.core.util.Conversions import org.signal.core.util.readNBytesOrThrow +import org.signal.core.util.readVarInt32 import org.thoughtcrime.securesms.backup.v2.proto.Frame import java.io.EOFException import java.io.InputStream @@ -14,7 +14,7 @@ import java.io.InputStream /** * Reads a plaintext backup import stream one frame at a time. */ -class PlainTextBackupImportStream(val inputStream: InputStream) : BackupImportStream, Iterator { +class PlainTextBackupReader(val inputStream: InputStream) : Iterator { var next: Frame? = null @@ -33,15 +33,12 @@ class PlainTextBackupImportStream(val inputStream: InputStream) : BackupImportSt } ?: throw NoSuchElementException() } - override fun read(): Frame? { + private fun read(): Frame? { try { - val lengthBytes: ByteArray = inputStream.readNBytesOrThrow(4) - val length = Conversions.byteArrayToInt(lengthBytes) - + val length = inputStream.readVarInt32().also { if (it < 0) return null } val frameBytes: ByteArray = inputStream.readNBytesOrThrow(length) - val frame: Frame = Frame.ADAPTER.decode(frameBytes) - return frame + return Frame.ADAPTER.decode(frameBytes) } catch (e: EOFException) { return null } diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/stream/PlainTextBackupExportStream.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/stream/PlainTextBackupWriter.kt similarity index 66% rename from app/src/main/java/org/thoughtcrime/securesms/backup/v2/stream/PlainTextBackupExportStream.kt rename to app/src/main/java/org/thoughtcrime/securesms/backup/v2/stream/PlainTextBackupWriter.kt index 1916ec3eed..a4e414dcba 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/stream/PlainTextBackupExportStream.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/stream/PlainTextBackupWriter.kt @@ -5,7 +5,7 @@ package org.thoughtcrime.securesms.backup.v2.stream -import org.signal.core.util.Conversions +import org.signal.core.util.writeVarInt32 import org.thoughtcrime.securesms.backup.v2.proto.Frame import java.io.IOException import java.io.OutputStream @@ -13,14 +13,17 @@ import java.io.OutputStream /** * Writes backup frames to the wrapped stream in plain text. Only for testing! */ -class PlainTextBackupExportStream(private val outputStream: OutputStream) : BackupExportStream { +class PlainTextBackupWriter(private val outputStream: OutputStream) : BackupExportWriter { @Throws(IOException::class) override fun write(frame: Frame) { val frameBytes: ByteArray = frame.encode() - val lengthBytes: ByteArray = Conversions.intToByteArray(frameBytes.size) - outputStream.write(lengthBytes) + outputStream.writeVarInt32(frameBytes.size) outputStream.write(frameBytes) } + + override fun close() { + outputStream.close() + } } diff --git a/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/backup/InternalBackupPlaygroundFragment.kt b/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/backup/InternalBackupPlaygroundFragment.kt index b95b7f776b..3d35f92578 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/backup/InternalBackupPlaygroundFragment.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/backup/InternalBackupPlaygroundFragment.kt @@ -5,23 +5,70 @@ package org.thoughtcrime.securesms.components.settings.app.internal.backup +import android.app.Activity.RESULT_OK +import android.content.Intent +import android.content.res.Configuration +import android.os.Bundle +import android.widget.Toast +import androidx.activity.result.ActivityResultLauncher +import androidx.activity.result.contract.ActivityResultContracts import androidx.compose.foundation.layout.Arrangement import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.Spacer +import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.foundation.layout.height +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.layout.width +import androidx.compose.material3.MaterialTheme import androidx.compose.material3.Surface +import androidx.compose.material3.Switch import androidx.compose.material3.Text import androidx.compose.runtime.Composable import androidx.compose.runtime.getValue import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.text.style.TextAlign import androidx.compose.ui.tooling.preview.Preview +import androidx.compose.ui.unit.dp import androidx.fragment.app.viewModels import org.signal.core.ui.Buttons +import org.signal.core.ui.theme.SignalTheme +import org.signal.core.util.getLength import org.thoughtcrime.securesms.components.settings.app.internal.backup.InternalBackupPlaygroundViewModel.BackupState import org.thoughtcrime.securesms.components.settings.app.internal.backup.InternalBackupPlaygroundViewModel.ScreenState import org.thoughtcrime.securesms.compose.ComposeFragment class InternalBackupPlaygroundFragment : ComposeFragment() { - val viewModel: InternalBackupPlaygroundViewModel by viewModels() + private val viewModel: InternalBackupPlaygroundViewModel by viewModels() + private lateinit var exportFileLauncher: ActivityResultLauncher + private lateinit var importFileLauncher: ActivityResultLauncher + + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) + + exportFileLauncher = registerForActivityResult(ActivityResultContracts.StartActivityForResult()) { result -> + if (result.resultCode == RESULT_OK) { + result.data?.data?.let { uri -> + requireContext().contentResolver.openOutputStream(uri)?.use { outputStream -> + outputStream.write(viewModel.backupData!!) + Toast.makeText(requireContext(), "Saved successfully", Toast.LENGTH_SHORT).show() + } ?: Toast.makeText(requireContext(), "Failed to open output stream", Toast.LENGTH_SHORT).show() + } ?: Toast.makeText(requireContext(), "No URI selected", Toast.LENGTH_SHORT).show() + } + } + + importFileLauncher = registerForActivityResult(ActivityResultContracts.StartActivityForResult()) { result -> + if (result.resultCode == RESULT_OK) { + result.data?.data?.let { uri -> + requireContext().contentResolver.getLength(uri)?.let { length -> + viewModel.import(length) { requireContext().contentResolver.openInputStream(uri)!! } + } + } ?: Toast.makeText(requireContext(), "No URI selected", Toast.LENGTH_SHORT).show() + } + } + } @Composable override fun FragmentContent() { @@ -30,22 +77,65 @@ class InternalBackupPlaygroundFragment : ComposeFragment() { Screen( state = state, onExportClicked = { viewModel.export() }, - onImportClicked = { viewModel.import() } + onImportMemoryClicked = { viewModel.import() }, + onImportFileClicked = { + val intent = Intent().apply { + action = Intent.ACTION_GET_CONTENT + type = "application/octet-stream" + addCategory(Intent.CATEGORY_OPENABLE) + } + + importFileLauncher.launch(intent) + }, + onPlaintextClicked = { viewModel.onPlaintextToggled() }, + onSaveToDiskClicked = { + val intent = Intent().apply { + action = Intent.ACTION_CREATE_DOCUMENT + type = "application/octet-stream" + addCategory(Intent.CATEGORY_OPENABLE) + putExtra(Intent.EXTRA_TITLE, "backup-${if (state.plaintext) "plaintext" else "encrypted"}-${System.currentTimeMillis()}.bin") + } + + exportFileLauncher.launch(intent) + } ) } + + override fun onActivityResult(requestCode: Int, resultCode: Int, data: Intent?) { + super.onActivityResult(requestCode, resultCode, data) + } } @Composable fun Screen( state: ScreenState, onExportClicked: () -> Unit = {}, - onImportClicked: () -> Unit = {} + onImportMemoryClicked: () -> Unit = {}, + onImportFileClicked: () -> Unit = {}, + onPlaintextClicked: () -> Unit = {}, + onSaveToDiskClicked: () -> Unit = {} ) { Surface { Column( horizontalAlignment = Alignment.CenterHorizontally, - verticalArrangement = Arrangement.Center + verticalArrangement = Arrangement.Center, + modifier = Modifier + .fillMaxSize() + .padding(16.dp) ) { + Row( + verticalAlignment = Alignment.CenterVertically + ) { + StateLabel(text = "Plaintext?") + Spacer(modifier = Modifier.width(8.dp)) + Switch( + checked = state.plaintext, + onCheckedChange = { onPlaintextClicked() } + ) + } + + Spacer(modifier = Modifier.height(8.dp)) + Buttons.LargePrimary( onClick = onExportClicked, enabled = !state.backupState.inProgress @@ -53,17 +143,92 @@ fun Screen( Text("Export") } Buttons.LargeTonal( - onClick = onImportClicked, + onClick = onImportMemoryClicked, enabled = state.backupState == BackupState.EXPORT_DONE ) { - Text("Import") + Text("Import from memory") + } + Buttons.LargeTonal( + onClick = onImportFileClicked + ) { + Text("Import from file") + } + + Spacer(modifier = Modifier.height(16.dp)) + + when (state.backupState) { + BackupState.NONE -> { + StateLabel("") + } + BackupState.EXPORT_IN_PROGRESS -> { + StateLabel("Export in progress...") + } + BackupState.EXPORT_DONE -> { + StateLabel("Export complete. Sitting in memory. You can click 'Import' to import that data, or you can save it to a file.") + + Spacer(modifier = Modifier.height(8.dp)) + + Buttons.MediumTonal(onClick = onSaveToDiskClicked) { + Text("Save to file") + } + } + BackupState.IMPORT_IN_PROGRESS -> { + StateLabel("Import in progress...") + } } } } } -@Preview +@Composable +private fun StateLabel(text: String) { + Text( + text = text, + style = MaterialTheme.typography.labelSmall, + textAlign = TextAlign.Center + ) +} + +@Preview(name = "Light Theme", group = "screen", uiMode = Configuration.UI_MODE_NIGHT_NO) +@Preview(name = "Dark Theme", group = "screen", uiMode = Configuration.UI_MODE_NIGHT_YES) @Composable fun PreviewScreen() { - Screen(state = ScreenState(backupState = BackupState.NONE)) + SignalTheme { + Surface { + Screen(state = ScreenState(backupState = BackupState.NONE, plaintext = false)) + } + } +} + +@Preview(name = "Light Theme", group = "screen", uiMode = Configuration.UI_MODE_NIGHT_NO) +@Preview(name = "Dark Theme", group = "screen", uiMode = Configuration.UI_MODE_NIGHT_YES) +@Composable +fun PreviewScreenExportInProgress() { + SignalTheme { + Surface { + Screen(state = ScreenState(backupState = BackupState.EXPORT_IN_PROGRESS, plaintext = false)) + } + } +} + +@Preview(name = "Light Theme", group = "screen", uiMode = Configuration.UI_MODE_NIGHT_NO) +@Preview(name = "Dark Theme", group = "screen", uiMode = Configuration.UI_MODE_NIGHT_YES) +@Composable +fun PreviewScreenExportDone() { + SignalTheme { + Surface { + Screen(state = ScreenState(backupState = BackupState.EXPORT_DONE, plaintext = false)) + } + } +} + +@Preview(name = "Light Theme", group = "screen", uiMode = Configuration.UI_MODE_NIGHT_NO) +@Preview(name = "Dark Theme", group = "screen", uiMode = Configuration.UI_MODE_NIGHT_YES) +@Composable +fun PreviewScreenImportInProgress() { + SignalTheme { + Surface { + Screen(state = ScreenState(backupState = BackupState.IMPORT_IN_PROGRESS, plaintext = false)) + } + } } diff --git a/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/backup/InternalBackupPlaygroundViewModel.kt b/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/backup/InternalBackupPlaygroundViewModel.kt index 645cd14bf4..fe91923e06 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/backup/InternalBackupPlaygroundViewModel.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/backup/InternalBackupPlaygroundViewModel.kt @@ -14,7 +14,11 @@ import io.reactivex.rxjava3.core.Single import io.reactivex.rxjava3.disposables.CompositeDisposable import io.reactivex.rxjava3.kotlin.plusAssign import io.reactivex.rxjava3.schedulers.Schedulers +import org.signal.libsignal.zkgroup.profiles.ProfileKey import org.thoughtcrime.securesms.backup.v2.BackupRepository +import org.thoughtcrime.securesms.recipients.Recipient +import java.io.ByteArrayInputStream +import java.io.InputStream class InternalBackupPlaygroundViewModel : ViewModel() { @@ -22,13 +26,14 @@ class InternalBackupPlaygroundViewModel : ViewModel() { val disposables = CompositeDisposable() - private val _state: MutableState = mutableStateOf(ScreenState(backupState = BackupState.NONE)) + private val _state: MutableState = mutableStateOf(ScreenState(backupState = BackupState.NONE, plaintext = false)) val state: State = _state fun export() { _state.value = _state.value.copy(backupState = BackupState.EXPORT_IN_PROGRESS) + val plaintext = _state.value.plaintext - disposables += Single.fromCallable { BackupRepository.export() } + disposables += Single.fromCallable { BackupRepository.export(plaintext = plaintext) } .subscribeOn(Schedulers.io()) .observeOn(AndroidSchedulers.mainThread()) .subscribe { data -> @@ -40,8 +45,12 @@ class InternalBackupPlaygroundViewModel : ViewModel() { fun import() { backupData?.let { _state.value = _state.value.copy(backupState = BackupState.IMPORT_IN_PROGRESS) + val plaintext = _state.value.plaintext - disposables += Single.fromCallable { BackupRepository.import(it) } + val self = Recipient.self() + val selfData = BackupRepository.SelfData(self.aci.get(), self.pni.get(), self.e164.get(), ProfileKey(self.profileKey)) + + disposables += Single.fromCallable { BackupRepository.import(it.size.toLong(), { ByteArrayInputStream(it) }, selfData, plaintext = plaintext) } .subscribeOn(Schedulers.io()) .observeOn(AndroidSchedulers.mainThread()) .subscribe { nothing -> @@ -51,12 +60,33 @@ class InternalBackupPlaygroundViewModel : ViewModel() { } } + fun import(length: Long, inputStreamFactory: () -> InputStream) { + _state.value = _state.value.copy(backupState = BackupState.IMPORT_IN_PROGRESS) + val plaintext = _state.value.plaintext + + val self = Recipient.self() + val selfData = BackupRepository.SelfData(self.aci.get(), self.pni.get(), self.e164.get(), ProfileKey(self.profileKey)) + + disposables += Single.fromCallable { BackupRepository.import(length, inputStreamFactory, selfData, plaintext = plaintext) } + .subscribeOn(Schedulers.io()) + .observeOn(AndroidSchedulers.mainThread()) + .subscribe { nothing -> + backupData = null + _state.value = _state.value.copy(backupState = BackupState.NONE) + } + } + + fun onPlaintextToggled() { + _state.value = _state.value.copy(plaintext = !_state.value.plaintext) + } + override fun onCleared() { disposables.clear() } data class ScreenState( - val backupState: BackupState + val backupState: BackupState, + val plaintext: Boolean ) enum class BackupState(val inProgress: Boolean = false) { diff --git a/app/src/main/java/org/thoughtcrime/securesms/database/MessageTable.kt b/app/src/main/java/org/thoughtcrime/securesms/database/MessageTable.kt index 4a68e67214..d49ca61eaf 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/database/MessageTable.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/database/MessageTable.kt @@ -207,6 +207,9 @@ open class MessageTable(context: Context?, databaseHelper: SignalDatabase) : Dat const val ORIGINAL_MESSAGE_ID = "original_message_id" const val REVISION_NUMBER = "revision_number" + const val QUOTE_NOT_PRESENT_ID = 0L + const val QUOTE_TARGET_MISSING_ID = -1L + const val CREATE_TABLE = """ CREATE TABLE $TABLE_NAME ( $ID INTEGER PRIMARY KEY AUTOINCREMENT, @@ -2316,7 +2319,7 @@ open class MessageTable(context: Context?, databaseHelper: SignalDatabase) : Dat val quoteAttachments: List = associatedAttachments.filter { it.isQuote }.toList() val quoteMentions: List = parseQuoteMentions(cursor) val quoteBodyRanges: BodyRangeList? = parseQuoteBodyRanges(cursor) - val quote: QuoteModel? = if (quoteId > 0 && quoteAuthor > 0 && (!TextUtils.isEmpty(quoteText) || quoteAttachments.isNotEmpty())) { + val quote: QuoteModel? = if (quoteId != QUOTE_NOT_PRESENT_ID && quoteAuthor > 0 && (!TextUtils.isEmpty(quoteText) || quoteAttachments.isNotEmpty())) { QuoteModel(quoteId, RecipientId.from(quoteAuthor), quoteText ?: "", quoteMissing, quoteAttachments, quoteMentions, QuoteModel.Type.fromCode(quoteType), quoteBodyRanges) } else { null @@ -5119,7 +5122,7 @@ open class MessageTable(context: Context?, databaseHelper: SignalDatabase) : Dat val quoteAttachments: List = attachments.filter { it.isQuote } val quoteDeck = SlideDeck(quoteAttachments) - return if (quoteId > 0 && quoteAuthor > 0) { + return if (quoteId != QUOTE_NOT_PRESENT_ID && quoteAuthor > 0) { if (quoteText != null && (quoteMentions.isNotEmpty() || bodyRanges != null)) { val updated: UpdatedBodyAndMentions = MentionUtil.updateBodyAndMentionsWithDisplayNames(context, quoteText, quoteMentions) val styledText = SpannableString(updated.body) diff --git a/app/src/main/java/org/thoughtcrime/securesms/database/RecipientTable.kt b/app/src/main/java/org/thoughtcrime/securesms/database/RecipientTable.kt index 6e7629d871..7a168d01ba 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/database/RecipientTable.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/database/RecipientTable.kt @@ -688,7 +688,7 @@ open class RecipientTable(context: Context, databaseHelper: SignalDatabase) : Da val foundRecords = queries.flatMap { query -> readableDatabase.query(TABLE_NAME, null, query.where, query.whereArgs, null, null, null).readToList { cursor -> - getRecord(context, cursor) + RecipientTableCursorUtil.getRecord(context, cursor) } } diff --git a/app/src/main/java/org/thoughtcrime/securesms/util/FeatureFlags.java b/app/src/main/java/org/thoughtcrime/securesms/util/FeatureFlags.java index 3be6fc371f..13129279a5 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/util/FeatureFlags.java +++ b/app/src/main/java/org/thoughtcrime/securesms/util/FeatureFlags.java @@ -200,7 +200,6 @@ public final class FeatureFlags { @SuppressWarnings("MismatchedQueryAndUpdateOfCollection") @VisibleForTesting static final Map FORCED_VALUES = new HashMap() {{ - put(INTERNAL_USER, true); }}; /** diff --git a/app/src/main/protowire/Backup.proto b/app/src/main/protowire/Backup.proto index ef4ffdf1b7..5596d81855 100644 --- a/app/src/main/protowire/Backup.proto +++ b/app/src/main/protowire/Backup.proto @@ -6,7 +6,8 @@ option java_package = "org.thoughtcrime.securesms.backup.v2.proto"; message BackupInfo { uint64 version = 1; - uint64 backupTime = 2; + uint64 backupTimeMs = 2; + bytes iv = 3; } message Frame { @@ -45,46 +46,36 @@ message AccountData { } message AccountSettings { - bool noteToSelfArchived = 1; - bool readReceipts = 2; - bool sealedSenderIndicators = 3; - bool typingIndicators = 4; - bool proxiedLinkPreviews = 5; - bool noteToSelfMarkedUnread = 6; - bool linkPreviews = 7; - bool unlistedPhoneNumber = 8; - bool preferContactAvatars = 9; - uint32 universalExpireTimer = 10; - repeated string preferredReactionEmoji = 11; - bool displayBadgesOnProfile = 12; - bool keepMutedChatsArchived = 13; - bool hasSetMyStoriesPrivacy = 14; - bool hasViewedOnboardingStory = 15; - bool storiesDisabled = 16; - optional bool storyViewReceiptsEnabled = 17; - bool hasReadOnboardingStory = 18; - bool hasSeenGroupStoryEducationSheet = 19; - bool hasCompletedUsernameOnboarding = 20; - PhoneNumberSharingMode phoneNumberSharingMode = 21; + bool readReceipts = 1; + bool sealedSenderIndicators = 2; + bool typingIndicators = 3; + bool noteToSelfMarkedUnread = 4; + bool linkPreviews = 5; + bool notDiscoverableByPhoneNumber = 6; + bool preferContactAvatars = 7; + uint32 universalExpireTimer = 8; + repeated string preferredReactionEmoji = 9; + bool displayBadgesOnProfile = 10; + bool keepMutedChatsArchived = 11; + bool hasSetMyStoriesPrivacy = 12; + bool hasViewedOnboardingStory = 13; + bool storiesDisabled = 14; + optional bool storyViewReceiptsEnabled = 15; + bool hasSeenGroupStoryEducationSheet = 16; + bool hasCompletedUsernameOnboarding = 17; + PhoneNumberSharingMode phoneNumberSharingMode = 18; } - bytes aciIdentityPublicKey = 1; - bytes aciIdentityPrivateKey = 2; - bytes pniIdentityPublicKey = 3; - bytes pniIdentityPrivateKey = 4; - bytes profileKey = 5; - optional string username = 6; - UsernameLink usernameLink = 7; - string givenName = 8; - string familyName = 9; - string avatarUrlPath = 10; - bytes subscriberId = 11; - string subscriberCurrencyCode = 12; - bool subscriptionManuallyCancelled = 13; - AccountSettings accountSettings = 14; - bytes aci = 15; - bytes pni = 16; - uint64 e164 = 17; + bytes profileKey = 1; + optional string username = 2; + UsernameLink usernameLink = 3; + string givenName = 4; + string familyName = 5; + string avatarUrlPath = 6; + bytes subscriberId = 7; + string subscriberCurrencyCode = 8; + bool subscriptionManuallyCancelled = 9; + AccountSettings accountSettings = 10; } message Recipient { @@ -94,29 +85,30 @@ message Recipient { Group group = 3; DistributionList distributionList = 4; Self self = 5; + ReleaseNotes releaseNotes = 6; } } message Contact { + enum Registered { + UNKNOWN = 0; + REGISTERED = 1; + NOT_REGISTERED = 2; + } + optional bytes aci = 1; // should be 16 bytes optional bytes pni = 2; // should be 16 bytes optional string username = 3; optional uint64 e164 = 4; bool blocked = 5; bool hidden = 6; - enum Registered { - UNKNOWN = 0; - REGISTERED = 1; - NOT_REGISTERED = 2; - } Registered registered = 7; uint64 unregisteredTimestamp = 8; optional bytes profileKey = 9; bool profileSharing = 10; optional string profileGivenName = 11; optional string profileFamilyName = 12; - optional string profileJoinedName = 13; - bool hideStory = 14; + bool hideStory = 13; } message Group { @@ -134,30 +126,34 @@ message Group { message Self {} +message ReleaseNotes {} + message Chat { uint64 id = 1; // generated id for reference only within this file uint64 recipientId = 2; bool archived = 3; - bool pinned = 4; - uint64 expirationTimer = 5; - uint64 muteUntil = 6; + uint32 pinnedOrder = 4; // 0 = unpinned, otherwise chat is considered pinned and will be displayed in ascending order + uint64 expirationTimerMs = 5; + uint64 muteUntilMs = 6; bool markedUnread = 7; bool dontNotifyForMentionsIfMuted = 8; + FilePointer wallpaper = 9; } message DistributionList { + enum PrivacyMode { + UNKNOWN = 0; + ONLY_WITH = 1; + ALL_EXCEPT = 2; + ALL = 3; + } + string name = 1; bytes distributionId = 2; // distribution list ids are uuids bool allowReplies = 3; uint64 deletionTimestamp = 4; - bool isUnknown = 5; - enum PrivacyMode { - ONLY_WITH = 0; - ALL_EXCEPT = 1; - ALL = 2; - } - PrivacyMode privacyMode = 6; - repeated uint64 memberRecipientIds = 7; // generated recipient id + PrivacyMode privacyMode = 5; + repeated uint64 memberRecipientIds = 6; // generated recipient id } message Identity { @@ -170,38 +166,41 @@ message Identity { } message Call { - uint64 callId = 1; - uint64 peerRecipientId = 2; enum Type { - AUDIO_CALL = 0; - VIDEO_CALL = 1; - GROUP_CALL = 2; - AD_HOC_CALL = 3; + UNKNOWN_TYPE = 0; + AUDIO_CALL = 1; + VIDEO_CALL = 2; + GROUP_CALL = 3; + AD_HOC_CALL = 4; } + + enum Event { + UNKNOWN_EVENT = 0; + OUTGOING = 1; // 1:1 calls only + ACCEPTED = 2; // 1:1 and group calls. Group calls: You accepted a ring. + NOT_ACCEPTED = 3; // 1:1 calls only, + MISSED = 4; // 1:1 and group. Group calls: The remote ring has expired or was cancelled by the ringer. + DELETE = 5; // 1:1 and Group/Ad-Hoc Calls. + GENERIC_GROUP_CALL = 6; // Group/Ad-Hoc Calls only. Initial state + JOINED = 7; // Group Calls: User has joined the group call. + DECLINED = 8; // Group Calls: If you declined a ring. + OUTGOING_RING = 9; // Group Calls: If you are ringing a group. + } + + uint64 callId = 1; + uint64 conversationRecipientId = 2; Type type = 3; bool outgoing = 4; uint64 timestamp = 5; uint64 ringerRecipientId = 6; - enum Event { - OUTGOING = 0; // 1:1 calls only - ACCEPTED = 1; // 1:1 and group calls. Group calls: You accepted a ring. - NOT_ACCEPTED = 2; // 1:1 calls only, - MISSED = 3; // 1:1 and group/ad-hoc calls. Group calls: The remote ring has expired or was cancelled by the ringer. - DELETE = 4; // 1:1 and Group/Ad-Hoc Calls. - GENERIC_GROUP_CALL = 5; // Group/Ad-Hoc Calls only. Initial state - JOINED = 6; // Group Calls: User has joined the group call. - RINGING = 7; // Group Calls: If a ring was requested by another user. - DECLINED = 8; // Group Calls: If you declined a ring. - OUTGOING_RING = 9; // Group Calls: If you are ringing a group. - } Event event = 7; } message ChatItem { message IncomingMessageDetails { - uint64 dateServerSent = 1; - bool read = 2; - bool sealedSender = 3; + uint64 dateReceived = 1; + uint64 dateServerSent = 2; + bool read = 3; } message OutgoingMessageDetails { @@ -211,43 +210,45 @@ message ChatItem { uint64 chatId = 1; // conversation id uint64 authorId = 2; // recipient id uint64 dateSent = 3; - uint64 dateReceived = 4; - optional uint64 expireStart = 5; // timestamp of when expiration timer started ticking down - optional uint64 expiresIn = 6; // how long timer of message is (ms) - repeated ChatItem revisions = 7; + bool sealedSender = 4; + optional uint64 expireStartDate = 5; // timestamp of when expiration timer started ticking down + optional uint64 expiresInMs = 6; // how long timer of message is (ms) + repeated ChatItem revisions = 7; // ordered from oldest to newest bool sms = 8; oneof directionalDetails { - IncomingMessageDetails incoming = 9; - OutgoingMessageDetails outgoing = 10; + IncomingMessageDetails incoming = 10; + OutgoingMessageDetails outgoing = 12; } oneof item { - StandardMessage standardMessage = 11; - ContactMessage contactMessage = 12; - VoiceMessage voiceMessage = 13; - StickerMessage stickerMessage = 14; - RemoteDeletedMessage remoteDeletedMessage = 15; - UpdateMessage updateMessage = 16; + StandardMessage standardMessage = 13; + ContactMessage contactMessage = 14; + VoiceMessage voiceMessage = 15; + StickerMessage stickerMessage = 16; + RemoteDeletedMessage remoteDeletedMessage = 17; + ChatUpdateMessage updateMessage = 18; } } message SendStatus { enum Status { - FAILED = 0; - PENDING = 1; - SENT = 2; - DELIVERED = 3; - READ = 4; - VIEWED = 5; - SKIPPED = 6; // e.g. user in group was blocked, so we skipped sending to them + UNKNOWN = 0; + FAILED = 1; + PENDING = 2; + SENT = 3; + DELIVERED = 4; + READ = 5; + VIEWED = 6; + SKIPPED = 7; // e.g. user in group was blocked, so we skipped sending to them } + uint64 recipientId = 1; Status deliveryStatus = 2; bool networkFailure = 3; bool identityKeyMismatch = 4; bool sealedSender = 5; - uint64 timestamp = 6; + uint64 lastStatusUpdateTimestamp = 6; // the time the status was last updated -- if from a receipt, it should be the sentTime of the receipt } message Text { @@ -258,9 +259,9 @@ message Text { message StandardMessage { optional Quote quote = 1; optional Text text = 2; - repeated AttachmentPointer attachments = 3; - optional LinkPreview linkPreview = 4; - optional AttachmentPointer longText = 5; + repeated FilePointer attachments = 3; + repeated LinkPreview linkPreview = 4; + optional FilePointer longText = 5; repeated Reaction reactions = 6; } @@ -281,10 +282,11 @@ message ContactAttachment { message Phone { enum Type { - HOME = 0; - MOBILE = 1; - WORK = 2; - CUSTOM = 3; + UNKNOWN = 0; + HOME = 1; + MOBILE = 2; + WORK = 3; + CUSTOM = 4; } optional string value = 1; @@ -294,10 +296,11 @@ message ContactAttachment { message Email { enum Type { - HOME = 0; - MOBILE = 1; - WORK = 2; - CUSTOM = 3; + UNKNOWN = 0; + HOME = 1; + MOBILE = 2; + WORK = 3; + CUSTOM = 4; } optional string value = 1; @@ -307,9 +310,10 @@ message ContactAttachment { message PostalAddress { enum Type { - HOME = 0; - WORK = 1; - CUSTOM = 2; + UNKNOWN = 0; + HOME = 1; + WORK = 2; + CUSTOM = 3; } optional Type type = 1; @@ -324,8 +328,7 @@ message ContactAttachment { } message Avatar { - optional AttachmentPointer avatar = 1; - optional bool isProfile = 2; + FilePointer avatar = 1; } optional Name name = 1; @@ -338,13 +341,13 @@ message ContactAttachment { message DocumentMessage { Text text = 1; - AttachmentPointer document = 2; + FilePointer document = 2; repeated Reaction reactions = 3; } message VoiceMessage { optional Quote quote = 1; - AttachmentPointer audio = 2; + FilePointer audio = 2; repeated Reaction reactions = 3; } @@ -356,11 +359,6 @@ message StickerMessage { // Tombstone for remote delete message RemoteDeletedMessage {} -message ScheduledMessage { - ChatItem message = 1; - uint64 scheduledTime = 2; -} - message Sticker { bytes packId = 1; bytes packKey = 2; @@ -371,37 +369,62 @@ message Sticker { message LinkPreview { string url = 1; optional string title = 2; - optional AttachmentPointer image = 3; + optional FilePointer image = 3; optional string description = 4; optional uint64 date = 5; } -message AttachmentPointer { +message FilePointer { + message BackupLocator { + string mediaName = 1; + uint32 cdnNumber = 2; + } + + message AttachmentLocator { + string cdnKey = 1; + uint32 cdnNumber = 2; + uint64 uploadTimestamp = 3; + } + + message LegacyAttachmentLocator { + fixed64 cdnId = 1; + } + + // An attachment that was backed up without being downloaded. + // Its MediaName should be generated as “{sender_aci}_{cdn_attachment_key}”, + // but should eventually transition to a BackupLocator with mediaName + // being the content hash once it is downloaded. + message UndownloadedBackupLocator { + bytes senderAci = 1; + string cdnKey = 2; + uint32 cdnNumber = 3; + } + enum Flags { VOICE_MESSAGE = 0; BORDERLESS = 1; GIF = 2; } - oneof attachmentIdentifier { - fixed64 cdnId = 1; - string cdnKey = 2; + oneof locator { + BackupLocator backupLocator = 1; + AttachmentLocator attachmentLocator= 2; + LegacyAttachmentLocator legacyAttachmentLocator = 3; + UndownloadedBackupLocator undownloadedBackupLocator = 4; } - optional string contentType = 3; - optional bytes key = 4; - optional uint32 size = 5; - optional bytes digest = 6; - optional bytes incrementalMac = 7; - optional bytes incrementalMacChunkSize = 8; - optional string fileName = 9; - optional uint32 flags = 10; - optional uint32 width = 11; - optional uint32 height = 12; - optional string caption = 13; - optional string blurHash = 14; - optional uint64 uploadTimestamp = 15; - optional uint32 cdnNumber = 16; + optional bytes key = 5; + optional string contentType = 6; + optional uint32 size = 7; + optional bytes digest = 8; + optional bytes incrementalMac = 9; + optional bytes incrementalMacChunkSize = 10; + optional string fileName = 11; + optional uint32 flags = 12; + optional uint32 width = 13; + optional uint32 height = 14; + optional string caption = 15; + optional string blurHash = 16; } message Quote { @@ -414,16 +437,15 @@ message Quote { message QuotedAttachment { optional string contentType = 1; optional string fileName = 2; - optional AttachmentPointer thumbnail = 3; + optional FilePointer thumbnail = 3; } - uint64 targetSentTimestamp = 1; + optional uint64 targetSentTimestamp = 1; // null if the target message could not be found at time of quote insert uint64 authorId = 2; optional string text = 3; repeated QuotedAttachment attachments = 4; repeated BodyRange bodyRanges = 5; Type type = 6; - bool originalMessageMissing = 7; } message BodyRange { @@ -440,7 +462,7 @@ message BodyRange { optional uint32 length = 2; oneof associatedValue { - string mentionAci = 3; + bytes mentionAci = 3; Style style = 4; } } @@ -449,82 +471,85 @@ message Reaction { string emoji = 1; uint64 authorId = 2; uint64 sentTimestamp = 3; - uint64 receivedTimestamp = 4; + optional uint64 receivedTimestamp = 4; + uint64 sortOrder = 5; // A higher sort order means that a reaction is more recent } -message UpdateMessage { +message ChatUpdateMessage { oneof update { - SimpleUpdate simpleUpdate = 1; - GroupDescriptionUpdate groupDescription = 2; - ExpirationTimerChange expirationTimerChange = 3; - ProfileChange profileChange = 4; - ThreadMergeEvent threadMerge = 5; - SessionSwitchoverEvent sessionSwitchover = 6; - CallingMessage callingMessage = 7; + SimpleChatUpdate simpleUpdate = 1; + GroupDescriptionChatUpdate groupDescription = 2; + ExpirationTimerChatUpdate expirationTimerChange = 3; + ProfileChangeChatUpdate profileChange = 4; + ThreadMergeChatUpdate threadMerge = 5; + SessionSwitchoverChatUpdate sessionSwitchover = 6; + CallChatUpdate callingMessage = 7; } } -message CallingMessage { +message CallChatUpdate{ oneof call { uint64 callId = 1; // maps to id of Call from call log - CallMessage callMessage = 2; - GroupCallMessage groupCall = 3; + IndividualCallChatUpdate callMessage = 2; + GroupCallChatUpdate groupCall = 3; } } -message CallMessage { +message IndividualCallChatUpdate { enum Type { - INCOMING_AUDIO_CALL = 0; - INCOMING_VIDEO_CALL = 1; - OUTGOING_AUDIO_CALL = 2; - OUTGOING_VIDEO_CALL = 3; - MISSED_AUDIO_CALL = 4; - MISSED_VIDEO_CALL = 5; + UNKNOWN = 0; + INCOMING_AUDIO_CALL = 1; + INCOMING_VIDEO_CALL = 2; + OUTGOING_AUDIO_CALL = 3; + OUTGOING_VIDEO_CALL = 4; + MISSED_AUDIO_CALL = 5; + MISSED_VIDEO_CALL = 6; } } -message GroupCallMessage { - bytes startedCallUuid = 1; +message GroupCallChatUpdate { + bytes startedCallAci = 1; uint64 startedCallTimestamp = 2; - repeated bytes inCallUuids = 3; - bool isCallFull = 4; + repeated bytes inCallAcis = 3; } -message SimpleUpdate { +message SimpleChatUpdate { enum Type { - JOINED_SIGNAL = 0; - IDENTITY_UPDATE = 1; - IDENTITY_VERIFIED = 2; - IDENTITY_DEFAULT = 3; // marking as unverified - CHANGE_NUMBER = 4; - BOOST_REQUEST = 5; - END_SESSION = 6; - CHAT_SESSION_REFRESH = 7; - BAD_DECRYPT = 8; - PAYMENTS_ACTIVATED = 9; - PAYMENT_ACTIVATION_REQUEST = 10; + UNKNOWN = 0; + JOINED_SIGNAL = 1; + IDENTITY_UPDATE = 2; + IDENTITY_VERIFIED = 3; + IDENTITY_DEFAULT = 4; // marking as unverified + CHANGE_NUMBER = 5; + BOOST_REQUEST = 6; + END_SESSION = 7; + CHAT_SESSION_REFRESH = 8; + BAD_DECRYPT = 9; + PAYMENTS_ACTIVATED = 10; + PAYMENT_ACTIVATION_REQUEST = 11; } + Type type = 1; } -message GroupDescriptionUpdate { - string body = 1; +message GroupDescriptionChatUpdate { + string newDescription = 1; } -message ExpirationTimerChange { - uint32 expiresIn = 1; +message ExpirationTimerChatUpdate { + uint32 expiresInMs = 1; } -message ProfileChange { +message ProfileChangeChatUpdate { string previousName = 1; string newName = 2; } -message ThreadMergeEvent { +message ThreadMergeChatUpdate { uint64 previousE164 = 1; } -message SessionSwitchoverEvent { +message SessionSwitchoverChatUpdate { uint64 e164 = 1; } @@ -537,6 +562,6 @@ message StickerPack { } message StickerPackSticker { - AttachmentPointer data = 1; + FilePointer data = 1; string emoji = 2; -} +} \ No newline at end of file diff --git a/app/src/test/java/org/thoughtcrime/securesms/backup/v2/stream/EncryptedBackupReaderWriterTest.kt b/app/src/test/java/org/thoughtcrime/securesms/backup/v2/stream/EncryptedBackupReaderWriterTest.kt new file mode 100644 index 0000000000..48d14a4683 --- /dev/null +++ b/app/src/test/java/org/thoughtcrime/securesms/backup/v2/stream/EncryptedBackupReaderWriterTest.kt @@ -0,0 +1,46 @@ +/* + * Copyright 2023 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ + +package org.thoughtcrime.securesms.backup.v2.stream + +import org.junit.Assert.assertEquals +import org.junit.Test +import org.thoughtcrime.securesms.backup.v2.proto.AccountData +import org.thoughtcrime.securesms.backup.v2.proto.Frame +import org.thoughtcrime.securesms.util.Util +import org.whispersystems.signalservice.api.backup.BackupKey +import org.whispersystems.signalservice.api.push.ServiceId.ACI +import java.io.ByteArrayOutputStream +import java.util.UUID + +class EncryptedBackupReaderWriterTest { + + @Test + fun `can read back all of the frames we write`() { + val key = BackupKey(Util.getSecretBytes(32)) + val aci = ACI.from(UUID.randomUUID()) + + val outputStream = ByteArrayOutputStream() + + val frameCount = 10_000 + EncryptedBackupWriter(key, aci, outputStream, append = { outputStream.write(it) }).use { writer -> + for (i in 0 until frameCount) { + writer.write(Frame(account = AccountData(username = "username-$i"))) + } + } + + val ciphertext: ByteArray = outputStream.toByteArray() + + val frames: List = EncryptedBackupReader(key, aci, ciphertext.size.toLong()) { ciphertext.inputStream() }.use { reader -> + reader.asSequence().toList() + } + + assertEquals(frameCount, frames.size) + + for (i in 0 until frameCount) { + assertEquals("username-$i", frames[i].account?.username) + } + } +} diff --git a/core-util-jvm/build.gradle.kts b/core-util-jvm/build.gradle.kts index 566f1e1e0a..b9dd6777b2 100644 --- a/core-util-jvm/build.gradle.kts +++ b/core-util-jvm/build.gradle.kts @@ -18,4 +18,5 @@ java { dependencies { testImplementation(testLibs.junit.junit) + testImplementation(testLibs.assertj.core) } diff --git a/core-util-jvm/src/main/java/org/signal/core/util/InputStreamExtensions.kt b/core-util-jvm/src/main/java/org/signal/core/util/InputStreamExtensions.kt new file mode 100644 index 0000000000..e4d2f8e2ee --- /dev/null +++ b/core-util-jvm/src/main/java/org/signal/core/util/InputStreamExtensions.kt @@ -0,0 +1,82 @@ +/* + * Copyright 2023 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ + +package org.signal.core.util + +import java.io.IOException +import java.io.InputStream +import kotlin.jvm.Throws + +/** + * Reads a 32-bit variable-length integer from the stream. + * + * The format uses one byte for each 7 bits of the integer, with the most significant bit (MSB) of each byte indicating whether more bytes need to be read. + * If the MSB is 0, it indicates the final byte. The actual integer value is constructed from the remaining 7 bits of each byte. + */ +fun InputStream.readVarInt32(): Int { + var result = 0 + + // We read 7 bits of the integer at a time, up to the full size of an integer (32 bits). + for (shift in 0 until 32 step 7) { + // Despite returning an int, the range of the returned value is 0..255, so it's just a byte. + // I believe it's an int just so it can return -1 when the stream ends. + val byte: Int = read() + if (byte < 0) { + return -1 + } + + val lowestSevenBits = byte and 0x7F + val shiftedBits = lowestSevenBits shl shift + + result = result or shiftedBits + + // If the MSB is 0, that means the varint is finished, and we have our full result + if (byte and 0x80 == 0) { + return result + } + } + + throw IOException("Malformed varint!") +} + +/** + * Reads the entire stream into a [ByteArray]. + */ +@Throws(IOException::class) +fun InputStream.readFully(autoClose: Boolean = true): ByteArray { + return StreamUtil.readFully(this, Integer.MAX_VALUE, autoClose) +} + +/** + * Fills reads data from the stream into the [buffer] until it is full. + * Throws an [IOException] if the stream doesn't have enough data to fill the buffer. + */ +@Throws(IOException::class) +fun InputStream.readFully(buffer: ByteArray) { + return StreamUtil.readFully(this, buffer) +} + +/** + * Reads the specified number of bytes from the stream and returns it as a [ByteArray]. + * Throws an [IOException] if the stream doesn't have that many bytes. + */ +@Throws(IOException::class) +fun InputStream.readNBytesOrThrow(length: Int): ByteArray { + val buffer = ByteArray(length) + this.readFully(buffer) + return buffer +} + +@Throws(IOException::class) +fun InputStream.readLength(): Long? { + val buffer = ByteArray(4096) + var count = 0L + + while (this.read(buffer).also { if (it > 0) count += it } != -1) { + // do nothing, all work is in the while condition + } + + return count +} diff --git a/core-util-jvm/src/main/java/org/signal/core/util/OutputStreamExtensions.kt b/core-util-jvm/src/main/java/org/signal/core/util/OutputStreamExtensions.kt new file mode 100644 index 0000000000..5da9dee2d9 --- /dev/null +++ b/core-util-jvm/src/main/java/org/signal/core/util/OutputStreamExtensions.kt @@ -0,0 +1,32 @@ +/* + * Copyright 2023 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ + +package org.signal.core.util + +import java.io.OutputStream + +/** + * Writes a 32-bit variable-length integer to the stream. + * + * The format uses one byte for each 7 bits of the integer, with the most significant bit (MSB) of each byte indicating whether more bytes need to be read. + */ +fun OutputStream.writeVarInt32(value: Int) { + var remaining = value + + while (true) { + // We write 7 bits of the integer at a time + val lowestSevenBits = remaining and 0x7F + remaining = remaining ushr 7 + + if (remaining == 0) { + // If there are no more bits to write, we're done + write(lowestSevenBits) + return + } else { + // Otherwise, we need to write the next 7 bits, and set the MSB to 1 to indicate that there are more bits to come + write(lowestSevenBits or 0x80) + } + } +} diff --git a/core-util/src/main/java/org/signal/core/util/StreamUtil.java b/core-util-jvm/src/main/java/org/signal/core/util/StreamUtil.java similarity index 87% rename from core-util/src/main/java/org/signal/core/util/StreamUtil.java rename to core-util-jvm/src/main/java/org/signal/core/util/StreamUtil.java index 373e1a05ec..2ac01b443e 100644 --- a/core-util/src/main/java/org/signal/core/util/StreamUtil.java +++ b/core-util-jvm/src/main/java/org/signal/core/util/StreamUtil.java @@ -1,6 +1,9 @@ -package org.signal.core.util; +/* + * Copyright 2023 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ -import androidx.annotation.Nullable; +package org.signal.core.util; import org.signal.core.util.logging.Log; @@ -20,7 +23,7 @@ public final class StreamUtil { private StreamUtil() {} - public static void close(@Nullable Closeable closeable) { + public static void close(Closeable closeable) { if (closeable == null) return; try { @@ -64,6 +67,10 @@ public final class StreamUtil { } public static byte[] readFully(InputStream in, int maxBytes) throws IOException { + return readFully(in, maxBytes, true); + } + + public static byte[] readFully(InputStream in, int maxBytes, boolean closeWhenDone) throws IOException { ByteArrayOutputStream bout = new ByteArrayOutputStream(); byte[] buffer = new byte[4096]; int totalRead = 0; @@ -77,7 +84,9 @@ public final class StreamUtil { } } - in.close(); + if (closeWhenDone) { + in.close(); + } return bout.toByteArray(); } diff --git a/core-util-jvm/src/main/java/org/signal/core/util/stream/MacInputStream.kt b/core-util-jvm/src/main/java/org/signal/core/util/stream/MacInputStream.kt new file mode 100644 index 0000000000..04593532fc --- /dev/null +++ b/core-util-jvm/src/main/java/org/signal/core/util/stream/MacInputStream.kt @@ -0,0 +1,43 @@ +/* + * Copyright 2023 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ + +package org.signal.core.util.stream + +import java.io.FilterInputStream +import java.io.InputStream +import javax.crypto.Mac + +/** + * Calculates a [Mac] as data is read from the target [InputStream]. + * To get the final MAC, read the [mac] property after the stream has been fully read. + * + * Example: + * ```kotlin + * val stream = MacInputStream(myStream, myMac) + * stream.readFully() + * val mac = stream.mac.doFinal() + * ``` + */ +class MacInputStream(val wrapped: InputStream, val mac: Mac) : FilterInputStream(wrapped) { + override fun read(): Int { + return wrapped.read().also { byte -> + if (byte >= 0) { + mac.update(byte.toByte()) + } + } + } + + override fun read(destination: ByteArray): Int { + return read(destination, 0, destination.size) + } + + override fun read(destination: ByteArray, offset: Int, length: Int): Int { + return wrapped.read(destination, offset, length).also { bytesRead -> + if (bytesRead > 0) { + mac.update(destination, offset, bytesRead) + } + } + } +} diff --git a/core-util-jvm/src/main/java/org/signal/core/util/stream/MacOutputStream.kt b/core-util-jvm/src/main/java/org/signal/core/util/stream/MacOutputStream.kt new file mode 100644 index 0000000000..e745fb58ef --- /dev/null +++ b/core-util-jvm/src/main/java/org/signal/core/util/stream/MacOutputStream.kt @@ -0,0 +1,37 @@ +/* + * Copyright 2023 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ + +package org.signal.core.util.stream + +import java.io.FilterOutputStream +import java.io.OutputStream +import javax.crypto.Mac + +/** + * Calculates a [Mac] as data is written to the target [OutputStream]. + * To get the final MAC, read the [mac] property after the stream has been fully written. + * + * Example: + * ```kotlin + * val stream = MacOutputStream(myStream, myMac) + * // write data to stream + * val mac = stream.mac.doFinal() + * ``` + */ +class MacOutputStream(val wrapped: OutputStream, val mac: Mac) : FilterOutputStream(wrapped) { + override fun write(byte: Int) { + wrapped.write(byte) + mac.update(byte.toByte()) + } + + override fun write(data: ByteArray) { + write(data, 0, data.size) + } + + override fun write(data: ByteArray, offset: Int, length: Int) { + wrapped.write(data, offset, length) + mac.update(data, offset, length) + } +} diff --git a/core-util-jvm/src/main/java/org/signal/core/util/stream/TruncatingInputStream.kt b/core-util-jvm/src/main/java/org/signal/core/util/stream/TruncatingInputStream.kt new file mode 100644 index 0000000000..36c7dec3d1 --- /dev/null +++ b/core-util-jvm/src/main/java/org/signal/core/util/stream/TruncatingInputStream.kt @@ -0,0 +1,58 @@ +/* + * Copyright 2023 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ + +package org.signal.core.util.stream + +import java.io.FilterInputStream +import java.io.InputStream +import java.lang.UnsupportedOperationException + +/** + * An [InputStream] that will read from the target [InputStream] until it reaches the end, or until it has read [maxBytes] bytes. + */ +class TruncatingInputStream(private val wrapped: InputStream, private val maxBytes: Long) : FilterInputStream(wrapped) { + + private var bytesRead: Long = 0 + + override fun read(): Int { + if (bytesRead >= maxBytes) { + return -1 + } + + return wrapped.read().also { + if (it >= 0) { + bytesRead++ + } + } + } + + override fun read(destination: ByteArray): Int { + return read(destination, 0, destination.size) + } + + override fun read(destination: ByteArray, offset: Int, length: Int): Int { + if (bytesRead >= maxBytes) { + return -1 + } + + val bytesRemaining: Long = maxBytes - bytesRead + val bytesToRead: Int = if (bytesRemaining > length) length else Math.toIntExact(bytesRemaining) + val bytesRead = wrapped.read(destination, offset, bytesToRead) + + if (bytesRead > 0) { + this.bytesRead += bytesRead + } + + return bytesRead + } + + override fun skip(n: Long): Long { + throw UnsupportedOperationException() + } + + override fun reset() { + throw UnsupportedOperationException() + } +} diff --git a/core-util-jvm/src/test/java/org/signal/core/util/VarInt32Tests.kt b/core-util-jvm/src/test/java/org/signal/core/util/VarInt32Tests.kt new file mode 100644 index 0000000000..597b4012c6 --- /dev/null +++ b/core-util-jvm/src/test/java/org/signal/core/util/VarInt32Tests.kt @@ -0,0 +1,75 @@ +/* + * Copyright 2023 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ + +package org.signal.core.util + +import org.junit.Assert.assertEquals +import org.junit.Ignore +import org.junit.Test +import java.io.ByteArrayOutputStream +import java.util.concurrent.atomic.AtomicInteger +import kotlin.random.Random + +class VarInt32Tests { + + /** + * Tests a random sampling of integers. The faster and more practical version of [testAll]. + */ + @Test + fun testRandomSampling() { + val randomInts = (0..100_000).map { Random.nextInt() } + + val bytes = ByteArrayOutputStream().use { outputStream -> + for (value in randomInts) { + outputStream.writeVarInt32(value) + } + outputStream + }.toByteArray() + + bytes.inputStream().use { inputStream -> + for (value in randomInts) { + val read = inputStream.readVarInt32() + assertEquals(value, read) + } + } + } + + /** + * Exhaustively checks reading and writing a varint for all possible integers. + * We can't keep everything in memory, so instead we use sequences to grab a million at a time, + * then run smaller chunks of those in parallel. + */ + @Ignore("This test is very slow (over a minute). It was run once to verify correctness, but the random sampling test should be sufficient for catching regressions.") + @Test + fun testAll() { + val counter = AtomicInteger(0) + + (Int.MIN_VALUE..Int.MAX_VALUE) + .asSequence() + .chunked(1_000_000) + .forEach { bigChunk -> + bigChunk + .chunked(100_000) + .parallelStream() + .forEach { smallChunk -> + println("Chunk ${counter.addAndGet(1)}") + + val bytes = ByteArrayOutputStream().use { outputStream -> + for (value in smallChunk) { + outputStream.writeVarInt32(value) + } + outputStream + }.toByteArray() + + bytes.inputStream().use { inputStream -> + for (value in smallChunk) { + val read = inputStream.readVarInt32() + assertEquals(value, read) + } + } + } + } + } +} diff --git a/core-util-jvm/src/test/java/org/signal/core/util/stream/MacInputStreamTest.kt b/core-util-jvm/src/test/java/org/signal/core/util/stream/MacInputStreamTest.kt new file mode 100644 index 0000000000..520889939c --- /dev/null +++ b/core-util-jvm/src/test/java/org/signal/core/util/stream/MacInputStreamTest.kt @@ -0,0 +1,56 @@ +/* + * Copyright 2023 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ + +package org.signal.core.util.stream + +import org.junit.Assert.assertArrayEquals +import org.junit.Test +import org.signal.core.util.readFully +import java.io.InputStream +import javax.crypto.Mac +import javax.crypto.spec.SecretKeySpec +import kotlin.random.Random + +class MacInputStreamTest { + + @Test + fun `stream mac matches normal mac when reading via buffer`() { + testMacEquality { inputStream -> + inputStream.readFully() + } + } + + @Test + fun `stream mac matches normal mac when reading one byte at a time`() { + testMacEquality { inputStream -> + var lastRead = inputStream.read() + while (lastRead != -1) { + lastRead = inputStream.read() + } + } + } + + private fun testMacEquality(read: (InputStream) -> Unit) { + val data = Random.nextBytes(1_000) + val key = Random.nextBytes(32) + + val mac1 = Mac.getInstance("HmacSHA256").apply { + init(SecretKeySpec(key, "HmacSHA256")) + } + + val mac2 = Mac.getInstance("HmacSHA256").apply { + init(SecretKeySpec(key, "HmacSHA256")) + } + + val expectedMac = mac1.doFinal(data) + + val actualMac = MacInputStream(data.inputStream(), mac2).use { stream -> + read(stream) + stream.mac.doFinal() + } + + assertArrayEquals(expectedMac, actualMac) + } +} diff --git a/core-util-jvm/src/test/java/org/signal/core/util/stream/MacOutputStreamTest.kt b/core-util-jvm/src/test/java/org/signal/core/util/stream/MacOutputStreamTest.kt new file mode 100644 index 0000000000..c7a941df3a --- /dev/null +++ b/core-util-jvm/src/test/java/org/signal/core/util/stream/MacOutputStreamTest.kt @@ -0,0 +1,56 @@ +/* + * Copyright 2023 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ + +package org.signal.core.util.stream + +import org.junit.Assert.assertArrayEquals +import org.junit.Test +import org.signal.core.util.StreamUtil +import java.io.ByteArrayOutputStream +import java.io.OutputStream +import javax.crypto.Mac +import javax.crypto.spec.SecretKeySpec +import kotlin.random.Random + +class MacOutputStreamTest { + + @Test + fun `stream mac matches normal mac when writing via buffer`() { + testMacEquality { data, outputStream -> + StreamUtil.copy(data.inputStream(), outputStream) + } + } + + @Test + fun `stream mac matches normal mac when writing one byte at a time`() { + testMacEquality { data, outputStream -> + for (byte in data) { + outputStream.write(byte.toInt()) + } + } + } + + private fun testMacEquality(write: (ByteArray, OutputStream) -> Unit) { + val data = Random.nextBytes(1_000) + val key = Random.nextBytes(32) + + val mac1 = Mac.getInstance("HmacSHA256").apply { + init(SecretKeySpec(key, "HmacSHA256")) + } + + val mac2 = Mac.getInstance("HmacSHA256").apply { + init(SecretKeySpec(key, "HmacSHA256")) + } + + val expectedMac = mac1.doFinal(data) + + val actualMac = MacOutputStream(ByteArrayOutputStream(), mac2).use { stream -> + write(data, stream) + stream.mac.doFinal() + } + + assertArrayEquals(expectedMac, actualMac) + } +} diff --git a/core-util-jvm/src/test/java/org/signal/core/util/stream/TruncatingInputStreamTest.kt b/core-util-jvm/src/test/java/org/signal/core/util/stream/TruncatingInputStreamTest.kt new file mode 100644 index 0000000000..c62f239497 --- /dev/null +++ b/core-util-jvm/src/test/java/org/signal/core/util/stream/TruncatingInputStreamTest.kt @@ -0,0 +1,35 @@ +/* + * Copyright 2023 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ + +package org.signal.core.util.stream + +import org.junit.Assert.assertEquals +import org.junit.Test +import org.signal.core.util.readFully + +class TruncatingInputStreamTest { + + @Test + fun `when I fully read the stream via a buffer, I should only get maxBytes`() { + val inputStream = TruncatingInputStream(ByteArray(100).inputStream(), maxBytes = 75) + val data = inputStream.readFully() + + assertEquals(75, data.size) + } + + @Test + fun `when I fully read the stream one byte at a time, I should only get maxBytes`() { + val inputStream = TruncatingInputStream(ByteArray(100).inputStream(), maxBytes = 75) + + var count = 0 + var lastRead = inputStream.read() + while (lastRead != -1) { + count++ + lastRead = inputStream.read() + } + + assertEquals(75, count) + } +} diff --git a/core-util/src/main/java/org/signal/core/util/ContentResolverExtensions.kt b/core-util/src/main/java/org/signal/core/util/ContentResolverExtensions.kt new file mode 100644 index 0000000000..28f6d2cc90 --- /dev/null +++ b/core-util/src/main/java/org/signal/core/util/ContentResolverExtensions.kt @@ -0,0 +1,22 @@ +/* + * Copyright 2023 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ + +package org.signal.core.util + +import android.content.ContentResolver +import android.net.Uri +import android.provider.OpenableColumns +import okio.IOException + +@Throws(IOException::class) +fun ContentResolver.getLength(uri: Uri): Long? { + return this.query(uri, arrayOf(OpenableColumns.SIZE), null, null, null)?.use { cursor -> + if (cursor.moveToFirst()) { + cursor.requireLongOrNull(OpenableColumns.SIZE) + } else { + null + } + } ?: openInputStream(uri)?.use { it.readLength() } +} diff --git a/core-util/src/main/java/org/signal/core/util/InputStreamExtensions.kt b/core-util/src/main/java/org/signal/core/util/InputStreamExtensions.kt deleted file mode 100644 index 2c567d556d..0000000000 --- a/core-util/src/main/java/org/signal/core/util/InputStreamExtensions.kt +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright 2023 Signal Messenger, LLC - * SPDX-License-Identifier: AGPL-3.0-only - */ - -package org.signal.core.util - -import java.io.IOException -import java.io.InputStream -import kotlin.jvm.Throws - -/** - * Reads the entire stream into a [ByteArray]. - */ -@Throws(IOException::class) -fun InputStream.readFully(): ByteArray { - return StreamUtil.readFully(this) -} - -/** - * Fills reads data from the stream into the [buffer] until it is full. - * Throws an [IOException] if the stream doesn't have enough data to fill the buffer. - */ -@Throws(IOException::class) -fun InputStream.readFully(buffer: ByteArray) { - return StreamUtil.readFully(this, buffer) -} - -/** - * Reads the specified number of bytes from the stream and returns it as a [ByteArray]. - * Throws an [IOException] if the stream doesn't have that many bytes. - */ -@Throws(IOException::class) -fun InputStream.readNBytesOrThrow(length: Int): ByteArray { - val buffer: ByteArray = ByteArray(length) - this.readFully(buffer) - return buffer -} diff --git a/core-util/src/test/java/org/signal/core/util/InputStreamExtensionTests.kt b/core-util/src/test/java/org/signal/core/util/InputStreamExtensionTests.kt new file mode 100644 index 0000000000..1aeaa1c05e --- /dev/null +++ b/core-util/src/test/java/org/signal/core/util/InputStreamExtensionTests.kt @@ -0,0 +1,22 @@ +/* + * Copyright 2023 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ + +package org.signal.core.util + +import org.junit.Assert.assertEquals +import org.junit.Test +import kotlin.random.Random + +class InputStreamExtensionTests { + + @Test + fun `when I call readLength, it returns the correct length`() { + for (i in 1..10) { + val bytes = ByteArray(Random.nextInt(from = 512, until = 8092)) + val length = bytes.inputStream().readLength() + assertEquals(bytes.size.toLong(), length) + } + } +} diff --git a/libsignal-service/src/main/java/org/whispersystems/signalservice/api/backup/BackupId.kt b/libsignal-service/src/main/java/org/whispersystems/signalservice/api/backup/BackupId.kt new file mode 100644 index 0000000000..6d6a386614 --- /dev/null +++ b/libsignal-service/src/main/java/org/whispersystems/signalservice/api/backup/BackupId.kt @@ -0,0 +1,17 @@ +/* + * Copyright 2023 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ + +package org.whispersystems.signalservice.api.backup + +/** + * Safe typing around a backupId, which is a 16-byte array. + */ +@JvmInline +value class BackupId(val value: ByteArray) { + + init { + require(value.size == 16) { "BackupId must be 16 bytes!" } + } +} diff --git a/libsignal-service/src/main/java/org/whispersystems/signalservice/api/backup/BackupKey.kt b/libsignal-service/src/main/java/org/whispersystems/signalservice/api/backup/BackupKey.kt new file mode 100644 index 0000000000..48343a0e82 --- /dev/null +++ b/libsignal-service/src/main/java/org/whispersystems/signalservice/api/backup/BackupKey.kt @@ -0,0 +1,40 @@ +/* + * Copyright 2023 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ + +package org.whispersystems.signalservice.api.backup + +import org.signal.libsignal.protocol.kdf.HKDF +import org.whispersystems.signalservice.api.push.ServiceId.ACI + +/** + * Safe typing around a backup key, which is a 32-byte array. + */ +class BackupKey(val value: ByteArray) { + init { + require(value.size == 32) { "Backup key must be 32 bytes!" } + } + + fun deriveSecrets(aci: ACI): KeyMaterial { + val backupId = BackupId( + HKDF.deriveSecrets(this.value, aci.toByteArray(), "20231003_Signal_Backups_GenerateBackupId".toByteArray(), 16) + ) + + val extendedKey = HKDF.deriveSecrets(this.value, backupId.value, "20231003_Signal_Backups_EncryptMessageBackup".toByteArray(), 80) + + return KeyMaterial( + backupId = backupId, + macKey = extendedKey.copyOfRange(0, 32), + cipherKey = extendedKey.copyOfRange(32, 64), + iv = extendedKey.copyOfRange(64, 80) + ) + } + + class KeyMaterial( + val backupId: BackupId, + val macKey: ByteArray, + val cipherKey: ByteArray, + val iv: ByteArray + ) +} diff --git a/libsignal-service/src/main/java/org/whispersystems/signalservice/api/kbs/MasterKey.java b/libsignal-service/src/main/java/org/whispersystems/signalservice/api/kbs/MasterKey.java index ca60a0f5c7..68a673acee 100644 --- a/libsignal-service/src/main/java/org/whispersystems/signalservice/api/kbs/MasterKey.java +++ b/libsignal-service/src/main/java/org/whispersystems/signalservice/api/kbs/MasterKey.java @@ -1,5 +1,7 @@ package org.whispersystems.signalservice.api.kbs; +import org.signal.libsignal.protocol.kdf.HKDF; +import org.whispersystems.signalservice.api.backup.BackupKey; import org.whispersystems.signalservice.api.storage.StorageKey; import org.whispersystems.signalservice.internal.util.Hex; import org.signal.core.util.Base64; @@ -44,6 +46,10 @@ public final class MasterKey { return derive("Logging Key"); } + public BackupKey deriveBackupKey() { + return new BackupKey(HKDF.deriveSecrets(masterKey, "20231003_Signal_Backups_GenerateBackupKey".getBytes(), 32)); + } + private byte[] derive(String keyName) { return hmacSha256(masterKey, StringUtil.utf8(keyName)); }