diff --git a/app/src/androidTest/assets/backupTests/account-data.binproto b/app/src/androidTest/assets/backupTests/account-data.binproto deleted file mode 100644 index 8bca63e497..0000000000 Binary files a/app/src/androidTest/assets/backupTests/account-data.binproto and /dev/null differ diff --git a/app/src/androidTest/assets/backupTests/expiration-timer-chat-update-message.binproto b/app/src/androidTest/assets/backupTests/expiration-timer-chat-update-message.binproto deleted file mode 100644 index 29cb01be84..0000000000 Binary files a/app/src/androidTest/assets/backupTests/expiration-timer-chat-update-message.binproto and /dev/null differ diff --git a/app/src/androidTest/assets/backupTests/profile-change-chat-update-message.binproto b/app/src/androidTest/assets/backupTests/profile-change-chat-update-message.binproto deleted file mode 100644 index 937991ded4..0000000000 Binary files a/app/src/androidTest/assets/backupTests/profile-change-chat-update-message.binproto and /dev/null differ diff --git a/app/src/androidTest/assets/backupTests/registered-blocked-contact.binproto b/app/src/androidTest/assets/backupTests/registered-blocked-contact.binproto deleted file mode 100644 index 7f0dc12dad..0000000000 Binary files a/app/src/androidTest/assets/backupTests/registered-blocked-contact.binproto and /dev/null differ diff --git a/app/src/androidTest/assets/backupTests/session-switchover-chat-update-message.binproto b/app/src/androidTest/assets/backupTests/session-switchover-chat-update-message.binproto deleted file mode 100644 index b856d50ac6..0000000000 Binary files a/app/src/androidTest/assets/backupTests/session-switchover-chat-update-message.binproto and /dev/null differ diff --git a/app/src/androidTest/assets/backupTests/simple-chat-update-message.binproto b/app/src/androidTest/assets/backupTests/simple-chat-update-message.binproto deleted file mode 100644 index c8ee14d4d1..0000000000 Binary files a/app/src/androidTest/assets/backupTests/simple-chat-update-message.binproto and /dev/null differ diff --git a/app/src/androidTest/assets/backupTests/story-distribution-list.binproto b/app/src/androidTest/assets/backupTests/story-distribution-list.binproto deleted file mode 100644 index e5eb145ca7..0000000000 Binary files a/app/src/androidTest/assets/backupTests/story-distribution-list.binproto and /dev/null differ diff --git a/app/src/androidTest/assets/backupTests/thread-merge-chat-update-message.binproto b/app/src/androidTest/assets/backupTests/thread-merge-chat-update-message.binproto deleted file mode 100644 index 88c8560982..0000000000 Binary files a/app/src/androidTest/assets/backupTests/thread-merge-chat-update-message.binproto and /dev/null differ diff --git a/app/src/androidTest/assets/backupTests/unregistered-contact.binproto b/app/src/androidTest/assets/backupTests/unregistered-contact.binproto deleted file mode 100644 index f18e15f802..0000000000 Binary files a/app/src/androidTest/assets/backupTests/unregistered-contact.binproto and /dev/null differ diff --git a/app/src/androidTest/java/org/thoughtcrime/securesms/backup/v2/ArchiveImportExportTests.kt b/app/src/androidTest/java/org/thoughtcrime/securesms/backup/v2/ArchiveImportExportTests.kt index 9d9c3a3667..bc69294c6c 100644 --- a/app/src/androidTest/java/org/thoughtcrime/securesms/backup/v2/ArchiveImportExportTests.kt +++ b/app/src/androidTest/java/org/thoughtcrime/securesms/backup/v2/ArchiveImportExportTests.kt @@ -10,7 +10,6 @@ import androidx.test.platform.app.InstrumentationRegistry import com.github.difflib.DiffUtils import com.github.difflib.UnifiedDiffUtils import junit.framework.Assert.assertTrue -import org.junit.Ignore import org.junit.Test import org.junit.runner.RunWith import org.signal.core.util.Base64 @@ -40,84 +39,172 @@ class ArchiveImportExportTests { const val TAG = "ImportExport" const val TESTS_FOLDER = "backupTests" - val SELF_ACI = ServiceId.ACI.from(UUID(100, 100)) - val SELF_PNI = ServiceId.PNI.from(UUID(101, 101)) + val SELF_ACI = ServiceId.ACI.from(UUID.fromString("00000000-0000-4000-8000-000000000001")) + val SELF_PNI = ServiceId.PNI.from(UUID.fromString("00000000-0000-4000-8000-000000000002")) val SELF_E164 = "+10000000000" val SELF_PROFILE_KEY: ByteArray = Base64.decode("YQKRq+3DQklInaOaMcmlzZnN0m/1hzLiaONX7gB12dg=") val MASTER_KEY = Base64.decode("sHuBMP4ToZk4tcNU+S8eBUeCt8Am5EZnvuqTBJIR4Do") } - @Test +// @Test fun all() { runTests() } - @Ignore("Just for debugging") + @Test + fun temp() { + runTests { it == "chat_item_standard_message_standard_attachments_02.binproto" } + } + @Test fun accountData() { runTests { it.startsWith("account_data_") } } - @Ignore("Just for debugging") @Test + fun adHocCall() { + runTests { it.startsWith("ad_hoc_call") } + } + + // Passing +// @Test + fun chat() { + runTests { it.startsWith("chat_") && !it.contains("_item") } + } + + @Test + fun chatItemContactMessage() { + runTests { it.startsWith("chat_item_contact_message_") } + } + + // Passing +// @Test + fun chatItemExpirationTimerUpdate() { + runTests { it.startsWith("chat_item_expiration_timer_") } + } + + // Passing +// @Test + fun chatItemGiftBadge() { + runTests { it.startsWith("chat_item_gift_badge_") } + } + + @Test + fun chatItemGroupCallUpdate() { + runTests { it.startsWith("chat_item_group_call_update_") } + } + + @Test + fun chatItemIndividualCallUpdate() { + runTests { it.startsWith("chat_item_individual_call_update_") } + } + + // Passing +// @Test + fun chatItemLearnedProfileUpdate() { + runTests { it.startsWith("chat_item_learned_profile_update_") } + } + + @Test + fun chatItemPaymentNotification() { + runTests { it.startsWith("chat_item_payment_notification_") } + } + + // Passing +// @Test + fun chatItemProfileChangeUpdate() { + runTests { it.startsWith("chat_item_profile_change_update_") } + } + + // Passing +// @Test + fun chatItemRemoteDelete() { + runTests { it.startsWith("chat_item_remote_delete_") } + } + + // Passing +// @Test + fun chatItemSessionSwitchoverUpdate() { + runTests { it.startsWith("chat_item_session_switchover_update_") } + } + + @Test + fun chatItemSimpleUpdates() { + runTests { it.startsWith("chat_item_simple_updates_") } + } + + @Test + fun chatItemStandardMessageFormattedText() { + runTests { it.startsWith("chat_item_standard_message_formatted_text_") } + } + + @Test + fun chatItemStandardMessageLongText() { + runTests { it.startsWith("chat_item_standard_message_long_text_") } + } + + // Passing +// @Test + fun chatItemStandardMessageSpecialAttachments() { + runTests { it.startsWith("chat_item_standard_message_special_attachments_") } + } + + // Passing +// @Test + fun chatItemStandardMessageStandardAttachments() { + runTests { it.startsWith("chat_item_standard_message_standard_attachments_") } + } + + // Passing +// @Test + fun chatItemStandardMessageTextOnly() { + runTests { it.startsWith("chat_item_standard_message_text_only_") } + } + + @Test + fun chatItemStandardMessageWithEdits() { + runTests { it.startsWith("chat_item_standard_message_with_edits_") } + } + + @Test + fun chatItemStandardMessageWithQuote() { + runTests { it.startsWith("chat_item_standard_message_with_quote_") } + } + + @Test + fun chatItemStickerMessage() { + runTests { it.startsWith("chat_item_sticker_message_") } + } + + // Passing +// @Test + fun chatItemThreadMergeUpdate() { + runTests { it.startsWith("chat_item_thread_merge_update_") } + } + + @Test + fun recipientCallLink() { + runTests { it.startsWith("recipient_call_link_") } + } + + // Passing +// @Test fun recipientContacts() { runTests { it.startsWith("recipient_contacts_") } } - @Ignore("Just for debugging") - @Test + // Passing +// @Test fun recipientDistributionLists() { runTests { it.startsWith("recipient_distribution_list_") } } - @Ignore("Just for debugging") - @Test + // Passing +// @Test fun recipientGroups() { runTests { it.startsWith("recipient_groups_") } } - @Ignore("Just for debugging") - @Test - fun chatStandardMessageTextOnly() { - runTests { it.startsWith("chat_standard_message_text_only_") } - } - - @Ignore("Just for debugging") - @Test - fun chatStandardMessageFormattedText() { - runTests { it.startsWith("chat_standard_message_formatted_text_") } - } - - @Ignore("Just for debugging") - @Test - fun chatStandardMessageLongText() { - runTests { it.startsWith("chat_standard_message_long_text_") } - } - - @Ignore("Just for debugging") - @Test - fun chatStandardMessageStandardAttachments() { - runTests { it.startsWith("chat_standard_message_standard_attachments_") } - } - - @Ignore("Just for debugging") - @Test - fun chatStandardMessageSpecialAttachments() { - runTests { it.startsWith("chat_standard_message_special_attachments_") } - } - - @Ignore("Just for debugging") - @Test - fun chatSimpleUpdates() { - runTests { it.startsWith("chat_simple_updates_") } - } - - @Ignore("Just for debugging") - @Test - fun chatContactMessage() { - runTests { it.startsWith("chat_contact_message_") } - } - private fun runTests(predicate: (String) -> Boolean = { true }) { val testFiles = InstrumentationRegistry.getInstrumentation().context.resources.assets.list(TESTS_FOLDER)!!.filter(predicate) val results: MutableList = mutableListOf() @@ -152,7 +239,7 @@ class ArchiveImportExportTests { val message = "Some tests failed! Only $successCount/${results.size} passed. Failure details are above. Failing tests:\n$failingTestNames" Log.d(TAG, message) - throw AssertionError(message) + throw AssertionError("Some tests failed!") } else { Log.d(TAG, "All ${results.size} tests passed!") } diff --git a/app/src/main/java/org/thoughtcrime/securesms/attachments/TombstoneAttachment.kt b/app/src/main/java/org/thoughtcrime/securesms/attachments/TombstoneAttachment.kt index 3dd2898617..11fe754ada 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/attachments/TombstoneAttachment.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/attachments/TombstoneAttachment.kt @@ -48,6 +48,7 @@ class TombstoneAttachment : Attachment { width: Int?, height: Int?, caption: String?, + fileName: String? = null, blurHash: String?, voiceNote: Boolean = false, borderless: Boolean = false, @@ -59,7 +60,7 @@ class TombstoneAttachment : Attachment { quote = quote, transferState = AttachmentTable.TRANSFER_PROGRESS_PERMANENT_FAILURE, size = 0, - fileName = null, + fileName = fileName, cdn = Cdn.CDN_0, remoteLocation = null, remoteKey = null, diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/ChatItemExportIterator.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/ChatItemExportIterator.kt index 3283d3242c..745e1c51ca 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/ChatItemExportIterator.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/ChatItemExportIterator.kt @@ -21,15 +21,12 @@ import org.signal.core.util.requireLong import org.signal.core.util.requireLongOrNull import org.signal.core.util.requireString import org.thoughtcrime.securesms.attachments.AttachmentId -import org.thoughtcrime.securesms.attachments.Cdn import org.thoughtcrime.securesms.attachments.DatabaseAttachment -import org.thoughtcrime.securesms.backup.v2.BackupRepository.getMediaName import org.thoughtcrime.securesms.backup.v2.proto.ChatItem import org.thoughtcrime.securesms.backup.v2.proto.ChatUpdateMessage import org.thoughtcrime.securesms.backup.v2.proto.ContactAttachment import org.thoughtcrime.securesms.backup.v2.proto.ContactMessage import org.thoughtcrime.securesms.backup.v2.proto.ExpirationTimerChatUpdate -import org.thoughtcrime.securesms.backup.v2.proto.FilePointer import org.thoughtcrime.securesms.backup.v2.proto.GroupCall import org.thoughtcrime.securesms.backup.v2.proto.IndividualCall import org.thoughtcrime.securesms.backup.v2.proto.LearnedProfileChatUpdate @@ -47,6 +44,7 @@ import org.thoughtcrime.securesms.backup.v2.proto.Sticker import org.thoughtcrime.securesms.backup.v2.proto.StickerMessage import org.thoughtcrime.securesms.backup.v2.proto.Text import org.thoughtcrime.securesms.backup.v2.proto.ThreadMergeChatUpdate +import org.thoughtcrime.securesms.backup.v2.util.toRemoteFilePointer import org.thoughtcrime.securesms.contactshare.Contact import org.thoughtcrime.securesms.database.AttachmentTable import org.thoughtcrime.securesms.database.CallTable @@ -96,7 +94,7 @@ import org.thoughtcrime.securesms.backup.v2.proto.GiftBadge as BackupGiftBadge * * All of this complexity is hidden from the user -- they just get a normal iterator interface. */ -class ChatItemExportIterator(private val cursor: Cursor, private val batchSize: Int, private val archiveMedia: Boolean) : Iterator, Closeable { +class ChatItemExportIterator(private val cursor: Cursor, private val batchSize: Int, private val mediaArchiveEnabled: Boolean) : Iterator, Closeable { companion object { private val TAG = Log.tag(ChatItemExportIterator::class.java) @@ -582,7 +580,7 @@ class ChatItemExportIterator(private val cursor: Cursor, private val batchSize: return org.thoughtcrime.securesms.backup.v2.proto.LinkPreview( url = url, title = title, - image = (thumbnail.orNull() as? DatabaseAttachment)?.toBackupAttachment()?.pointer, + image = (thumbnail.orNull() as? DatabaseAttachment)?.toRemoteMessageAttachment()?.pointer, description = description, date = date ) @@ -594,7 +592,7 @@ class ChatItemExportIterator(private val cursor: Cursor, private val batchSize: val contacts = sharedContacts.map { ContactAttachment( name = it.name.toBackup(), - avatar = (it.avatar?.attachment as? DatabaseAttachment)?.toBackupAttachment()?.pointer, + avatar = (it.avatar?.attachment as? DatabaseAttachment)?.toRemoteMessageAttachment()?.pointer, organization = it.organization, number = it.phoneNumbers.map { phone -> ContactAttachment.Phone( @@ -741,7 +739,7 @@ class ChatItemExportIterator(private val cursor: Cursor, private val batchSize: packKey = Hex.fromStringCondensed(stickerLocator.packKey).toByteString(), stickerId = stickerLocator.stickerId, emoji = stickerLocator.emoji, - data_ = this.toBackupAttachment().pointer + data_ = this.toRemoteMessageAttachment().pointer ), reactions = reactions.toBackupReactions() ) @@ -752,50 +750,14 @@ class ChatItemExportIterator(private val cursor: Cursor, private val batchSize: Quote.QuotedAttachment( contentType = attachment.contentType, fileName = attachment.fileName, - thumbnail = attachment.toBackupAttachment() + thumbnail = attachment.toRemoteMessageAttachment() ) } } - private fun DatabaseAttachment.toBackupAttachment(): MessageAttachment { - val builder = FilePointer.Builder() - builder.contentType = this.contentType?.takeUnless { it.isBlank() } - builder.incrementalMac = this.incrementalDigest?.toByteString() - builder.incrementalMacChunkSize = this.incrementalMacChunkSize.takeIf { it > 0 } - builder.fileName = this.fileName - builder.width = this.width.takeUnless { it == 0 } - builder.height = this.height.takeUnless { it == 0 } - builder.caption = this.caption - builder.blurHash = this.blurHash?.hash - - if (this.remoteKey.isNullOrBlank() || this.remoteDigest == null || this.size == 0L) { - builder.invalidAttachmentLocator = FilePointer.InvalidAttachmentLocator() - } else { - if (archiveMedia) { - builder.backupLocator = FilePointer.BackupLocator( - mediaName = this.archiveMediaName ?: this.getMediaName().toString(), - cdnNumber = if (this.archiveMediaName != null) this.archiveCdn else Cdn.CDN_3.cdnNumber, // TODO (clark): Update when new proto with optional cdn is landed - key = Base64.decode(remoteKey).toByteString(), - size = this.size.toInt(), - digest = this.remoteDigest.toByteString() - ) - } else { - if (this.remoteLocation.isNullOrBlank()) { - builder.invalidAttachmentLocator = FilePointer.InvalidAttachmentLocator() - } else { - builder.attachmentLocator = FilePointer.AttachmentLocator( - cdnKey = this.remoteLocation, - cdnNumber = this.cdn.cdnNumber, - uploadTimestamp = this.uploadTimestamp, - key = Base64.decode(remoteKey).toByteString(), - size = this.size.toInt(), - digest = this.remoteDigest.toByteString() - ) - } - } - } + private fun DatabaseAttachment.toRemoteMessageAttachment(): MessageAttachment { return MessageAttachment( - pointer = builder.build(), + pointer = this.toRemoteFilePointer(mediaArchiveEnabled), wasDownloaded = this.transferState == AttachmentTable.TRANSFER_PROGRESS_DONE || this.transferState == AttachmentTable.TRANSFER_NEEDS_RESTORE, flag = if (this.voiceNote) { MessageAttachment.Flag.VOICE_MESSAGE @@ -812,7 +774,7 @@ class ChatItemExportIterator(private val cursor: Cursor, private val batchSize: private fun List.toBackupAttachments(): List { return this.map { attachment -> - attachment.toBackupAttachment() + attachment.toRemoteMessageAttachment() } } @@ -937,13 +899,8 @@ class ChatItemExportIterator(private val cursor: Cursor, private val batchSize: reason = SendStatus.Failed.FailureReason.NETWORK ) } - this.baseType == MessageTypes.BASE_SENT_TYPE -> { - statusBuilder.sent = SendStatus.Sent( - sealedSender = this.sealedSender - ) - } - this.hasDeliveryReceipt -> { - statusBuilder.delivered = SendStatus.Delivered( + this.viewed -> { + statusBuilder.viewed = SendStatus.Viewed( sealedSender = this.sealedSender ) } @@ -952,8 +909,21 @@ class ChatItemExportIterator(private val cursor: Cursor, private val batchSize: sealedSender = this.sealedSender ) } - this.viewed -> { - statusBuilder.viewed = SendStatus.Viewed( + this.hasDeliveryReceipt -> { + statusBuilder.delivered = SendStatus.Delivered( + sealedSender = this.sealedSender + ) + } + this.baseType == MessageTypes.BASE_SENT_FAILED_TYPE -> { + statusBuilder.failed = SendStatus.Failed( + reason = SendStatus.Failed.FailureReason.UNKNOWN + ) + } + this.baseType == MessageTypes.BASE_SENDING_SKIPPED_TYPE -> { + statusBuilder.skipped = SendStatus.Skipped() + } + this.baseType == MessageTypes.BASE_SENT_TYPE -> { + statusBuilder.sent = SendStatus.Sent( sealedSender = this.sealedSender ) } diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/ChatItemImportInserter.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/ChatItemImportInserter.kt index 55324d3bf0..ade08b8942 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/ChatItemImportInserter.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/ChatItemImportInserter.kt @@ -94,6 +94,7 @@ class ChatItemImportInserter( MessageTable.DATE_SENT, MessageTable.DATE_RECEIVED, MessageTable.DATE_SERVER, + MessageTable.RECEIPT_TIMESTAMP, MessageTable.TYPE, MessageTable.THREAD_ID, MessageTable.READ, @@ -566,8 +567,19 @@ class ChatItemImportInserter( var type: Long = if (this.outgoing != null) { if (this.outgoing.sendStatus.count { it.failed?.reason == SendStatus.Failed.FailureReason.IDENTITY_KEY_MISMATCH } > 0) { MessageTypes.BASE_SENT_FAILED_TYPE + } else if (this.outgoing.sendStatus.count { it.failed?.reason == SendStatus.Failed.FailureReason.UNKNOWN } > 0) { + MessageTypes.BASE_SENT_FAILED_TYPE } else if (this.outgoing.sendStatus.count { it.failed?.reason == SendStatus.Failed.FailureReason.NETWORK } > 0) { MessageTypes.BASE_SENDING_TYPE + } else if (this.outgoing.sendStatus.count { it.pending != null } > 0) { + MessageTypes.BASE_SENDING_TYPE + } else if (this.outgoing.sendStatus.count { it.skipped != null } > 0) { + val count = this.outgoing.sendStatus.count { it.skipped != null } + if (count == this.outgoing.sendStatus.size) { + MessageTypes.BASE_SENDING_SKIPPED_TYPE + } else { + MessageTypes.BASE_SENDING_TYPE + } } else { MessageTypes.BASE_SENT_TYPE } diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/RecipientTableBackupExtensions.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/RecipientTableBackupExtensions.kt index 7d7fe0a21d..9ac5820866 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/RecipientTableBackupExtensions.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/RecipientTableBackupExtensions.kt @@ -310,7 +310,7 @@ private fun DecryptedGroup.toSnapshot(): Group.GroupSnapshot? { return Group.GroupSnapshot( title = Group.GroupAttributeBlob(title = this.title), avatarUrl = this.avatar, - disappearingMessagesTimer = Group.GroupAttributeBlob(disappearingMessagesDuration = this.disappearingMessagesTimer?.duration ?: 0), + disappearingMessagesTimer = this.disappearingMessagesTimer?.takeIf { it.duration > 0 }?.let { Group.GroupAttributeBlob(disappearingMessagesDuration = it.duration) }, accessControl = this.accessControl?.toSnapshot(), version = this.revision, members = this.members.map { it.toSnapshot() }, diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/ThreadTableBackupExtensions.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/ThreadTableBackupExtensions.kt index 638b2e1b36..ba276e15bc 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/ThreadTableBackupExtensions.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/ThreadTableBackupExtensions.kt @@ -24,7 +24,6 @@ import org.thoughtcrime.securesms.backup.v2.util.toLocal import org.thoughtcrime.securesms.backup.v2.util.toLocalAttachment import org.thoughtcrime.securesms.conversation.colors.ChatColors import org.thoughtcrime.securesms.database.RecipientTable -import org.thoughtcrime.securesms.database.SQLiteDatabase import org.thoughtcrime.securesms.database.SignalDatabase import org.thoughtcrime.securesms.database.ThreadTable import org.thoughtcrime.securesms.database.model.databaseprotos.ChatColor @@ -39,7 +38,7 @@ import java.io.Closeable private val TAG = Log.tag(ThreadTable::class.java) -fun ThreadTable.getThreadsForBackup(): ChatExportIterator { +fun ThreadTable.getThreadsForBackup(db: SignalDatabase): ChatExportIterator { //language=sql val query = """ SELECT @@ -61,7 +60,7 @@ fun ThreadTable.getThreadsForBackup(): ChatExportIterator { """ val cursor = readableDatabase.query(query) - return ChatExportIterator(cursor, readableDatabase) + return ChatExportIterator(cursor, db) } fun ThreadTable.clearAllDataForBackupRestore() { @@ -72,13 +71,6 @@ fun ThreadTable.clearAllDataForBackupRestore() { fun ThreadTable.restoreFromBackup(chat: Chat, recipientId: RecipientId, importState: ImportState): Long { val chatColor = chat.style?.toLocal(importState) - val chatColorWithId = if (chatColor != null && chatColor.id is ChatColors.Id.NotSet) { - val savedColors = SignalDatabase.chatColors.getSavedChatColors() - val match = savedColors.find { it.matchesWithoutId(chatColor) } - match ?: SignalDatabase.chatColors.saveChatColors(chatColor) - } else { - chatColor - } val wallpaperAttachmentId: AttachmentId? = chat.style?.wallpaperPhoto?.let { filePointer -> filePointer.toLocalAttachment(importState)?.let { @@ -125,7 +117,7 @@ fun ThreadTable.restoreFromBackup(chat: Chat, recipientId: RecipientId, importSt return threadId } -class ChatExportIterator(private val cursor: Cursor, private val readableDatabase: SQLiteDatabase) : Iterator, Closeable { +class ChatExportIterator(private val cursor: Cursor, private val db: SignalDatabase) : Iterator, Closeable { override fun hasNext(): Boolean { return cursor.count > 0 && !cursor.isLast } @@ -157,7 +149,7 @@ class ChatExportIterator(private val cursor: Cursor, private val readableDatabas markedUnread = ThreadTable.ReadStatus.deserialize(cursor.requireInt(ThreadTable.READ)) == ThreadTable.ReadStatus.FORCED_UNREAD, dontNotifyForMentionsIfMuted = RecipientTable.MentionSetting.DO_NOT_NOTIFY.id == cursor.requireInt(RecipientTable.MENTION_SETTING), style = ChatStyleConverter.constructRemoteChatStyle( - readableDatabase = readableDatabase, + db = db, chatColors = chatColors, chatColorId = customChatColorsId, chatWallpaper = chatWallpaper diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/AccountDataProcessor.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/AccountDataProcessor.kt index 0ab62eec08..e493c9e6ae 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/AccountDataProcessor.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/AccountDataProcessor.kt @@ -89,7 +89,7 @@ object AccountDataProcessor { hasCompletedUsernameOnboarding = signalStore.uiHintValues.hasCompletedUsernameOnboarding(), customChatColors = db.chatColorsTable.getSavedChatColors().toRemoteChatColors(), defaultChatStyle = ChatStyleConverter.constructRemoteChatStyle( - readableDatabase = db.signalReadableDatabase, + db = db, chatColors = chatColors, chatColorId = chatColors?.id ?: ChatColors.Id.NotSet, chatWallpaper = chatWallpaper diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/ChatBackupProcessor.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/ChatBackupProcessor.kt index 4ff4d2fe9f..b4969c3ad2 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/ChatBackupProcessor.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/ChatBackupProcessor.kt @@ -20,7 +20,7 @@ object ChatBackupProcessor { val TAG = Log.tag(ChatBackupProcessor::class.java) fun export(db: SignalDatabase, exportState: ExportState, emitter: BackupFrameEmitter) { - db.threadTable.getThreadsForBackup().use { reader -> + db.threadTable.getThreadsForBackup(db).use { reader -> for (chat in reader) { if (exportState.recipientIds.contains(chat.recipientId)) { exportState.threadIds.add(chat.id) diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/util/ArchiveProtoExtensions.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/util/ArchiveConverterExtensions.kt similarity index 62% rename from app/src/main/java/org/thoughtcrime/securesms/backup/v2/util/ArchiveProtoExtensions.kt rename to app/src/main/java/org/thoughtcrime/securesms/backup/v2/util/ArchiveConverterExtensions.kt index c26d17f0c8..ea23c9429e 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/util/ArchiveProtoExtensions.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/util/ArchiveConverterExtensions.kt @@ -6,12 +6,16 @@ package org.thoughtcrime.securesms.backup.v2.util import okio.ByteString +import okio.ByteString.Companion.toByteString +import org.signal.core.util.Base64 import org.signal.core.util.orNull import org.thoughtcrime.securesms.attachments.ArchivedAttachment import org.thoughtcrime.securesms.attachments.Attachment import org.thoughtcrime.securesms.attachments.Cdn +import org.thoughtcrime.securesms.attachments.DatabaseAttachment import org.thoughtcrime.securesms.attachments.PointerAttachment import org.thoughtcrime.securesms.attachments.TombstoneAttachment +import org.thoughtcrime.securesms.backup.v2.BackupRepository.getMediaName import org.thoughtcrime.securesms.backup.v2.ImportState import org.thoughtcrime.securesms.backup.v2.proto.FilePointer import org.thoughtcrime.securesms.database.AttachmentTable @@ -73,6 +77,7 @@ fun FilePointer?.toLocalAttachment( width = this.width, height = this.height, caption = this.caption, + fileName = this.fileName, blurHash = this.blurHash, voiceNote = voiceNote, borderless = borderless, @@ -110,3 +115,58 @@ fun FilePointer?.toLocalAttachment( } return null } + +/** + * @param mediaArchiveEnabled True if this user has enable media backup, otherwise false. + */ +fun DatabaseAttachment.toRemoteFilePointer(mediaArchiveEnabled: Boolean): FilePointer { + val builder = FilePointer.Builder() + builder.contentType = this.contentType?.takeUnless { it.isBlank() } + builder.incrementalMac = this.incrementalDigest?.toByteString() + builder.incrementalMacChunkSize = this.incrementalMacChunkSize.takeIf { it > 0 } + builder.fileName = this.fileName + builder.width = this.width.takeIf { it > 0 } + builder.height = this.height.takeIf { it > 0 } + builder.caption = this.caption + builder.blurHash = this.blurHash?.hash + + if (this.remoteKey.isNullOrBlank() || this.remoteDigest == null || this.size == 0L) { + builder.invalidAttachmentLocator = FilePointer.InvalidAttachmentLocator() + return builder.build() + } + + if (this.transferState == AttachmentTable.TRANSFER_PROGRESS_PERMANENT_FAILURE && this.archiveTransferState != AttachmentTable.ArchiveTransferState.FINISHED) { + builder.invalidAttachmentLocator = FilePointer.InvalidAttachmentLocator() + return builder.build() + } + + val pending = this.archiveTransferState != AttachmentTable.ArchiveTransferState.FINISHED && (this.transferState != AttachmentTable.TRANSFER_PROGRESS_DONE && this.transferState != AttachmentTable.TRANSFER_RESTORE_OFFLOADED) + + if (mediaArchiveEnabled && !pending) { + builder.backupLocator = FilePointer.BackupLocator( + mediaName = this.archiveMediaName ?: this.getMediaName().toString(), + cdnNumber = if (this.archiveMediaName != null) this.archiveCdn else Cdn.CDN_3.cdnNumber, // TODO [backup]: Update when new proto with optional cdn is landed + key = Base64.decode(remoteKey).toByteString(), + size = this.size.toInt(), + digest = this.remoteDigest.toByteString(), + transitCdnNumber = this.cdn.cdnNumber.takeIf { this.remoteLocation != null }, + transitCdnKey = this.remoteLocation + ) + return builder.build() + } + + if (this.remoteLocation.isNullOrBlank()) { + builder.invalidAttachmentLocator = FilePointer.InvalidAttachmentLocator() + return builder.build() + } + + builder.attachmentLocator = FilePointer.AttachmentLocator( + cdnKey = this.remoteLocation, + cdnNumber = this.cdn.cdnNumber, + uploadTimestamp = this.uploadTimestamp, + key = Base64.decode(remoteKey).toByteString(), + size = this.size.toInt(), + digest = this.remoteDigest.toByteString() + ) + return builder.build() +} diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/util/ChatStyleConverter.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/util/ChatStyleConverter.kt index 309af8e92d..b14d2ca149 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/util/ChatStyleConverter.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/util/ChatStyleConverter.kt @@ -5,24 +5,13 @@ package org.thoughtcrime.securesms.backup.v2.util -import android.database.Cursor -import okio.ByteString -import okio.ByteString.Companion.toByteString -import org.signal.core.util.Base64 -import org.signal.core.util.readToSingleObject -import org.signal.core.util.requireBlob -import org.signal.core.util.requireInt -import org.signal.core.util.requireLong -import org.signal.core.util.requireString -import org.signal.core.util.select import org.thoughtcrime.securesms.attachments.AttachmentId import org.thoughtcrime.securesms.backup.v2.ImportState import org.thoughtcrime.securesms.backup.v2.proto.ChatStyle import org.thoughtcrime.securesms.backup.v2.proto.FilePointer import org.thoughtcrime.securesms.conversation.colors.ChatColors import org.thoughtcrime.securesms.conversation.colors.ChatColorsPalette -import org.thoughtcrime.securesms.database.AttachmentTable -import org.thoughtcrime.securesms.database.SQLiteDatabase +import org.thoughtcrime.securesms.database.SignalDatabase import org.thoughtcrime.securesms.database.model.databaseprotos.Wallpaper import org.thoughtcrime.securesms.mms.PartUriParser import org.thoughtcrime.securesms.util.UriUtil @@ -36,7 +25,7 @@ import org.thoughtcrime.securesms.wallpaper.SingleColorChatWallpaper */ object ChatStyleConverter { fun constructRemoteChatStyle( - readableDatabase: SQLiteDatabase, + db: SignalDatabase, chatColors: ChatColors?, chatColorId: ChatColors.Id, chatWallpaper: Wallpaper? @@ -73,7 +62,7 @@ object ChatStyleConverter { chatStyleBuilder.wallpaperPreset = chatWallpaper.linearGradient.toRemoteWallpaperPreset() } chatWallpaper.file_ != null -> { - chatStyleBuilder.wallpaperPhoto = chatWallpaper.file_.toFilePointer(readableDatabase) + chatStyleBuilder.wallpaperPhoto = chatWallpaper.file_.toFilePointer(db) } } @@ -219,61 +208,8 @@ private fun Wallpaper.LinearGradient.toRemoteWallpaperPreset(): ChatStyle.Wallpa } } -private fun Wallpaper.File.toFilePointer(readableDatabase: SQLiteDatabase): FilePointer? { +private fun Wallpaper.File.toFilePointer(db: SignalDatabase): FilePointer? { val attachmentId: AttachmentId = UriUtil.parseOrNull(this.uri)?.let { PartUriParser(it).partId } ?: return null - - val wallpaperAttachment: ArchiveAttachmentData? = readableDatabase - .select( - AttachmentTable.ARCHIVE_MEDIA_NAME, - AttachmentTable.ARCHIVE_CDN, - AttachmentTable.REMOTE_KEY, - AttachmentTable.REMOTE_DIGEST, - AttachmentTable.DATA_SIZE, - AttachmentTable.CONTENT_TYPE, - AttachmentTable.WIDTH, - AttachmentTable.HEIGHT - ) - .from(AttachmentTable.TABLE_NAME) - .where("${AttachmentTable.ID} = ?", attachmentId.id) - .run() - .readToSingleObject { cursor -> cursor.toArchiveAttachmentData() } - - return wallpaperAttachment?.let { attachment -> - FilePointer( - backupLocator = FilePointer.BackupLocator( - mediaName = attachment.archiveMediaName ?: "", - cdnNumber = attachment.archiveCdn, - key = attachment.remoteKey?.toByteString() ?: ByteString.EMPTY, - size = attachment.size.toInt(), - digest = attachment.remoteDigest?.toByteString() ?: ByteString.EMPTY - ), - contentType = attachment.contentType, - width = attachment.width, - height = attachment.height - ) - } + val attachment = db.attachmentTable.getAttachment(attachmentId) + return attachment?.toRemoteFilePointer(mediaArchiveEnabled = true) } - -private fun Cursor.toArchiveAttachmentData(): ArchiveAttachmentData { - return ArchiveAttachmentData( - archiveMediaName = this.requireString(AttachmentTable.ARCHIVE_MEDIA_NAME), - archiveCdn = this.requireInt(AttachmentTable.ARCHIVE_CDN), - remoteKey = this.requireString(AttachmentTable.REMOTE_KEY)?.let { Base64.decodeOrNull(it) }, - remoteDigest = this.requireBlob(AttachmentTable.REMOTE_DIGEST), - size = this.requireLong(AttachmentTable.DATA_SIZE), - contentType = this.requireString(AttachmentTable.CONTENT_TYPE), - width = this.requireInt(AttachmentTable.WIDTH), - height = this.requireInt(AttachmentTable.HEIGHT) - ) -} - -private class ArchiveAttachmentData( - val archiveMediaName: String?, - val archiveCdn: Int, - val remoteKey: ByteArray?, - val remoteDigest: ByteArray?, - val size: Long, - val contentType: String?, - val width: Int, - val height: Int -) diff --git a/app/src/main/java/org/thoughtcrime/securesms/database/AttachmentTable.kt b/app/src/main/java/org/thoughtcrime/securesms/database/AttachmentTable.kt index a6ff807ed0..aa5bf79f87 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/database/AttachmentTable.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/database/AttachmentTable.kt @@ -2227,6 +2227,7 @@ class AttachmentTable( put(ARCHIVE_MEDIA_NAME, attachment.archiveMediaName) put(ARCHIVE_MEDIA_ID, attachment.archiveMediaId) put(ARCHIVE_THUMBNAIL_MEDIA_ID, attachment.archiveThumbnailMediaId) + put(ARCHIVE_TRANSFER_STATE, ArchiveTransferState.FINISHED.value) put(THUMBNAIL_RESTORE_STATE, ThumbnailRestoreState.NEEDS_RESTORE.value) put(ATTACHMENT_UUID, attachment.uuid?.toString()) put(BLUR_HASH, attachment.blurHash?.hash) diff --git a/app/src/main/java/org/thoughtcrime/securesms/database/MessageTypes.java b/app/src/main/java/org/thoughtcrime/securesms/database/MessageTypes.java index 416f731a90..616db75cfc 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/database/MessageTypes.java +++ b/app/src/main/java/org/thoughtcrime/securesms/database/MessageTypes.java @@ -60,12 +60,14 @@ public interface MessageTypes { long BASE_PENDING_SECURE_SMS_FALLBACK = 25; long BASE_PENDING_INSECURE_SMS_FALLBACK = 26; long BASE_DRAFT_TYPE = 27; + long BASE_SENDING_SKIPPED_TYPE = 28; long[] OUTGOING_MESSAGE_TYPES = { BASE_OUTBOX_TYPE, BASE_SENT_TYPE, BASE_SENDING_TYPE, BASE_SENT_FAILED_TYPE, BASE_PENDING_SECURE_SMS_FALLBACK, BASE_PENDING_INSECURE_SMS_FALLBACK, - OUTGOING_AUDIO_CALL_TYPE, OUTGOING_VIDEO_CALL_TYPE }; + OUTGOING_AUDIO_CALL_TYPE, OUTGOING_VIDEO_CALL_TYPE, + BASE_SENDING_SKIPPED_TYPE }; // Message attributes long MESSAGE_ATTRIBUTE_MASK = 0xE0;