diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/ArchiveValidator.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/ArchiveValidator.kt new file mode 100644 index 0000000000..69c05a968c --- /dev/null +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/ArchiveValidator.kt @@ -0,0 +1,43 @@ +/* + * Copyright 2024 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ + +package org.thoughtcrime.securesms.backup.v2 + +import org.signal.libsignal.messagebackup.MessageBackup +import org.signal.libsignal.messagebackup.ValidationError +import org.thoughtcrime.securesms.keyvalue.SignalStore +import org.whispersystems.signalservice.api.backup.MessageBackupKey +import java.io.File +import java.io.IOException +import org.signal.libsignal.messagebackup.BackupKey as LibSignalBackupKey +import org.signal.libsignal.messagebackup.MessageBackupKey as LibSignalMessageBackupKey + +object ArchiveValidator { + + /** + * Validates the provided [backupFile] that is encrypted with the provided [backupKey]. + */ + fun validate(backupFile: File, backupKey: MessageBackupKey): ValidationResult { + return try { + val backupId = backupKey.deriveBackupId(SignalStore.account.requireAci()) + val libSignalBackupKey = LibSignalBackupKey(backupKey.value) + val backupKey = LibSignalMessageBackupKey(libSignalBackupKey, backupId.value) + + MessageBackup.validate(backupKey, MessageBackup.Purpose.REMOTE_BACKUP, { backupFile.inputStream() }, backupFile.length()) + + ValidationResult.Success + } catch (e: IOException) { + ValidationResult.ReadError(e) + } catch (e: ValidationError) { + ValidationResult.ValidationError(e) + } + } + + sealed interface ValidationResult { + data object Success : ValidationResult + data class ReadError(val exception: IOException) : ValidationResult + data class ValidationError(val exception: org.signal.libsignal.messagebackup.ValidationError) : ValidationResult + } +} diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/BackupRepository.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/BackupRepository.kt index 590018e0da..82baed10c0 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/BackupRepository.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/BackupRepository.kt @@ -276,6 +276,7 @@ object BackupRepository { fun localExport( main: OutputStream, localBackupProgressEmitter: ExportProgressListener, + cancellationSignal: () -> Boolean = { false }, archiveAttachment: (AttachmentTable.LocalArchivableAttachment, () -> InputStream?) -> Unit ) { val writer = EncryptedBackupWriter( @@ -285,7 +286,7 @@ object BackupRepository { append = { main.write(it) } ) - export(currentTime = System.currentTimeMillis(), isLocal = true, writer = writer, progressEmitter = localBackupProgressEmitter) { dbSnapshot -> + export(currentTime = System.currentTimeMillis(), isLocal = true, writer = writer, progressEmitter = localBackupProgressEmitter, cancellationSignal = cancellationSignal) { dbSnapshot -> val localArchivableAttachments = dbSnapshot .attachmentTable .getLocalArchivableAttachments() @@ -308,10 +309,11 @@ object BackupRepository { } } + @JvmOverloads fun export( outputStream: OutputStream, append: (ByteArray) -> Unit, - messageBackupKey: org.whispersystems.signalservice.api.backup.MessageBackupKey = SignalStore.backup.messageBackupKey, + messageBackupKey: MessageBackupKey = SignalStore.backup.messageBackupKey, plaintext: Boolean = false, currentTime: Long = System.currentTimeMillis(), mediaBackupEnabled: Boolean = SignalStore.backup.backsUpMedia, @@ -361,6 +363,7 @@ object BackupRepository { eventTimer.emit("store-db-snapshot") val exportState = ExportState(backupTime = currentTime, mediaBackupEnabled = mediaBackupEnabled) + val selfRecipientId = dbSnapshot.recipientTable.getByAci(signalStoreSnapshot.accountValues.aci!!).get().toLong().let { RecipientId.from(it) } var frameCount = 0L @@ -383,18 +386,18 @@ object BackupRepository { frameCount++ } if (cancellationSignal()) { - Log.w(TAG, "[import] Cancelled! Stopping") + Log.w(TAG, "[export] Cancelled! Stopping") return@export } progressEmitter?.onRecipient() - RecipientArchiveProcessor.export(dbSnapshot, signalStoreSnapshot, exportState) { + RecipientArchiveProcessor.export(dbSnapshot, signalStoreSnapshot, exportState, selfRecipientId) { writer.write(it) eventTimer.emit("recipient") frameCount++ } if (cancellationSignal()) { - Log.w(TAG, "[import] Cancelled! Stopping") + Log.w(TAG, "[export] Cancelled! Stopping") return@export } @@ -409,11 +412,15 @@ object BackupRepository { } progressEmitter?.onCall() - AdHocCallArchiveProcessor.export(dbSnapshot) { frame -> + AdHocCallArchiveProcessor.export(dbSnapshot, exportState) { frame -> writer.write(frame) eventTimer.emit("call") frameCount++ } + if (cancellationSignal()) { + Log.w(TAG, "[export] Cancelled! Stopping") + return@export + } progressEmitter?.onSticker() StickerArchiveProcessor.export(dbSnapshot) { frame -> @@ -421,15 +428,23 @@ object BackupRepository { eventTimer.emit("sticker-pack") frameCount++ } + if (cancellationSignal()) { + Log.w(TAG, "[export] Cancelled! Stopping") + return@export + } progressEmitter?.onMessage() - ChatItemArchiveProcessor.export(dbSnapshot, exportState, cancellationSignal) { frame -> + ChatItemArchiveProcessor.export(dbSnapshot, exportState, selfRecipientId, cancellationSignal) { frame -> writer.write(frame) eventTimer.emit("message") frameCount++ if (frameCount % 1000 == 0L) { Log.d(TAG, "[export] Exported $frameCount frames so far.") + if (cancellationSignal()) { + Log.w(TAG, "[export] Cancelled! Stopping") + return@export + } } } } diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/MessageTableArchiveExtensions.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/MessageTableArchiveExtensions.kt index 585ee46a1a..d3b49ce4cb 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/MessageTableArchiveExtensions.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/MessageTableArchiveExtensions.kt @@ -7,15 +7,18 @@ package org.thoughtcrime.securesms.backup.v2.database import org.signal.core.util.logging.Log import org.signal.core.util.select +import org.thoughtcrime.securesms.backup.v2.ExportState import org.thoughtcrime.securesms.backup.v2.ImportState import org.thoughtcrime.securesms.backup.v2.exporters.ChatItemArchiveExporter import org.thoughtcrime.securesms.backup.v2.importer.ChatItemArchiveImporter +import org.thoughtcrime.securesms.database.GroupTable import org.thoughtcrime.securesms.database.MessageTable import org.thoughtcrime.securesms.database.SignalDatabase +import org.thoughtcrime.securesms.recipients.RecipientId private val TAG = "MessageTableArchiveExtensions" -fun MessageTable.getMessagesForBackup(db: SignalDatabase, backupTime: Long, mediaBackupEnabled: Boolean): ChatItemArchiveExporter { +fun MessageTable.getMessagesForBackup(db: SignalDatabase, backupTime: Long, mediaBackupEnabled: Boolean, selfRecipientId: RecipientId, exportState: ExportState): ChatItemArchiveExporter { // We create a covering index for the query to drastically speed up perf here. // Remember that we're working on a temporary snapshot of the database, so we can create an index and not worry about cleaning it up. val startTime = System.currentTimeMillis() @@ -62,11 +65,26 @@ fun MessageTable.getMessagesForBackup(db: SignalDatabase, backupTime: Long, medi ) Log.d(TAG, "Creating index took ${System.currentTimeMillis() - startTime} ms") + // Unfortunately we have some bad legacy data where the from_recipient_id is a group. + // This cleans it up. Reminder, this is only a snapshot of the data. + db.rawWritableDatabase.execSQL( + """ + UPDATE ${MessageTable.TABLE_NAME} + SET ${MessageTable.FROM_RECIPIENT_ID} = ${selfRecipientId.toLong()} + WHERE ${MessageTable.FROM_RECIPIENT_ID} IN ( + SELECT ${GroupTable.RECIPIENT_ID} + FROM ${GroupTable.TABLE_NAME} + ) + """ + ) + return ChatItemArchiveExporter( db = db, backupStartTime = backupTime, batchSize = 10_000, mediaArchiveEnabled = mediaBackupEnabled, + selfRecipientId = selfRecipientId, + exportState = exportState, cursorGenerator = { lastSeenReceivedTime, count -> readableDatabase .select( diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/RecipientTableArchiveExtensions.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/RecipientTableArchiveExtensions.kt index 0bef7777e6..c246b289e0 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/RecipientTableArchiveExtensions.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/RecipientTableArchiveExtensions.kt @@ -54,7 +54,7 @@ fun RecipientTable.getContactsForBackup(selfId: Long): ContactArchiveExporter { """ ${RecipientTable.TYPE} = ? AND ( ${RecipientTable.ACI_COLUMN} NOT NULL OR - ${RecipientTable.PNI_COLUMN} NOT NULL OR + (${RecipientTable.PNI_COLUMN} NOT NULL AND ${RecipientTable.E164} NOT NULL) OR ${RecipientTable.E164} NOT NULL ) """, @@ -84,7 +84,12 @@ fun RecipientTable.getGroupsForBackup(): GroupArchiveExporter { INNER JOIN ${GroupTable.TABLE_NAME} ON ${RecipientTable.TABLE_NAME}.${RecipientTable.ID} = ${GroupTable.TABLE_NAME}.${GroupTable.RECIPIENT_ID} """ ) - .where("${GroupTable.TABLE_NAME}.${GroupTable.V2_MASTER_KEY} IS NOT NULL") + .where( + """ + ${GroupTable.TABLE_NAME}.${GroupTable.V2_MASTER_KEY} IS NOT NULL AND + ${GroupTable.TABLE_NAME}.${GroupTable.V2_REVISION} >= 0 + """ + ) .run() return GroupArchiveExporter(cursor) diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/ThreadTableArchiveExtensions.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/ThreadTableArchiveExtensions.kt index 21b0f08150..b41a55153c 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/ThreadTableArchiveExtensions.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/ThreadTableArchiveExtensions.kt @@ -5,7 +5,13 @@ package org.thoughtcrime.securesms.backup.v2.database +import org.signal.core.util.SqlUtil +import org.signal.core.util.forEach +import org.signal.core.util.requireInt +import org.signal.core.util.requireLong +import org.signal.core.util.select import org.thoughtcrime.securesms.backup.v2.exporters.ChatArchiveExporter +import org.thoughtcrime.securesms.database.MessageTable import org.thoughtcrime.securesms.database.RecipientTable import org.thoughtcrime.securesms.database.SignalDatabase import org.thoughtcrime.securesms.database.ThreadTable @@ -13,7 +19,7 @@ import org.thoughtcrime.securesms.database.ThreadTable fun ThreadTable.getThreadsForBackup(db: SignalDatabase): ChatArchiveExporter { //language=sql val query = """ - SELECT + SELECT ${ThreadTable.TABLE_NAME}.${ThreadTable.ID}, ${ThreadTable.RECIPIENT_ID}, ${ThreadTable.PINNED}, @@ -26,11 +32,44 @@ fun ThreadTable.getThreadsForBackup(db: SignalDatabase): ChatArchiveExporter { ${RecipientTable.TABLE_NAME}.${RecipientTable.CHAT_COLORS}, ${RecipientTable.TABLE_NAME}.${RecipientTable.CUSTOM_CHAT_COLORS_ID}, ${RecipientTable.TABLE_NAME}.${RecipientTable.WALLPAPER} - FROM ${ThreadTable.TABLE_NAME} + FROM ${ThreadTable.TABLE_NAME} LEFT OUTER JOIN ${RecipientTable.TABLE_NAME} ON ${ThreadTable.TABLE_NAME}.${ThreadTable.RECIPIENT_ID} = ${RecipientTable.TABLE_NAME}.${RecipientTable.ID} - WHERE ${ThreadTable.ACTIVE} = 1 + WHERE + ${ThreadTable.ACTIVE} = 1 AND + ${RecipientTable.TABLE_NAME}.${RecipientTable.TYPE} NOT IN (${RecipientTable.RecipientType.DISTRIBUTION_LIST.id}, ${RecipientTable.RecipientType.CALL_LINK.id}) """ val cursor = readableDatabase.query(query) return ChatArchiveExporter(cursor, db) } + +fun ThreadTable.getThreadGroupStatus(messageIds: Collection): Map { + if (messageIds.isEmpty()) { + return emptyMap() + } + + val out: MutableMap = mutableMapOf() + + val query = SqlUtil.buildFastCollectionQuery("${MessageTable.TABLE_NAME}.${MessageTable.ID}", messageIds) + readableDatabase + .select( + "${MessageTable.TABLE_NAME}.${MessageTable.ID}", + "${RecipientTable.TABLE_NAME}.${RecipientTable.TYPE}" + ) + .from( + """ + ${MessageTable.TABLE_NAME} + INNER JOIN ${ThreadTable.TABLE_NAME} ON ${MessageTable.TABLE_NAME}.${MessageTable.THREAD_ID} = ${ThreadTable.TABLE_NAME}.${ThreadTable.ID} + INNER JOIN ${RecipientTable.TABLE_NAME} ON ${ThreadTable.TABLE_NAME}.${ThreadTable.RECIPIENT_ID} = ${RecipientTable.TABLE_NAME}.${RecipientTable.ID} + """ + ) + .where(query.where, query.whereArgs) + .run() + .forEach { cursor -> + val messageId = cursor.requireLong(MessageTable.ID) + val type = cursor.requireInt(RecipientTable.TYPE) + out[messageId] = type != RecipientTable.RecipientType.INDIVIDUAL.id + } + + return out +} diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/exporters/ChatItemArchiveExporter.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/exporters/ChatItemArchiveExporter.kt index 6892c9920f..8777585da2 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/exporters/ChatItemArchiveExporter.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/exporters/ChatItemArchiveExporter.kt @@ -24,12 +24,16 @@ import org.signal.core.util.requireLongOrNull import org.signal.core.util.requireString import org.thoughtcrime.securesms.attachments.AttachmentId import org.thoughtcrime.securesms.attachments.DatabaseAttachment +import org.thoughtcrime.securesms.backup.v2.ExportState +import org.thoughtcrime.securesms.backup.v2.database.getThreadGroupStatus import org.thoughtcrime.securesms.backup.v2.proto.ChatItem import org.thoughtcrime.securesms.backup.v2.proto.ChatUpdateMessage import org.thoughtcrime.securesms.backup.v2.proto.ContactAttachment import org.thoughtcrime.securesms.backup.v2.proto.ContactMessage import org.thoughtcrime.securesms.backup.v2.proto.ExpirationTimerChatUpdate import org.thoughtcrime.securesms.backup.v2.proto.GroupCall +import org.thoughtcrime.securesms.backup.v2.proto.GroupChangeChatUpdate +import org.thoughtcrime.securesms.backup.v2.proto.GroupV2MigrationUpdate import org.thoughtcrime.securesms.backup.v2.proto.IndividualCall import org.thoughtcrime.securesms.backup.v2.proto.LearnedProfileChatUpdate import org.thoughtcrime.securesms.backup.v2.proto.MessageAttachment @@ -75,6 +79,7 @@ import org.thoughtcrime.securesms.mms.QuoteModel import org.thoughtcrime.securesms.payments.FailureReason import org.thoughtcrime.securesms.payments.State import org.thoughtcrime.securesms.payments.proto.PaymentMetaData +import org.thoughtcrime.securesms.recipients.RecipientId import org.thoughtcrime.securesms.util.JsonUtils import org.whispersystems.signalservice.api.push.ServiceId.ACI import org.whispersystems.signalservice.api.util.UuidUtil @@ -88,6 +93,7 @@ import java.util.concurrent.Callable import java.util.concurrent.ExecutorService import java.util.concurrent.Future import kotlin.jvm.optionals.getOrNull +import kotlin.math.max import kotlin.time.Duration.Companion.days import org.thoughtcrime.securesms.backup.v2.proto.BodyRange as BackupBodyRange import org.thoughtcrime.securesms.backup.v2.proto.GiftBadge as BackupGiftBadge @@ -103,9 +109,11 @@ private val TAG = Log.tag(ChatItemArchiveExporter::class.java) */ class ChatItemArchiveExporter( private val db: SignalDatabase, + private val selfRecipientId: RecipientId, private val backupStartTime: Long, private val batchSize: Int, private val mediaArchiveEnabled: Boolean, + private val exportState: ExportState, private val cursorGenerator: (Long, Int) -> Cursor ) : Iterator, Closeable { @@ -136,7 +144,11 @@ class ChatItemArchiveExporter( eventTimer.emit("extra-data") for ((id, record) in records) { - val builder = record.toBasicChatItemBuilder(extraData.groupReceiptsById[id]) + val builder = record.toBasicChatItemBuilder(selfRecipientId, extraData.isGroupThreadById[id] ?: false, extraData.groupReceiptsById[id], exportState, backupStartTime) + + if (builder == null) { + continue + } when { record.remoteDeleted -> { @@ -209,11 +221,12 @@ class ChatItemArchiveExporter( MessageTypes.isExpirationTimerUpdate(record.type) -> { builder.updateMessage = ChatUpdateMessage(expirationTimerChange = ExpirationTimerChatUpdate(record.expiresIn)) + builder.expireStartDate = 0 builder.expiresInMs = 0 } MessageTypes.isProfileChange(record.type) -> { - builder.updateMessage = record.toRemoteProfileChangeUpdate() + builder.updateMessage = record.toRemoteProfileChangeUpdate() ?: continue } MessageTypes.isSessionSwitchoverType(record.type) -> { @@ -225,12 +238,20 @@ class ChatItemArchiveExporter( } MessageTypes.isGroupV2(record.type) && MessageTypes.isGroupUpdate(record.type) -> { - builder.updateMessage = record.toRemoteGroupUpdate() + builder.updateMessage = record.toRemoteGroupUpdate() ?: continue + } + + MessageTypes.isGroupV1MigrationEvent(record.type) -> { + builder.updateMessage = ChatUpdateMessage( + groupChange = GroupChangeChatUpdate( + updates = listOf(GroupChangeChatUpdate.Update(groupV2MigrationUpdate = GroupV2MigrationUpdate())) + ) + ) } MessageTypes.isCallLog(record.type) -> { val call = db.callTable.getCallByMessageId(record.id) - builder.updateMessage = call?.toRemoteCallUpdate(db, record) + builder.updateMessage = call?.toRemoteCallUpdate(db, record) ?: continue } MessageTypes.isPaymentsNotification(record.type) -> { @@ -338,16 +359,22 @@ class ChatItemArchiveExporter( db.groupReceiptTable.getGroupReceiptInfoForMessages(messageIds) } + val isGroupThreadFuture = executor.submitTyped { + db.threadTable.getThreadGroupStatus(messageIds) + } + val mentionsResult = mentionsFuture.get() val reactionsResult = reactionsFuture.get() val attachmentsResult = attachmentsFuture.get() val groupReceiptsResult = groupReceiptsFuture.get() + val isGroupThreadResult = isGroupThreadFuture.get() return ExtraMessageData( mentionsById = mentionsResult, reactionsById = reactionsResult, attachmentsById = attachmentsResult, - groupReceiptsById = groupReceiptsResult + groupReceiptsById = groupReceiptsResult, + isGroupThreadById = isGroupThreadResult ) } } @@ -356,10 +383,10 @@ private fun simpleUpdate(type: SimpleChatUpdate.Type): ChatUpdateMessage { return ChatUpdateMessage(simpleUpdate = SimpleChatUpdate(type = type)) } -private fun BackupMessageRecord.toBasicChatItemBuilder(groupReceipts: List?): ChatItem.Builder { +private fun BackupMessageRecord.toBasicChatItemBuilder(selfRecipientId: RecipientId, isGroupThread: Boolean, groupReceipts: List?, exportState: ExportState, backupStartTime: Long): ChatItem.Builder? { val record = this - return ChatItem.Builder().apply { + val builder = ChatItem.Builder().apply { chatId = record.threadId authorId = record.fromRecipientId dateSent = record.dateSent @@ -369,19 +396,40 @@ private fun BackupMessageRecord.toBasicChatItemBuilder(groupReceipts: List 0 && outgoing?.sendStatus?.all { it.pending == null && it.failed == null } == true) { + Log.w(TAG, "Outgoing expiring message was sent but the timer wasn't started! Fixing.") + expireStartDate = record.dateReceived + } } else { incoming = ChatItem.IncomingMessageDetails( - dateServerSent = record.dateServer, + dateServerSent = max(record.dateServer, 0), dateReceived = record.dateReceived, read = record.read, sealedSender = record.sealedSender ) + + if (expiresInMs > 0 && incoming?.read == true && expireStartDate == 0L) { + Log.w(TAG, "Incoming expiring message was read but the timer wasn't started! Fixing.") + expireStartDate = record.dateReceived + } } } + + if (!MessageTypes.isExpirationTimerUpdate(record.type) && builder.expiresInMs > 0 && builder.expireStartDate + builder.expiresInMs < backupStartTime + 1.days.inWholeMilliseconds) { + Log.w(TAG, "Message expires too soon! Must skip.") + return null + } + + if (builder.expireStartDate > 0 && builder.expiresInMs == 0L) { + builder.expireStartDate = 0 + } + + return builder } private fun BackupMessageRecord.toRemoteProfileChangeUpdate(): ChatUpdateMessage? { @@ -492,6 +540,8 @@ private fun CallTable.Call.toRemoteCallUpdate(db: SignalDatabase, messageRecord: CallTable.Event.MISSED_NOTIFICATION_PROFILE -> IndividualCall.State.MISSED_NOTIFICATION_PROFILE CallTable.Event.ACCEPTED -> IndividualCall.State.ACCEPTED CallTable.Event.NOT_ACCEPTED -> IndividualCall.State.NOT_ACCEPTED + CallTable.Event.ONGOING -> IndividualCall.State.ACCEPTED + CallTable.Event.DELETE -> return null else -> IndividualCall.State.UNKNOWN_STATE }, startedCallTimestamp = this.timestamp, @@ -715,9 +765,10 @@ private fun BackupMessageRecord.toRemoteStandardMessage(db: SignalDatabase, medi ?.filterNot { linkPreviewAttachments.contains(it) } ?.filterNot { it == longTextAttachment } ?: emptyList() + val hasVoiceNote = messageAttachments.any { it.voiceNote } return StandardMessage( quote = this.toRemoteQuote(mediaArchiveEnabled, quotedAttachments), - text = text, + text = text.takeUnless { hasVoiceNote }, attachments = messageAttachments.toRemoteAttachments(mediaArchiveEnabled), linkPreview = linkPreviews.map { it.toRemoteLinkPreview(mediaArchiveEnabled) }, longText = longTextAttachment?.toRemoteFilePointer(mediaArchiveEnabled), @@ -786,16 +837,22 @@ private fun List.toRemoteQuoteAttachments(mediaArchiveEnable Quote.QuotedAttachment( contentType = attachment.contentType, fileName = attachment.fileName, - thumbnail = attachment.toRemoteMessageAttachment(mediaArchiveEnabled, contentTypeOverride = "image/jpeg").takeUnless { it.pointer?.invalidAttachmentLocator != null } + thumbnail = attachment.toRemoteMessageAttachment( + mediaArchiveEnabled = mediaArchiveEnabled, + flagOverride = MessageAttachment.Flag.NONE, + contentTypeOverride = "image/jpeg" + ).takeUnless { it.pointer?.invalidAttachmentLocator != null } ) } } -private fun DatabaseAttachment.toRemoteMessageAttachment(mediaArchiveEnabled: Boolean, contentTypeOverride: String? = null): MessageAttachment { +private fun DatabaseAttachment.toRemoteMessageAttachment(mediaArchiveEnabled: Boolean, flagOverride: MessageAttachment.Flag? = null, contentTypeOverride: String? = null): MessageAttachment { return MessageAttachment( pointer = this.toRemoteFilePointer(mediaArchiveEnabled, contentTypeOverride), wasDownloaded = this.transferState == AttachmentTable.TRANSFER_PROGRESS_DONE || this.transferState == AttachmentTable.TRANSFER_NEEDS_RESTORE, - flag = if (this.voiceNote) { + flag = if (flagOverride != null) { + flagOverride + } else if (this.voiceNote) { MessageAttachment.Flag.VOICE_MESSAGE } else if (this.videoGif) { MessageAttachment.Flag.GIF @@ -914,14 +971,18 @@ private fun List?.toRemote(): List { } ?: emptyList() } -private fun BackupMessageRecord.toRemoteSendStatus(groupReceipts: List?): List { - if (!groupReceipts.isNullOrEmpty()) { - return groupReceipts.toRemoteSendStatus(this, this.networkFailureRecipientIds, this.identityMismatchRecipientIds) +private fun BackupMessageRecord.toRemoteSendStatus(isGroupThread: Boolean, groupReceipts: List?, exportState: ExportState): List { + if (isGroupThread || !groupReceipts.isNullOrEmpty()) { + return groupReceipts.toRemoteSendStatus(this, this.networkFailureRecipientIds, this.identityMismatchRecipientIds, exportState) + } + + if (!exportState.recipientIds.contains(this.toRecipientId)) { + return emptyList() } val statusBuilder = SendStatus.Builder() .recipientId(this.toRecipientId) - .timestamp(this.receiptTimestamp) + .timestamp(max(this.receiptTimestamp, 0)) when { this.identityMismatchRecipientIds.contains(this.toRecipientId) -> { @@ -970,61 +1031,67 @@ private fun BackupMessageRecord.toRemoteSendStatus(groupReceipts: List.toRemoteSendStatus(messageRecord: BackupMessageRecord, networkFailureRecipientIds: Set, identityMismatchRecipientIds: Set): List { - return this.map { - val statusBuilder = SendStatus.Builder() - .recipientId(it.recipientId.toLong()) - .timestamp(it.timestamp) - - when { - identityMismatchRecipientIds.contains(it.recipientId.toLong()) -> { - statusBuilder.failed = SendStatus.Failed( - reason = SendStatus.Failed.FailureReason.IDENTITY_KEY_MISMATCH - ) - } - MessageTypes.isFailedMessageType(messageRecord.type) && networkFailureRecipientIds.contains(it.recipientId.toLong()) -> { - statusBuilder.failed = SendStatus.Failed( - reason = SendStatus.Failed.FailureReason.NETWORK - ) - } - MessageTypes.isFailedMessageType(messageRecord.type) -> { - statusBuilder.failed = SendStatus.Failed( - reason = SendStatus.Failed.FailureReason.UNKNOWN - ) - } - it.status == GroupReceiptTable.STATUS_UNKNOWN -> { - statusBuilder.pending = SendStatus.Pending() - } - it.status == GroupReceiptTable.STATUS_UNDELIVERED -> { - statusBuilder.sent = SendStatus.Sent( - sealedSender = it.isUnidentified - ) - } - it.status == GroupReceiptTable.STATUS_DELIVERED -> { - statusBuilder.delivered = SendStatus.Delivered( - sealedSender = it.isUnidentified - ) - } - it.status == GroupReceiptTable.STATUS_READ -> { - statusBuilder.read = SendStatus.Read( - sealedSender = it.isUnidentified - ) - } - it.status == GroupReceiptTable.STATUS_VIEWED -> { - statusBuilder.viewed = SendStatus.Viewed( - sealedSender = it.isUnidentified - ) - } - it.status == GroupReceiptTable.STATUS_SKIPPED -> { - statusBuilder.skipped = SendStatus.Skipped() - } - else -> { - statusBuilder.pending = SendStatus.Pending() - } - } - - statusBuilder.build() +private fun List?.toRemoteSendStatus(messageRecord: BackupMessageRecord, networkFailureRecipientIds: Set, identityMismatchRecipientIds: Set, exportState: ExportState): List { + if (this == null) { + return emptyList() } + + return this + .filter { exportState.recipientIds.contains(it.recipientId.toLong()) } + .map { + val statusBuilder = SendStatus.Builder() + .recipientId(it.recipientId.toLong()) + .timestamp(max(it.timestamp, 0)) + + when { + identityMismatchRecipientIds.contains(it.recipientId.toLong()) -> { + statusBuilder.failed = SendStatus.Failed( + reason = SendStatus.Failed.FailureReason.IDENTITY_KEY_MISMATCH + ) + } + MessageTypes.isFailedMessageType(messageRecord.type) && networkFailureRecipientIds.contains(it.recipientId.toLong()) -> { + statusBuilder.failed = SendStatus.Failed( + reason = SendStatus.Failed.FailureReason.NETWORK + ) + } + MessageTypes.isFailedMessageType(messageRecord.type) -> { + statusBuilder.failed = SendStatus.Failed( + reason = SendStatus.Failed.FailureReason.UNKNOWN + ) + } + it.status == GroupReceiptTable.STATUS_UNKNOWN -> { + statusBuilder.pending = SendStatus.Pending() + } + it.status == GroupReceiptTable.STATUS_UNDELIVERED -> { + statusBuilder.sent = SendStatus.Sent( + sealedSender = it.isUnidentified + ) + } + it.status == GroupReceiptTable.STATUS_DELIVERED -> { + statusBuilder.delivered = SendStatus.Delivered( + sealedSender = it.isUnidentified + ) + } + it.status == GroupReceiptTable.STATUS_READ -> { + statusBuilder.read = SendStatus.Read( + sealedSender = it.isUnidentified + ) + } + it.status == GroupReceiptTable.STATUS_VIEWED -> { + statusBuilder.viewed = SendStatus.Viewed( + sealedSender = it.isUnidentified + ) + } + it.status == GroupReceiptTable.STATUS_SKIPPED -> { + statusBuilder.skipped = SendStatus.Skipped() + } + else -> { + statusBuilder.pending = SendStatus.Pending() + } + } + + statusBuilder.build() + } } private fun String?.parseNetworkFailures(): Set { @@ -1205,5 +1272,6 @@ data class ExtraMessageData( val mentionsById: Map>, val reactionsById: Map>, val attachmentsById: Map>, - val groupReceiptsById: Map> + val groupReceiptsById: Map>, + val isGroupThreadById: Map ) diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/local/LocalArchiver.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/local/LocalArchiver.kt index 7c73cd0ae3..e851f789b4 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/local/LocalArchiver.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/local/LocalArchiver.kt @@ -37,7 +37,7 @@ object LocalArchiver { /** * Export archive to the provided [snapshotFileSystem] and store new files in [filesFileSystem]. */ - fun export(snapshotFileSystem: SnapshotFileSystem, filesFileSystem: FilesFileSystem, stopwatch: Stopwatch): ArchiveResult { + fun export(snapshotFileSystem: SnapshotFileSystem, filesFileSystem: FilesFileSystem, stopwatch: Stopwatch, cancellationSignal: () -> Boolean = { false }): ArchiveResult { Log.i(TAG, "Starting export") var metadataStream: OutputStream? = null @@ -58,7 +58,11 @@ object LocalArchiver { val mediaNames: MutableSet = Collections.synchronizedSet(HashSet()) Log.i(TAG, "Starting frame export") - BackupRepository.localExport(mainStream, LocalExportProgressListener()) { attachment, source -> + BackupRepository.localExport(mainStream, LocalExportProgressListener(), cancellationSignal) { attachment, source -> + if (cancellationSignal()) { + return@localExport + } + val mediaName = MediaName.fromDigest(attachment.remoteDigest) mediaNames.add(mediaName) @@ -105,6 +109,10 @@ object LocalArchiver { filesStream?.close() } + if (cancellationSignal()) { + return ArchiveResult.failure(FailureCause.CANCELLED) + } + return ArchiveResult.success(Unit) } @@ -135,7 +143,7 @@ object LocalArchiver { get() = AttachmentCipherStreamUtil.getCiphertextLength(PaddingInputStream.getPaddedSize(size)) enum class FailureCause { - METADATA_STREAM, MAIN_STREAM, FILES_STREAM + METADATA_STREAM, MAIN_STREAM, FILES_STREAM, CANCELLED } private class LocalExportProgressListener : BackupRepository.ExportProgressListener { diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/AdHocCallArchiveProcessor.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/AdHocCallArchiveProcessor.kt index 2e55bbf9f5..72038ca9f4 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/AdHocCallArchiveProcessor.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/AdHocCallArchiveProcessor.kt @@ -6,6 +6,7 @@ package org.thoughtcrime.securesms.backup.v2.processor import org.signal.core.util.logging.Log +import org.thoughtcrime.securesms.backup.v2.ExportState import org.thoughtcrime.securesms.backup.v2.ImportState import org.thoughtcrime.securesms.backup.v2.database.getAdhocCallsForBackup import org.thoughtcrime.securesms.backup.v2.importer.AdHodCallArchiveImporter @@ -21,10 +22,14 @@ object AdHocCallArchiveProcessor { val TAG = Log.tag(AdHocCallArchiveProcessor::class.java) - fun export(db: SignalDatabase, emitter: BackupFrameEmitter) { + fun export(db: SignalDatabase, exportState: ExportState, emitter: BackupFrameEmitter) { db.callTable.getAdhocCallsForBackup().use { reader -> for (callLog in reader) { - emitter.emit(Frame(adHocCall = callLog)) + if (exportState.recipientIds.contains(callLog.recipientId)) { + emitter.emit(Frame(adHocCall = callLog)) + } else { + Log.w(TAG, "Dropping adhoc call for non-exported recipient.") + } } } } diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/ChatItemArchiveProcessor.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/ChatItemArchiveProcessor.kt index 402660573f..c5040d8b72 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/ChatItemArchiveProcessor.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/ChatItemArchiveProcessor.kt @@ -15,6 +15,7 @@ import org.thoughtcrime.securesms.backup.v2.proto.ChatItem import org.thoughtcrime.securesms.backup.v2.proto.Frame import org.thoughtcrime.securesms.backup.v2.stream.BackupFrameEmitter import org.thoughtcrime.securesms.database.SignalDatabase +import org.thoughtcrime.securesms.recipients.RecipientId /** * Handles importing/exporting [ChatItem] frames for an archive. @@ -22,8 +23,8 @@ import org.thoughtcrime.securesms.database.SignalDatabase object ChatItemArchiveProcessor { val TAG = Log.tag(ChatItemArchiveProcessor::class.java) - fun export(db: SignalDatabase, exportState: ExportState, cancellationSignal: () -> Boolean, emitter: BackupFrameEmitter) { - db.messageTable.getMessagesForBackup(db, exportState.backupTime, exportState.mediaBackupEnabled).use { chatItems -> + fun export(db: SignalDatabase, exportState: ExportState, selfRecipientId: RecipientId, cancellationSignal: () -> Boolean, emitter: BackupFrameEmitter) { + db.messageTable.getMessagesForBackup(db, exportState.backupTime, exportState.mediaBackupEnabled, selfRecipientId, exportState).use { chatItems -> var count = 0 while (chatItems.hasNext()) { if (count % 1000 == 0 && cancellationSignal()) { diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/RecipientArchiveProcessor.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/RecipientArchiveProcessor.kt index ef8104cf91..7af5c16e4e 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/RecipientArchiveProcessor.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/RecipientArchiveProcessor.kt @@ -24,6 +24,7 @@ import org.thoughtcrime.securesms.backup.v2.stream.BackupFrameEmitter import org.thoughtcrime.securesms.database.SignalDatabase import org.thoughtcrime.securesms.keyvalue.SignalStore import org.thoughtcrime.securesms.recipients.Recipient +import org.thoughtcrime.securesms.recipients.RecipientId /** * Handles importing/exporting [ArchiveRecipient] frames for an archive. @@ -32,8 +33,7 @@ object RecipientArchiveProcessor { val TAG = Log.tag(RecipientArchiveProcessor::class.java) - fun export(db: SignalDatabase, signalStore: SignalStore, exportState: ExportState, emitter: BackupFrameEmitter) { - val selfId = db.recipientTable.getByAci(signalStore.accountValues.aci!!).get().toLong() + fun export(db: SignalDatabase, signalStore: SignalStore, exportState: ExportState, selfRecipientId: RecipientId, emitter: BackupFrameEmitter) { val releaseChannelId = signalStore.releaseChannelValues.releaseChannelRecipientId if (releaseChannelId != null) { exportState.recipientIds.add(releaseChannelId.toLong()) @@ -49,7 +49,7 @@ object RecipientArchiveProcessor { Log.w(TAG, "Missing release channel id on export!") } - db.recipientTable.getContactsForBackup(selfId).use { reader -> + db.recipientTable.getContactsForBackup(selfRecipientId.toLong()).use { reader -> for (recipient in reader) { if (recipient != null) { exportState.recipientIds.add(recipient.id) diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/util/ArchiveConverterExtensions.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/util/ArchiveConverterExtensions.kt index 2f5b4c2832..2f12694a62 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/util/ArchiveConverterExtensions.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/util/ArchiveConverterExtensions.kt @@ -123,8 +123,8 @@ fun FilePointer?.toLocalAttachment( fun DatabaseAttachment.toRemoteFilePointer(mediaArchiveEnabled: Boolean, contentTypeOverride: String? = null): FilePointer { val builder = FilePointer.Builder() builder.contentType = contentTypeOverride ?: this.contentType?.takeUnless { it.isBlank() } - builder.incrementalMac = this.incrementalDigest?.toByteString() - builder.incrementalMacChunkSize = this.incrementalMacChunkSize.takeIf { it > 0 } + builder.incrementalMac = this.incrementalDigest?.takeIf { it.isNotEmpty() && this.incrementalMacChunkSize > 0 }?.toByteString() + builder.incrementalMacChunkSize = this.incrementalMacChunkSize.takeIf { it > 0 && builder.incrementalMac != null } builder.fileName = this.fileName builder.width = this.width.takeIf { it > 0 } builder.height = this.height.takeIf { it > 0 } diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/util/ChatStyleConverter.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/util/ChatStyleConverter.kt index 7d7938022b..57dabe9ebb 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/util/ChatStyleConverter.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/util/ChatStyleConverter.kt @@ -5,6 +5,7 @@ package org.thoughtcrime.securesms.backup.v2.util +import org.signal.core.util.logging.Log import org.thoughtcrime.securesms.attachments.AttachmentId import org.thoughtcrime.securesms.backup.v2.ImportState import org.thoughtcrime.securesms.backup.v2.proto.ChatStyle @@ -22,6 +23,8 @@ import org.thoughtcrime.securesms.wallpaper.GradientChatWallpaper import org.thoughtcrime.securesms.wallpaper.SingleColorChatWallpaper import org.thoughtcrime.securesms.wallpaper.UriChatWallpaper +private val TAG = Log.tag(ChatStyleConverter::class) + /** * Contains a collection of methods to chat styles to and from their archive format. * These are in a file of their own just because they're rather long (with all of the various constants to map between) and used in multiple places. @@ -37,6 +40,10 @@ object ChatStyleConverter { return null } + if (chatColorId == ChatColors.Id.NotSet && chatWallpaper == null) { + return null + } + val chatStyleBuilder = ChatStyle.Builder() if (chatColors != null) { @@ -72,6 +79,16 @@ object ChatStyleConverter { chatStyleBuilder.dimWallpaperInDarkMode = chatWallpaper.dimLevelInDarkTheme > 0 } + if (!chatStyleBuilder.hasBubbleColorSet()) { + if (chatStyleBuilder.hasWallpaperSet()) { + Log.w(TAG, "Wallpaper is set but no bubble color. Defaulting to automatic.") + chatStyleBuilder.autoBubbleColor = ChatStyle.AutomaticBubbleColor() + } else { + Log.w(TAG, "After building the chat style, it's empty. Returning null.") + return null + } + } + return chatStyleBuilder.build() } } @@ -147,6 +164,7 @@ fun ChatColors.toRemote(): ChatStyle.BubbleColorPreset? { ChatColorsPalette.Bubbles.SEA -> return ChatStyle.BubbleColorPreset.GRADIENT_SEA ChatColorsPalette.Bubbles.TANGERINE -> return ChatStyle.BubbleColorPreset.GRADIENT_TANGERINE } + Log.w(TAG, "No matching remote bubble preset for ${this.serialize()}") return null } @@ -193,7 +211,7 @@ fun ChatStyle.parseChatWallpaper(wallpaperAttachmentId: AttachmentId?): ChatWall } } -private fun Int.toRemoteWallpaperPreset(): ChatStyle.WallpaperPreset { +private fun Int.toRemoteWallpaperPreset(): ChatStyle.WallpaperPreset? { return when (this) { SingleColorChatWallpaper.BLUSH.color -> ChatStyle.WallpaperPreset.SOLID_BLUSH SingleColorChatWallpaper.COPPER.color -> ChatStyle.WallpaperPreset.SOLID_COPPER @@ -207,7 +225,10 @@ private fun Int.toRemoteWallpaperPreset(): ChatStyle.WallpaperPreset { SingleColorChatWallpaper.PINK.color -> ChatStyle.WallpaperPreset.SOLID_PINK SingleColorChatWallpaper.EGGPLANT.color -> ChatStyle.WallpaperPreset.SOLID_EGGPLANT SingleColorChatWallpaper.SILVER.color -> ChatStyle.WallpaperPreset.SOLID_SILVER - else -> ChatStyle.WallpaperPreset.UNKNOWN_WALLPAPER_PRESET + else -> { + Log.w(TAG, "No matching remote wallpaper preset for $this") + null + } } } @@ -232,3 +253,11 @@ private fun Wallpaper.File.toFilePointer(db: SignalDatabase): FilePointer? { val attachment = db.attachmentTable.getAttachment(attachmentId) return attachment?.toRemoteFilePointer(mediaArchiveEnabled = true) } + +private fun ChatStyle.Builder.hasBubbleColorSet(): Boolean { + return this.customColorId != null || this.autoBubbleColor != null || this.bubbleColorPreset != null +} + +private fun ChatStyle.Builder.hasWallpaperSet(): Boolean { + return this.wallpaperPreset != null || this.wallpaperPhoto != null +} diff --git a/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/backup/InternalBackupPlaygroundViewModel.kt b/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/backup/InternalBackupPlaygroundViewModel.kt index 6c3d7766a0..4f098b3efe 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/backup/InternalBackupPlaygroundViewModel.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/backup/InternalBackupPlaygroundViewModel.kt @@ -539,6 +539,7 @@ class InternalBackupPlaygroundViewModel : ViewModel() { AppDependencies.jobManager.cancelAllInQueue("ArchiveAttachmentJobs_1") AppDependencies.jobManager.cancelAllInQueue("ArchiveThumbnailUploadJob") AppDependencies.jobManager.cancelAllInQueue("BackupRestoreJob") + AppDependencies.jobManager.cancelAllInQueue("__LOCAL_BACKUP__") } fun fetchRemoteBackupAndWritePlaintext(outputStream: OutputStream?) { diff --git a/app/src/main/java/org/thoughtcrime/securesms/conversationlist/ConversationListFragment.java b/app/src/main/java/org/thoughtcrime/securesms/conversationlist/ConversationListFragment.java index f6b59c9813..55378ae53c 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/conversationlist/ConversationListFragment.java +++ b/app/src/main/java/org/thoughtcrime/securesms/conversationlist/ConversationListFragment.java @@ -94,6 +94,7 @@ import org.thoughtcrime.securesms.MainNavigator; import org.thoughtcrime.securesms.MuteDialog; import org.thoughtcrime.securesms.NewConversationActivity; import org.thoughtcrime.securesms.R; +import org.thoughtcrime.securesms.backup.v2.ArchiveValidator; import org.thoughtcrime.securesms.backup.v2.BackupRepository; import org.thoughtcrime.securesms.backup.v2.ui.BackupAlert; import org.thoughtcrime.securesms.backup.v2.ui.BackupAlertBottomSheet; @@ -162,6 +163,7 @@ import org.thoughtcrime.securesms.notifications.MarkReadReceiver; import org.thoughtcrime.securesms.notifications.profiles.NotificationProfile; import org.thoughtcrime.securesms.permissions.Permissions; import org.thoughtcrime.securesms.profiles.manage.UsernameEditFragment; +import org.thoughtcrime.securesms.providers.BlobProvider; import org.thoughtcrime.securesms.ratelimit.RecaptchaProofBottomSheetFragment; import org.thoughtcrime.securesms.recipients.Recipient; import org.thoughtcrime.securesms.recipients.RecipientId; @@ -191,6 +193,10 @@ import org.thoughtcrime.securesms.util.views.Stub; import org.thoughtcrime.securesms.wallpaper.ChatWallpaper; import org.whispersystems.signalservice.api.websocket.WebSocketConnectionState; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; import java.lang.ref.WeakReference; import java.util.ArrayList; import java.util.Collection; diff --git a/app/src/main/java/org/thoughtcrime/securesms/jobs/BackupMessagesJob.kt b/app/src/main/java/org/thoughtcrime/securesms/jobs/BackupMessagesJob.kt index 8cc3a27e08..6e5bae07b3 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/jobs/BackupMessagesJob.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/jobs/BackupMessagesJob.kt @@ -8,6 +8,7 @@ package org.thoughtcrime.securesms.jobs import org.signal.core.util.Stopwatch import org.signal.core.util.logging.Log import org.thoughtcrime.securesms.backup.ArchiveUploadProgress +import org.thoughtcrime.securesms.backup.v2.ArchiveValidator import org.thoughtcrime.securesms.backup.v2.BackupRepository import org.thoughtcrime.securesms.database.SignalDatabase import org.thoughtcrime.securesms.dependencies.AppDependencies @@ -82,9 +83,26 @@ class BackupMessagesJob private constructor(parameters: Parameters) : Job(parame val tempBackupFile = BlobProvider.getInstance().forNonAutoEncryptingSingleSessionOnDisk(AppDependencies.application) val outputStream = FileOutputStream(tempBackupFile) - BackupRepository.export(outputStream = outputStream, append = { tempBackupFile.appendBytes(it) }, plaintext = false, cancellationSignal = { this.isCanceled }) + val backupKey = SignalStore.backup.messageBackupKey + BackupRepository.export(outputStream = outputStream, messageBackupKey = backupKey, append = { tempBackupFile.appendBytes(it) }, plaintext = false, cancellationSignal = { this.isCanceled }) stopwatch.split("export") + when (val result = ArchiveValidator.validate(tempBackupFile, backupKey)) { + ArchiveValidator.ValidationResult.Success -> { + Log.d(TAG, "Successfully passed validation.") + } + is ArchiveValidator.ValidationResult.ReadError -> { + Log.w(TAG, "Failed to read the file during validation!", result.exception) + return Result.retry(defaultBackoff()) + } + is ArchiveValidator.ValidationResult.ValidationError -> { + // TODO [backup] UX + Log.w(TAG, "The backup file fails validation! Message: " + result.exception.message) + return Result.failure() + } + } + stopwatch.split("validate") + if (isCanceled) { return Result.failure() } diff --git a/app/src/main/java/org/thoughtcrime/securesms/jobs/LocalArchiveJob.kt b/app/src/main/java/org/thoughtcrime/securesms/jobs/LocalArchiveJob.kt index 4e56b0bed8..84cb3e3bee 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/jobs/LocalArchiveJob.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/jobs/LocalArchiveJob.kt @@ -96,7 +96,7 @@ class LocalArchiveJob internal constructor(parameters: Parameters) : Job(paramet try { try { - val result = LocalArchiver.export(snapshotFileSystem, archiveFileSystem.filesFileSystem, stopwatch) + val result = LocalArchiver.export(snapshotFileSystem, archiveFileSystem.filesFileSystem, stopwatch, cancellationSignal = { isCanceled }) Log.i(TAG, "Archive finished with result: $result") if (result !is org.signal.core.util.Result.Success) { return Result.failure() diff --git a/app/src/main/java/org/thoughtcrime/securesms/keyvalue/BackupValues.kt b/app/src/main/java/org/thoughtcrime/securesms/keyvalue/BackupValues.kt index d8748e404f..9295a148be 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/keyvalue/BackupValues.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/keyvalue/BackupValues.kt @@ -203,70 +203,6 @@ class BackupValues(store: KeyValueStore) : SignalStoreValues(store) { /** Store that lets you interact with media ZK credentials. */ val mediaCredentials = CredentialStore(KEY_MEDIA_CREDENTIALS, KEY_MEDIA_CDN_READ_CREDENTIALS, KEY_MEDIA_CDN_READ_CREDENTIALS_TIMESTAMP) - inner class CredentialStore(val authKey: String, val cdnKey: String, val cdnTimestampKey: String) { - /** - * Retrieves the stored media credentials, mapped by the day they're valid. The day is represented as - * the unix time (in seconds) of the start of the day. Wrapped in a [ArchiveServiceCredentials] - * type to make it easier to use. See [ArchiveServiceCredentials.getForCurrentTime]. - */ - val byDay: ArchiveServiceCredentials - get() { - val serialized = store.getString(authKey, null) ?: return ArchiveServiceCredentials() - - return try { - val map = JsonUtil.fromJson(serialized, SerializedCredentials::class.java).credentialsByDay - ArchiveServiceCredentials(map) - } catch (e: IOException) { - Log.w(TAG, "Invalid JSON! Clearing.", e) - putString(authKey, null) - ArchiveServiceCredentials() - } - } - - /** Adds the given credentials to the existing list of stored credentials. */ - fun add(credentials: List) { - val current: MutableMap = byDay.toMutableMap() - current.putAll(credentials.associateBy { it.redemptionTime }) - putString(authKey, JsonUtil.toJson(SerializedCredentials(current))) - } - - /** Trims out any credentials that are for days older than the given timestamp. */ - fun clearOlderThan(startOfDayInSeconds: Long) { - val current: MutableMap = byDay.toMutableMap() - val updated = current.filterKeys { it < startOfDayInSeconds } - putString(authKey, JsonUtil.toJson(SerializedCredentials(updated))) - } - - /** Clears all credentials. */ - fun clearAll() { - putString(authKey, null) - cdnReadCredentials = null - } - - /** Credentials to read from the CDN. */ - var cdnReadCredentials: GetArchiveCdnCredentialsResponse? - get() { - val cacheAge = System.currentTimeMillis() - getLong(cdnTimestampKey, 0) - val cached = getString(cdnKey, null) - - return if (cached != null && (cacheAge > 0 && cacheAge < cachedCdnCredentialsExpiresIn.inWholeMilliseconds)) { - try { - JsonUtil.fromJson(cached, GetArchiveCdnCredentialsResponse::class.java) - } catch (e: IOException) { - Log.w(TAG, "Invalid JSON! Clearing.", e) - putString(cdnKey, null) - null - } - } else { - null - } - } - set(value) { - putString(cdnKey, value?.let { JsonUtil.toJson(it) }) - putLong(cdnTimestampKey, System.currentTimeMillis()) - } - } - fun markMessageBackupFailure() { store.beginWrite() .putBoolean(KEY_BACKUP_FAIL, true) @@ -338,4 +274,69 @@ class BackupValues(store: KeyValueStore) : SignalStoreValues(store) { return this[startOfDayInSeconds] } } + + inner class CredentialStore(private val authKey: String, private val cdnKey: String, private val cdnTimestampKey: String) { + /** + * Retrieves the stored media credentials, mapped by the day they're valid. The day is represented as + * the unix time (in seconds) of the start of the day. Wrapped in a [ArchiveServiceCredentials] + * type to make it easier to use. See [ArchiveServiceCredentials.getForCurrentTime]. + */ + val byDay: ArchiveServiceCredentials + get() { + val serialized = store.getString(authKey, null) ?: return ArchiveServiceCredentials() + + return try { + val map = JsonUtil.fromJson(serialized, SerializedCredentials::class.java).credentialsByDay + ArchiveServiceCredentials(map) + } catch (e: IOException) { + Log.w(TAG, "Invalid JSON! Clearing.", e) + putString(authKey, null) + ArchiveServiceCredentials() + } + } + + /** Adds the given credentials to the existing list of stored credentials. */ + fun add(credentials: List) { + val current: MutableMap = byDay.toMutableMap() + current.putAll(credentials.associateBy { it.redemptionTime }) + putString(authKey, JsonUtil.toJson(SerializedCredentials(current))) + } + + /** Trims out any credentials that are for days older than the given timestamp. */ + fun clearOlderThan(startOfDayInSeconds: Long) { + val current: MutableMap = byDay.toMutableMap() + val updated = current.filterKeys { it < startOfDayInSeconds } + putString(authKey, JsonUtil.toJson(SerializedCredentials(updated))) + } + + /** Clears all credentials. */ + fun clearAll() { + putString(authKey, null) + putString(cdnKey, null) + putLong(cdnTimestampKey, 0) + } + + /** Credentials to read from the CDN. */ + var cdnReadCredentials: GetArchiveCdnCredentialsResponse? + get() { + val cacheAge = System.currentTimeMillis() - getLong(cdnTimestampKey, 0) + val cached = getString(cdnKey, null) + + return if (cached != null && (cacheAge > 0 && cacheAge < cachedCdnCredentialsExpiresIn.inWholeMilliseconds)) { + try { + JsonUtil.fromJson(cached, GetArchiveCdnCredentialsResponse::class.java) + } catch (e: IOException) { + Log.w(TAG, "Invalid JSON! Clearing.", e) + putString(cdnKey, null) + null + } + } else { + null + } + } + set(value) { + putString(cdnKey, value?.let { JsonUtil.toJson(it) }) + putLong(cdnTimestampKey, System.currentTimeMillis()) + } + } } diff --git a/app/src/main/java/org/thoughtcrime/securesms/linkdevice/LinkDeviceRepository.kt b/app/src/main/java/org/thoughtcrime/securesms/linkdevice/LinkDeviceRepository.kt index fdd6930c7d..a833d1d4d4 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/linkdevice/LinkDeviceRepository.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/linkdevice/LinkDeviceRepository.kt @@ -7,6 +7,7 @@ import org.signal.core.util.isNotNullOrBlank import org.signal.core.util.logging.Log import org.signal.core.util.logging.logW import org.signal.libsignal.protocol.ecc.Curve +import org.thoughtcrime.securesms.backup.v2.ArchiveValidator import org.thoughtcrime.securesms.backup.v2.BackupRepository import org.thoughtcrime.securesms.crypto.ProfileKeyUtil import org.thoughtcrime.securesms.dependencies.AppDependencies @@ -239,6 +240,21 @@ object LinkDeviceRepository { } stopwatch.split("create-backup") + when (val result = ArchiveValidator.validate(tempBackupFile, ephemeralMessageBackupKey)) { + ArchiveValidator.ValidationResult.Success -> { + Log.d(TAG, "Successfully passed validation.") + } + is ArchiveValidator.ValidationResult.ReadError -> { + Log.w(TAG, "Failed to read the file during validation!", result.exception) + return LinkUploadArchiveResult.BackupCreationFailure(result.exception) + } + is ArchiveValidator.ValidationResult.ValidationError -> { + Log.w(TAG, "The backup file fails validation!", result.exception) + return LinkUploadArchiveResult.BackupCreationFailure(result.exception) + } + } + stopwatch.split("validate-backup") + val uploadForm = when (val result = SignalNetwork.attachments.getAttachmentV4UploadForm()) { is NetworkResult.Success -> result.result is NetworkResult.ApplicationError -> throw result.throwable diff --git a/libsignal-service/src/main/java/org/whispersystems/signalservice/api/archive/ArchiveApi.kt b/libsignal-service/src/main/java/org/whispersystems/signalservice/api/archive/ArchiveApi.kt index 7b02697d19..56225415a6 100644 --- a/libsignal-service/src/main/java/org/whispersystems/signalservice/api/archive/ArchiveApi.kt +++ b/libsignal-service/src/main/java/org/whispersystems/signalservice/api/archive/ArchiveApi.kt @@ -89,7 +89,6 @@ class ArchiveApi(private val pushServiceSocket: PushServiceSocket) { * - 204: Success * - 400: Invalid credential * - 429: Rate-limited - * */ fun triggerBackupIdReservation(messageBackupKey: MessageBackupKey, mediaRootBackupKey: MediaRootBackupKey, aci: ACI): NetworkResult { return NetworkResult.fromFetch { diff --git a/libsignal-service/src/main/java/org/whispersystems/signalservice/api/backup/MessageBackupKey.kt b/libsignal-service/src/main/java/org/whispersystems/signalservice/api/backup/MessageBackupKey.kt index e9c4980d63..617725ff06 100644 --- a/libsignal-service/src/main/java/org/whispersystems/signalservice/api/backup/MessageBackupKey.kt +++ b/libsignal-service/src/main/java/org/whispersystems/signalservice/api/backup/MessageBackupKey.kt @@ -19,6 +19,13 @@ class MessageBackupKey(override val value: ByteArray) : BackupKey { require(value.size == 32) { "Backup key must be 32 bytes!" } } + /** + * The private key used to generate anonymous credentials when interacting with the backup service. + */ + override fun deriveAnonymousCredentialPrivateKey(aci: ACI): ECPrivateKey { + return LibSignalBackupKey(value).deriveEcKey(aci.libSignalAci) + } + /** * The cryptographic material used to encrypt a backup. */ @@ -34,17 +41,10 @@ class MessageBackupKey(override val value: ByteArray) : BackupKey { ) } - /** - * The private key used to generate anonymous credentials when interacting with the backup service. - */ - override fun deriveAnonymousCredentialPrivateKey(aci: ACI): ECPrivateKey { - return LibSignalBackupKey(value).deriveEcKey(aci.libSignalAci) - } - /** * Identifies a the location of a user's backup. */ - private fun deriveBackupId(aci: ACI): BackupId { + fun deriveBackupId(aci: ACI): BackupId { return BackupId( LibSignalBackupKey(value).deriveBackupId(aci.libSignalAci) )