From 497cec4c1797fa5a69451a9a742a468d11969eb0 Mon Sep 17 00:00:00 2001 From: Greyson Parrelli Date: Thu, 10 Oct 2024 10:55:27 -0400 Subject: [PATCH] Add some local timings to backup export. --- .../database/MessageTableArchiveExtensions.kt | 9 +- .../v2/exporters/ChatItemArchiveExporter.kt | 109 +++++++++++++----- .../InternalBackupPlaygroundViewModel.kt | 2 - .../securesms/database/AttachmentTable.kt | 7 +- .../securesms/database/GroupReceiptTable.kt | 2 +- 5 files changed, 91 insertions(+), 38 deletions(-) diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/MessageTableArchiveExtensions.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/MessageTableArchiveExtensions.kt index 4e0c5162e6..98a77f044c 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/MessageTableArchiveExtensions.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/MessageTableArchiveExtensions.kt @@ -11,18 +11,15 @@ import org.thoughtcrime.securesms.backup.v2.ImportState import org.thoughtcrime.securesms.backup.v2.exporters.ChatItemArchiveExporter import org.thoughtcrime.securesms.backup.v2.importer.ChatItemArchiveImporter import org.thoughtcrime.securesms.database.MessageTable -import org.thoughtcrime.securesms.database.MessageTypes import org.thoughtcrime.securesms.database.SignalDatabase import java.util.concurrent.TimeUnit -private const val COLUMN_BASE_TYPE = "base_type" - fun MessageTable.getMessagesForBackup(db: SignalDatabase, backupTime: Long, mediaBackupEnabled: Boolean): ChatItemArchiveExporter { // We create a temporary index on date_received to drastically speed up perf here. // Remember that we're working on a temporary snapshot of the database, so we can create an index and not worry about cleaning it up. val dateReceivedIndex = "message_date_received" - writableDatabase.execSQL("CREATE INDEX $dateReceivedIndex ON ${MessageTable.TABLE_NAME} (${MessageTable.DATE_RECEIVED} DESC)") + writableDatabase.execSQL("CREATE INDEX $dateReceivedIndex ON ${MessageTable.TABLE_NAME} (${MessageTable.DATE_RECEIVED} ASC)") val cursor = readableDatabase .select( @@ -57,7 +54,7 @@ fun MessageTable.getMessagesForBackup(db: SignalDatabase, backupTime: Long, medi MessageTable.READ, MessageTable.NETWORK_FAILURES, MessageTable.MISMATCHED_IDENTITIES, - "${MessageTable.TYPE} & ${MessageTypes.BASE_TYPE_MASK} AS $COLUMN_BASE_TYPE", + MessageTable.TYPE, MessageTable.MESSAGE_EXTRAS, MessageTable.VIEW_ONCE ) @@ -79,7 +76,7 @@ fun MessageTable.getMessagesForBackup(db: SignalDatabase, backupTime: Long, medi } fun MessageTable.createChatItemInserter(importState: ImportState): ChatItemArchiveImporter { - return ChatItemArchiveImporter(writableDatabase, importState, 100) + return ChatItemArchiveImporter(writableDatabase, importState, 500) } fun MessageTable.clearAllDataForBackupRestore() { diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/exporters/ChatItemArchiveExporter.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/exporters/ChatItemArchiveExporter.kt index 2e8664f439..f1aebdf93c 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/exporters/ChatItemArchiveExporter.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/exporters/ChatItemArchiveExporter.kt @@ -10,6 +10,7 @@ import okio.ByteString.Companion.toByteString import org.json.JSONArray import org.json.JSONException import org.signal.core.util.Base64 +import org.signal.core.util.EventTimer import org.signal.core.util.Hex import org.signal.core.util.logging.Log import org.signal.core.util.nullIfEmpty @@ -87,7 +88,6 @@ import org.thoughtcrime.securesms.backup.v2.proto.BodyRange as BackupBodyRange import org.thoughtcrime.securesms.backup.v2.proto.GiftBadge as BackupGiftBadge private val TAG = Log.tag(ChatItemArchiveExporter::class.java) -private const val COLUMN_BASE_TYPE = "base_type" /** * An iterator for chat items with a clever performance twist: rather than do the extra queries one at a time (for reactions, @@ -103,6 +103,8 @@ class ChatItemArchiveExporter( private val mediaArchiveEnabled: Boolean ) : Iterator, Closeable { + private val eventTimer = EventTimer() + /** * A queue of already-parsed ChatItems. Processing in batches means that we read ahead in the cursor and put * the pending items here. @@ -120,7 +122,7 @@ class ChatItemArchiveExporter( return buffer.remove() } - val records: LinkedHashMap = linkedMapOf() + val records: LinkedHashMap = LinkedHashMap(batchSize) for (i in 0 until batchSize) { if (cursor.moveToNext()) { @@ -130,14 +132,12 @@ class ChatItemArchiveExporter( break } } + eventTimer.emit("messages") - val reactionsById: Map> = db.reactionTable.getReactionsForMessages(records.keys).map { entry -> entry.key to entry.value.sortedBy { it.dateReceived } }.toMap() - val mentionsById: Map> = db.mentionTable.getMentionsForMessages(records.keys) - val attachmentsById: Map> = db.attachmentTable.getAttachmentsForMessages(records.keys) - val groupReceiptsById: Map> = db.groupReceiptTable.getGroupReceiptInfoForMessages(records.keys) + val extraData = fetchExtraMessageData(db, records.keys) for ((id, record) in records) { - val builder = record.toBasicChatItemBuilder(groupReceiptsById[id]) + val builder = record.toBasicChatItemBuilder(extraData.groupReceiptsById[id]) when { record.remoteDeleted -> { @@ -243,31 +243,31 @@ class ChatItemArchiveExporter( } !record.sharedContacts.isNullOrEmpty() -> { - builder.contactMessage = record.toRemoteContactMessage(mediaArchiveEnabled = mediaArchiveEnabled, reactionRecords = reactionsById[id], attachments = attachmentsById[id]) + builder.contactMessage = record.toRemoteContactMessage(mediaArchiveEnabled = mediaArchiveEnabled, reactionRecords = extraData.reactionsById[id], attachments = extraData.attachmentsById[id]) } record.viewOnce -> { - builder.viewOnceMessage = record.toRemoteViewOnceMessage(mediaArchiveEnabled = mediaArchiveEnabled, reactionRecords = reactionsById[id], attachments = attachmentsById[id]) + builder.viewOnceMessage = record.toRemoteViewOnceMessage(mediaArchiveEnabled = mediaArchiveEnabled, reactionRecords = extraData.reactionsById[id], attachments = extraData.attachmentsById[id]) } else -> { - if (record.body == null && !attachmentsById.containsKey(record.id)) { + if (record.body == null && !extraData.attachmentsById.containsKey(record.id)) { Log.w(TAG, "Record with ID ${record.id} missing a body and doesn't have attachments. Skipping.") continue } - val attachments = attachmentsById[record.id] + val attachments = extraData.attachmentsById[record.id] val sticker = attachments?.firstOrNull { dbAttachment -> dbAttachment.isSticker } if (sticker?.stickerLocator != null) { - builder.stickerMessage = sticker.toRemoteStickerMessage(mediaArchiveEnabled = mediaArchiveEnabled, reactions = reactionsById[id]) + builder.stickerMessage = sticker.toRemoteStickerMessage(mediaArchiveEnabled = mediaArchiveEnabled, reactions = extraData.reactionsById[id]) } else { builder.standardMessage = record.toRemoteStandardMessage( db = db, mediaArchiveEnabled = mediaArchiveEnabled, - reactionRecords = reactionsById[id], - mentions = mentionsById[id], - attachments = attachmentsById[record.id] + reactionRecords = extraData.reactionsById[id], + mentions = extraData.mentionsById[id], + attachments = extraData.attachmentsById[record.id] ) } } @@ -283,7 +283,7 @@ class ChatItemArchiveExporter( var previousEdits = revisionMap[record.latestRevisionId] if (previousEdits == null) { previousEdits = ArrayList() - revisionMap[record.latestRevisionId] = previousEdits + revisionMap[record.latestRevisionId!!] = previousEdits } previousEdits += builder.build() } @@ -298,6 +298,52 @@ class ChatItemArchiveExporter( override fun close() { cursor.close() + Log.w(TAG, "[ChatItemArchiveExporter] ${eventTimer.stop().summary}") + } + + private fun fetchExtraMessageData(db: SignalDatabase, messageIds: Set): ExtraMessageData { + // TODO [backup] This seems to be a wash +// val executor = SignalExecutors.BOUNDED +// +// val mentionsFuture = executor.submitTyped { +// db.mentionTable.getMentionsForMessages(messageIds) +// } +// +// val reactionsFuture = executor.submitTyped { +// db.reactionTable.getReactionsForMessages(messageIds) +// } +// +// val attachmentsFuture = executor.submitTyped { +// db.attachmentTable.getAttachmentsForMessages(messageIds) +// } +// +// val groupReceiptsFuture = executor.submitTyped { +// db.groupReceiptTable.getGroupReceiptInfoForMessages(messageIds) +// } +// +// val mentionsResult = mentionsFuture.get() +// val reactionsResult = reactionsFuture.get() +// val attachmentsResult = attachmentsFuture.get() +// val groupReceiptsResult = groupReceiptsFuture.get() + + val mentionsResult = db.mentionTable.getMentionsForMessages(messageIds) + eventTimer.emit("mentions") + + val reactionsResult = db.reactionTable.getReactionsForMessages(messageIds) + eventTimer.emit("reactions") + + val attachmentsResult = db.attachmentTable.getAttachmentsForMessages(messageIds) + eventTimer.emit("attachments") + + val groupReceiptsResult = db.groupReceiptTable.getGroupReceiptInfoForMessages(messageIds) + eventTimer.emit("receipts") + + return ExtraMessageData( + mentionsById = mentionsResult, + reactionsById = reactionsResult, + attachmentsById = attachmentsResult, + groupReceiptsById = groupReceiptsResult + ) } } @@ -334,11 +380,8 @@ private fun BackupMessageRecord.toBasicChatItemBuilder(groupReceipts: List return try { - val decoded: ByteArray = Base64.decode(this.body) + val decoded: ByteArray = Base64.decode(body) val context = DecryptedGroupV2Context.ADAPTER.decode(decoded) ChatUpdateMessage( groupChange = GroupsV2UpdateMessageConverter.translateDecryptedChange(selfIds = SignalStore.account.getServiceIds(), context) @@ -651,14 +694,13 @@ private fun Contact.PostalAddress.Type.toRemote(): ContactAttachment.PostalAddre } private fun BackupMessageRecord.toRemoteStandardMessage(db: SignalDatabase, mediaArchiveEnabled: Boolean, reactionRecords: List?, mentions: List?, attachments: List?): StandardMessage { - val text = if (body == null) { - null - } else { + val text = body?.let { Text( - body = this.body, + body = it, bodyRanges = (this.bodyRanges?.toRemoteBodyRanges() ?: emptyList()) + (mentions?.toRemoteBodyRanges(db) ?: emptyList()) ) } + val linkPreviews = this.toRemoteLinkPreviews(attachments) val linkPreviewAttachments = linkPreviews.mapNotNull { it.thumbnail.orElse(null) }.toSet() val quotedAttachments = attachments?.filter { it.quote } ?: emptyList() @@ -1062,6 +1104,10 @@ private fun String.e164ToLong(): Long? { return fixed.toLongOrNull() } +// private fun ExecutorService.submitTyped(callable: Callable): Future { +// return this.submit(callable) +// } + private fun Cursor.toBackupMessageRecord(): BackupMessageRecord { return BackupMessageRecord( id = this.requireLong(MessageTable.ID), @@ -1095,7 +1141,7 @@ private fun Cursor.toBackupMessageRecord(): BackupMessageRecord { receiptTimestamp = this.requireLong(MessageTable.RECEIPT_TIMESTAMP), networkFailureRecipientIds = this.requireString(MessageTable.NETWORK_FAILURES).parseNetworkFailures(), identityMismatchRecipientIds = this.requireString(MessageTable.MISMATCHED_IDENTITIES).parseIdentityMismatches(), - baseType = this.requireLong(COLUMN_BASE_TYPE), + baseType = this.requireLong(MessageTable.TYPE) and MessageTypes.BASE_TYPE_MASK, messageExtras = this.requireBlob(MessageTable.MESSAGE_EXTRAS).parseMessageExtras(), viewOnce = this.requireBoolean(MessageTable.VIEW_ONCE) ) @@ -1137,3 +1183,10 @@ private class BackupMessageRecord( val messageExtras: MessageExtras?, val viewOnce: Boolean ) + +data class ExtraMessageData( + val mentionsById: Map>, + val reactionsById: Map>, + val attachmentsById: Map>, + val groupReceiptsById: Map> +) diff --git a/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/backup/InternalBackupPlaygroundViewModel.kt b/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/backup/InternalBackupPlaygroundViewModel.kt index 59de6bf516..df69c7ea44 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/backup/InternalBackupPlaygroundViewModel.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/backup/InternalBackupPlaygroundViewModel.kt @@ -214,8 +214,6 @@ class InternalBackupPlaygroundViewModel : ViewModel() { fun wipeAllDataAndRestoreFromRemote() { SignalExecutors.BOUNDED_IO.execute { - SignalDatabase.threads.deleteAllConversations() - AppDependencies.messageNotifier.updateNotification(AppDependencies.application) restoreFromRemote() } } diff --git a/app/src/main/java/org/thoughtcrime/securesms/database/AttachmentTable.kt b/app/src/main/java/org/thoughtcrime/securesms/database/AttachmentTable.kt index 1e93d42861..da16cd706e 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/database/AttachmentTable.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/database/AttachmentTable.kt @@ -598,7 +598,12 @@ class AttachmentTable( val iv = cursor.requireBlob(REMOTE_IV) ?: Util.getSecretBytes(16) val digest = run { val fileInfo = getDataFileInfo(attachmentId)!! - calculateDigest(fileInfo, key, iv) + try { + calculateDigest(fileInfo, key, iv) + } catch (e: FileNotFoundException) { + Log.w(TAG, "[createKeyIvDigestForAttachmentsThatNeedArchiveUpload][$attachmentId] Could not find file ${fileInfo.file}. Delete all later?") + return@forEach + } } writableDatabase.update(TABLE_NAME) diff --git a/app/src/main/java/org/thoughtcrime/securesms/database/GroupReceiptTable.kt b/app/src/main/java/org/thoughtcrime/securesms/database/GroupReceiptTable.kt index 4603c6b245..6c0e72c823 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/database/GroupReceiptTable.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/database/GroupReceiptTable.kt @@ -133,7 +133,7 @@ class GroupReceiptTable(context: Context?, databaseHelper: SignalDatabase?) : Da .readToList { it.toGroupReceiptInfo() } } - fun getGroupReceiptInfoForMessages(ids: Set): Map> { + fun getGroupReceiptInfoForMessages(ids: Collection): Map> { if (ids.isEmpty()) { return emptyMap() }