diff --git a/app/src/androidTest/java/org/thoughtcrime/securesms/backup/v2/ArchiveImportExportTests.kt b/app/src/androidTest/java/org/thoughtcrime/securesms/backup/v2/ArchiveImportExportTests.kt index 6622510668..929cab9e29 100644 --- a/app/src/androidTest/java/org/thoughtcrime/securesms/backup/v2/ArchiveImportExportTests.kt +++ b/app/src/androidTest/java/org/thoughtcrime/securesms/backup/v2/ArchiveImportExportTests.kt @@ -14,7 +14,6 @@ import org.junit.Before import org.junit.Test import org.junit.runner.RunWith import org.signal.core.util.Base64 -import org.signal.core.util.SqlUtil import org.signal.core.util.logging.Log import org.signal.core.util.readFully import org.signal.libsignal.messagebackup.ComparableBackup @@ -22,12 +21,9 @@ import org.signal.libsignal.messagebackup.MessageBackup import org.signal.libsignal.zkgroup.profiles.ProfileKey import org.thoughtcrime.securesms.backup.v2.proto.Frame import org.thoughtcrime.securesms.backup.v2.stream.PlainTextBackupReader -import org.thoughtcrime.securesms.database.DistributionListTables import org.thoughtcrime.securesms.database.KeyValueDatabase -import org.thoughtcrime.securesms.database.SignalDatabase import org.thoughtcrime.securesms.dependencies.AppDependencies import org.thoughtcrime.securesms.keyvalue.SignalStore -import org.thoughtcrime.securesms.recipients.RecipientId import org.whispersystems.signalservice.api.kbs.MasterKey import org.whispersystems.signalservice.api.push.ServiceId import java.io.ByteArrayInputStream @@ -263,21 +259,7 @@ class ArchiveImportExportTests { } private fun resetAllData() { - // Need to delete these first to prevent foreign key crash - SignalDatabase.rawDatabase.execSQL("DELETE FROM ${DistributionListTables.ListTable.TABLE_NAME}") - SignalDatabase.rawDatabase.execSQL("DELETE FROM ${DistributionListTables.MembershipTable.TABLE_NAME}") - - SqlUtil.getAllTables(SignalDatabase.rawDatabase) - .filterNot { it.contains("sqlite") || it.contains("fts") || it.startsWith("emoji_search_") } // If we delete these we'll corrupt the DB - .sorted() - .forEach { table -> - SignalDatabase.rawDatabase.execSQL("DELETE FROM $table") - SqlUtil.resetAutoIncrementValue(SignalDatabase.rawDatabase, table) - } - - AppDependencies.recipientCache.clear() - AppDependencies.recipientCache.clearSelf() - RecipientId.clearCache() + // All the main database stuff is reset as a normal part of importing KeyValueDatabase.getInstance(AppDependencies.application).clear() SignalStore.resetCache() diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/BackupRepository.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/BackupRepository.kt index 5cc7aa476d..0d64251a1d 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/BackupRepository.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/BackupRepository.kt @@ -11,9 +11,15 @@ import kotlinx.coroutines.withContext import org.greenrobot.eventbus.EventBus import org.signal.core.util.Base64 import org.signal.core.util.EventTimer +import org.signal.core.util.Stopwatch import org.signal.core.util.concurrent.LimitedWorker import org.signal.core.util.concurrent.SignalExecutors +import org.signal.core.util.forceForeignKeyConstraintsEnabled import org.signal.core.util.fullWalCheckpoint +import org.signal.core.util.getAllIndexDefinitions +import org.signal.core.util.getAllTableDefinitions +import org.signal.core.util.getAllTriggerDefinitions +import org.signal.core.util.getForeignKeyViolations import org.signal.core.util.logging.Log import org.signal.core.util.stream.NonClosingOutputStream import org.signal.core.util.withinTransaction @@ -27,8 +33,6 @@ import org.thoughtcrime.securesms.attachments.Attachment import org.thoughtcrime.securesms.attachments.AttachmentId import org.thoughtcrime.securesms.attachments.Cdn import org.thoughtcrime.securesms.attachments.DatabaseAttachment -import org.thoughtcrime.securesms.backup.v2.database.clearAllDataForBackup -import org.thoughtcrime.securesms.backup.v2.database.clearAllDataForBackupRestore import org.thoughtcrime.securesms.backup.v2.importer.ChatItemArchiveImporter import org.thoughtcrime.securesms.backup.v2.processor.AccountDataArchiveProcessor import org.thoughtcrime.securesms.backup.v2.processor.AdHocCallArchiveProcessor @@ -49,6 +53,7 @@ import org.thoughtcrime.securesms.crypto.AttachmentSecretProvider import org.thoughtcrime.securesms.crypto.DatabaseSecretProvider import org.thoughtcrime.securesms.database.AttachmentTable import org.thoughtcrime.securesms.database.KeyValueDatabase +import org.thoughtcrime.securesms.database.SearchTable import org.thoughtcrime.securesms.database.SignalDatabase import org.thoughtcrime.securesms.database.model.InAppPaymentSubscriberRecord import org.thoughtcrime.securesms.dependencies.AppDependencies @@ -375,11 +380,11 @@ object BackupRepository { } return frameReader.use { reader -> - import(backupKey, reader, selfData) + import(backupKey, reader, selfData, cancellationSignal = { false }) } } - fun import(length: Long, inputStreamFactory: () -> InputStream, selfData: SelfData, plaintext: Boolean = false): ImportResult { + fun import(length: Long, inputStreamFactory: () -> InputStream, selfData: SelfData, plaintext: Boolean = false, cancellationSignal: () -> Boolean = { false }): ImportResult { val backupKey = SignalStore.svr.getOrCreateMasterKey().deriveBackupKey() val frameReader = if (plaintext) { @@ -394,79 +399,139 @@ object BackupRepository { } return frameReader.use { reader -> - import(backupKey, reader, selfData) + import(backupKey, reader, selfData, cancellationSignal) } } private fun import( backupKey: BackupKey, frameReader: BackupImportReader, - selfData: SelfData + selfData: SelfData, + cancellationSignal: () -> Boolean ): ImportResult { + val stopwatch = Stopwatch("import") val eventTimer = EventTimer() val header = frameReader.getHeader() if (header == null) { - Log.e(TAG, "Backup is missing header!") + Log.e(TAG, "[import] Backup is missing header!") return ImportResult.Failure } else if (header.version > VERSION) { - Log.e(TAG, "Backup version is newer than we understand: ${header.version}") + Log.e(TAG, "[import] Backup version is newer than we understand: ${header.version}") return ImportResult.Failure } - SignalDatabase.rawDatabase.withinTransaction { - SignalDatabase.recipients.clearAllDataForBackupRestore() - SignalDatabase.distributionLists.clearAllDataForBackupRestore() - SignalDatabase.threads.clearAllDataForBackupRestore() - SignalDatabase.messages.clearAllDataForBackupRestore() - SignalDatabase.attachments.clearAllDataForBackupRestore() - SignalDatabase.stickers.clearAllDataForBackupRestore() - SignalDatabase.reactions.clearAllDataForBackupRestore() - SignalDatabase.inAppPayments.clearAllDataForBackupRestore() - SignalDatabase.chatColors.clearAllDataForBackupRestore() - SignalDatabase.calls.clearAllDataForBackup() - SignalDatabase.callLinks.clearAllDataForBackup() + try { + // Removing all the data from the various tables is *very* expensive (i.e. can take *several* minutes) if we don't do some pre-work. + // SQLite optimizes deletes if there's no foreign keys, triggers, or WHERE clause, so that's the environment we're gonna create. + + Log.d(TAG, "[import] Disabling foreign keys...") + SignalDatabase.rawDatabase.forceForeignKeyConstraintsEnabled(false) + + Log.d(TAG, "[import] Acquiring transaction...") + SignalDatabase.rawDatabase.beginTransaction() + + Log.d(TAG, "[import] Inside transaction.") + stopwatch.split("get-transaction") + + Log.d(TAG, "[import] --- Dropping all indices ---") + val indexMetadata = SignalDatabase.rawDatabase.getAllIndexDefinitions() + for (index in indexMetadata) { + Log.d(TAG, "[import] Dropping index ${index.name}...") + SignalDatabase.rawDatabase.execSQL("DROP INDEX IF EXISTS ${index.name}") + } + stopwatch.split("drop-indices") + + if (cancellationSignal()) { + return ImportResult.Failure + } + + Log.d(TAG, "[import] --- Dropping all triggers ---") + val triggerMetadata = SignalDatabase.rawDatabase.getAllTriggerDefinitions() + for (trigger in triggerMetadata) { + Log.d(TAG, "[import] Dropping trigger ${trigger.name}...") + SignalDatabase.rawDatabase.execSQL("DROP TRIGGER IF EXISTS ${trigger.name}") + } + stopwatch.split("drop-triggers") + + if (cancellationSignal()) { + return ImportResult.Failure + } + + Log.d(TAG, "[import] --- Recreating all tables ---") + val tableMetadata = SignalDatabase.rawDatabase.getAllTableDefinitions().filter { !it.name.startsWith(SearchTable.FTS_TABLE_NAME + "_") } + for (table in tableMetadata) { + Log.d(TAG, "[import] Dropping table ${table.name}...") + SignalDatabase.rawDatabase.execSQL("DROP TABLE IF EXISTS ${table.name}") + + Log.d(TAG, "[import] Creating table ${table.name}...") + SignalDatabase.rawDatabase.execSQL(table.statement) + } + + RecipientId.clearCache() + AppDependencies.recipientCache.clear() + AppDependencies.recipientCache.clearSelf() + + stopwatch.split("drop-data") + + if (cancellationSignal()) { + return ImportResult.Failure + } // Add back self after clearing data val selfId: RecipientId = SignalDatabase.recipients.getAndPossiblyMerge(selfData.aci, selfData.pni, selfData.e164, pniVerified = true, changeSelf = true) SignalDatabase.recipients.setProfileKey(selfId, selfData.profileKey) SignalDatabase.recipients.setProfileSharing(selfId, true) - eventTimer.emit("setup") val importState = ImportState(backupKey) val chatItemInserter: ChatItemArchiveImporter = ChatItemArchiveProcessor.beginImport(importState) + Log.d(TAG, "[import] Beginning to read frames.") val totalLength = frameReader.getStreamLength() + var frameCount = 0 for (frame in frameReader) { when { frame.account != null -> { AccountDataArchiveProcessor.import(frame.account, selfId, importState) eventTimer.emit("account") + frameCount++ } frame.recipient != null -> { RecipientArchiveProcessor.import(frame.recipient, importState) eventTimer.emit("recipient") + frameCount++ } frame.chat != null -> { ChatArchiveProcessor.import(frame.chat, importState) eventTimer.emit("chat") + frameCount++ } frame.adHocCall != null -> { AdHocCallArchiveProcessor.import(frame.adHocCall, importState) eventTimer.emit("call") + frameCount++ } frame.stickerPack != null -> { StickerArchiveProcessor.import(frame.stickerPack) eventTimer.emit("sticker-pack") + frameCount++ } frame.chatItem != null -> { chatItemInserter.import(frame.chatItem) eventTimer.emit("chatItem") + frameCount++ + + if (frameCount % 1000 == 0) { + if (cancellationSignal()) { + return ImportResult.Failure + } + Log.d(TAG, "Imported $frameCount frames so far.") + } // TODO if there's stuff in the stream after chatItems, we need to flush the inserter before going to the next phase } @@ -479,16 +544,50 @@ object BackupRepository { eventTimer.emit("chatItem") } + stopwatch.split("frames") + + Log.d(TAG, "[import] Rebuilding FTS index...") + SignalDatabase.messageSearch.rebuildIndex() + + Log.d(TAG, "[import] --- Recreating indices ---") + for (index in indexMetadata) { + Log.d(TAG, "[import] Creating index ${index.name}...") + SignalDatabase.rawDatabase.execSQL(index.statement) + } + stopwatch.split("recreate-indices") + + Log.d(TAG, "[import] --- Recreating triggers ---") + for (trigger in triggerMetadata) { + Log.d(TAG, "[import] Creating trigger ${trigger.name}...") + SignalDatabase.rawDatabase.execSQL(trigger.statement) + } + stopwatch.split("recreate-triggers") + + Log.d(TAG, "[import] Updating threads...") importState.chatIdToLocalThreadId.values.forEach { SignalDatabase.threads.update(it, unarchive = false, allowDeletion = false) } + stopwatch.split("thread-updates") + + val foreignKeyViolations = SignalDatabase.rawDatabase.getForeignKeyViolations() + if (foreignKeyViolations.isNotEmpty()) { + throw IllegalStateException("Foreign key check failed! Violations: $foreignKeyViolations") + } + stopwatch.split("fk-check") + + SignalDatabase.rawDatabase.setTransactionSuccessful() + } finally { + if (SignalDatabase.rawDatabase.inTransaction()) { + SignalDatabase.rawDatabase.endTransaction() + } + + Log.d(TAG, "[import] Re-enabling foreign keys...") + SignalDatabase.rawDatabase.forceForeignKeyConstraintsEnabled(true) } AppDependencies.recipientCache.clear() AppDependencies.recipientCache.warmUp() - Log.d(TAG, "import() ${eventTimer.stop().summary}") - val groupJobs = SignalDatabase.groups.getGroups().use { groups -> groups .asSequence() @@ -502,6 +601,10 @@ object BackupRepository { .toList() } AppDependencies.jobManager.addAll(groupJobs) + stopwatch.split("group-jobs") + + Log.d(TAG, "[import] Finished! ${eventTimer.stop().summary}") + stopwatch.stop(TAG) return ImportResult.Success(backupTime = header.backupTimeMs) } @@ -1091,6 +1194,10 @@ class ImportState(val backupKey: BackupKey) { val chatIdToLocalRecipientId: MutableMap = hashMapOf() val chatIdToBackupRecipientId: MutableMap = hashMapOf() val remoteToLocalColorId: MutableMap = hashMapOf() + + fun requireLocalRecipientId(remoteId: Long): RecipientId { + return remoteToLocalRecipientId[remoteId] ?: throw IllegalArgumentException("There is no local recipientId for remote recipientId $remoteId!") + } } class BackupMetadata( diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/AttachmentTableArchiveExtensions.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/AttachmentTableArchiveExtensions.kt index 52394176ce..b100df773c 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/AttachmentTableArchiveExtensions.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/AttachmentTableArchiveExtensions.kt @@ -5,17 +5,10 @@ package org.thoughtcrime.securesms.backup.v2.database -import org.signal.core.util.SqlUtil -import org.signal.core.util.deleteAll import org.thoughtcrime.securesms.attachments.Attachment import org.thoughtcrime.securesms.attachments.AttachmentId import org.thoughtcrime.securesms.database.AttachmentTable -fun AttachmentTable.clearAllDataForBackupRestore() { - writableDatabase.deleteAll(AttachmentTable.TABLE_NAME) - SqlUtil.resetAutoIncrementValue(writableDatabase, AttachmentTable.TABLE_NAME) -} - fun AttachmentTable.restoreWallpaperAttachment(attachment: Attachment): AttachmentId? { return insertAttachmentsForMessage(AttachmentTable.WALLPAPER_MESSAGE_ID, listOf(attachment), emptyList()).values.firstOrNull() } diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/CallLinkTableArchiveExtensions.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/CallLinkTableArchiveExtensions.kt index 0ed2e934a4..28c499f983 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/CallLinkTableArchiveExtensions.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/CallLinkTableArchiveExtensions.kt @@ -5,8 +5,6 @@ package org.thoughtcrime.securesms.backup.v2.database -import org.signal.core.util.SqlUtil -import org.signal.core.util.deleteAll import org.signal.core.util.select import org.thoughtcrime.securesms.database.CallLinkTable @@ -14,12 +12,8 @@ fun CallLinkTable.getCallLinksForBackup(): CallLinkArchiveExporter { val cursor = readableDatabase .select() .from(CallLinkTable.TABLE_NAME) + .where("${CallLinkTable.ROOT_KEY} NOT NULL") .run() return CallLinkArchiveExporter(cursor) } - -fun CallLinkTable.clearAllDataForBackup() { - writableDatabase.deleteAll(CallLinkTable.TABLE_NAME) - SqlUtil.resetAutoIncrementValue(writableDatabase, CallLinkTable.TABLE_NAME) -} diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/CallTableArchiveExtensions.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/CallTableArchiveExtensions.kt index 5ed0cfe8b2..1c780545d6 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/CallTableArchiveExtensions.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/CallTableArchiveExtensions.kt @@ -5,8 +5,6 @@ package org.thoughtcrime.securesms.backup.v2.database -import org.signal.core.util.SqlUtil -import org.signal.core.util.deleteAll import org.signal.core.util.select import org.thoughtcrime.securesms.database.CallTable @@ -19,8 +17,3 @@ fun CallTable.getAdhocCallsForBackup(): AdHocCallArchiveExporter { .run() ) } - -fun CallTable.clearAllDataForBackup() { - writableDatabase.deleteAll(CallTable.TABLE_NAME) - SqlUtil.resetAutoIncrementValue(writableDatabase, CallTable.TABLE_NAME) -} diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/ChatColorsTableArchiveExtensions.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/ChatColorsTableArchiveExtensions.kt deleted file mode 100644 index e4458492c9..0000000000 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/ChatColorsTableArchiveExtensions.kt +++ /dev/null @@ -1,15 +0,0 @@ -/* - * Copyright 2024 Signal Messenger, LLC - * SPDX-License-Identifier: AGPL-3.0-only - */ - -package org.thoughtcrime.securesms.backup.v2.database - -import org.signal.core.util.SqlUtil -import org.signal.core.util.deleteAll -import org.thoughtcrime.securesms.database.ChatColorsTable - -fun ChatColorsTable.clearAllDataForBackupRestore() { - writableDatabase.deleteAll(ChatColorsTable.TABLE_NAME) - SqlUtil.resetAutoIncrementValue(writableDatabase, ChatColorsTable.TABLE_NAME) -} diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/DistributionListTablesArchiveExtensions.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/DistributionListTablesArchiveExtensions.kt index ec13aa231f..2bd60d1f90 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/DistributionListTablesArchiveExtensions.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/DistributionListTablesArchiveExtensions.kt @@ -5,8 +5,6 @@ package org.thoughtcrime.securesms.backup.v2.database -import org.signal.core.util.SqlUtil -import org.signal.core.util.deleteAll import org.signal.core.util.select import org.signal.core.util.withinTransaction import org.thoughtcrime.securesms.backup.v2.exporters.DistributionListArchiveExporter @@ -39,10 +37,3 @@ fun DistributionListTables.getMembersForBackup(id: DistributionListId): List rawMembers } } - -fun DistributionListTables.clearAllDataForBackupRestore() { - writableDatabase.deleteAll(DistributionListTables.ListTable.TABLE_NAME) - writableDatabase.deleteAll(DistributionListTables.MembershipTable.TABLE_NAME) - SqlUtil.resetAutoIncrementValue(writableDatabase, DistributionListTables.ListTable.TABLE_NAME) - SqlUtil.resetAutoIncrementValue(writableDatabase, DistributionListTables.MembershipTable.TABLE_NAME) -} diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/InAppPaymentTableArchiveExtensions.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/InAppPaymentTableArchiveExtensions.kt deleted file mode 100644 index 3ed4bcd030..0000000000 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/InAppPaymentTableArchiveExtensions.kt +++ /dev/null @@ -1,15 +0,0 @@ -/* - * Copyright 2024 Signal Messenger, LLC - * SPDX-License-Identifier: AGPL-3.0-only - */ - -package org.thoughtcrime.securesms.backup.v2.database - -import org.signal.core.util.SqlUtil -import org.signal.core.util.deleteAll -import org.thoughtcrime.securesms.database.InAppPaymentTable - -fun InAppPaymentTable.clearAllDataForBackupRestore() { - writableDatabase.deleteAll(InAppPaymentTable.TABLE_NAME) - SqlUtil.resetAutoIncrementValue(writableDatabase, InAppPaymentTable.TABLE_NAME) -} diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/MessageTableArchiveExtensions.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/MessageTableArchiveExtensions.kt index 98a77f044c..a460f9c44c 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/MessageTableArchiveExtensions.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/MessageTableArchiveExtensions.kt @@ -5,7 +5,6 @@ package org.thoughtcrime.securesms.backup.v2.database -import org.signal.core.util.SqlUtil import org.signal.core.util.select import org.thoughtcrime.securesms.backup.v2.ImportState import org.thoughtcrime.securesms.backup.v2.exporters.ChatItemArchiveExporter @@ -78,8 +77,3 @@ fun MessageTable.getMessagesForBackup(db: SignalDatabase, backupTime: Long, medi fun MessageTable.createChatItemInserter(importState: ImportState): ChatItemArchiveImporter { return ChatItemArchiveImporter(writableDatabase, importState, 500) } - -fun MessageTable.clearAllDataForBackupRestore() { - writableDatabase.delete(MessageTable.TABLE_NAME, null, null) - SqlUtil.resetAutoIncrementValue(writableDatabase, MessageTable.TABLE_NAME) -} diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/ReactionTableArchiveExtensions.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/ReactionTableArchiveExtensions.kt deleted file mode 100644 index cd5e8690b5..0000000000 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/ReactionTableArchiveExtensions.kt +++ /dev/null @@ -1,15 +0,0 @@ -/* - * Copyright 2023 Signal Messenger, LLC - * SPDX-License-Identifier: AGPL-3.0-only - */ - -package org.thoughtcrime.securesms.backup.v2.database - -import org.signal.core.util.SqlUtil -import org.signal.core.util.deleteAll -import org.thoughtcrime.securesms.database.ReactionTable - -fun ReactionTable.clearAllDataForBackupRestore() { - writableDatabase.deleteAll(ReactionTable.TABLE_NAME) - SqlUtil.resetAutoIncrementValue(writableDatabase, ReactionTable.TABLE_NAME) -} diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/RecipientTableArchiveExtensions.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/RecipientTableArchiveExtensions.kt index 4928136022..0bef7777e6 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/RecipientTableArchiveExtensions.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/RecipientTableArchiveExtensions.kt @@ -7,8 +7,6 @@ package org.thoughtcrime.securesms.backup.v2.database import android.content.ContentValues import org.signal.core.util.Base64 -import org.signal.core.util.SqlUtil -import org.signal.core.util.deleteAll import org.signal.core.util.logging.Log import org.signal.core.util.nullIfBlank import org.signal.core.util.select @@ -20,7 +18,6 @@ import org.thoughtcrime.securesms.backup.v2.proto.AccountData import org.thoughtcrime.securesms.database.GroupTable import org.thoughtcrime.securesms.database.RecipientTable import org.thoughtcrime.securesms.database.model.databaseprotos.RecipientExtras -import org.thoughtcrime.securesms.dependencies.AppDependencies import org.thoughtcrime.securesms.keyvalue.SignalStore import org.thoughtcrime.securesms.profiles.ProfileName import org.thoughtcrime.securesms.recipients.RecipientId @@ -123,15 +120,6 @@ fun RecipientTable.restoreSelfFromBackup(accountData: AccountData, selfId: Recip .run() } -fun RecipientTable.clearAllDataForBackupRestore() { - writableDatabase.deleteAll(RecipientTable.TABLE_NAME) - SqlUtil.resetAutoIncrementValue(writableDatabase, RecipientTable.TABLE_NAME) - - RecipientId.clearCache() - AppDependencies.recipientCache.clear() - AppDependencies.recipientCache.clearSelf() -} - fun RecipientTable.restoreReleaseNotes(): RecipientId { val releaseChannelId: RecipientId = insertReleaseChannelRecipient() SignalStore.releaseChannel.setReleaseChannelRecipientId(releaseChannelId) diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/StickerTableArchiveExtensions.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/StickerTableArchiveExtensions.kt deleted file mode 100644 index a9f5edf249..0000000000 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/StickerTableArchiveExtensions.kt +++ /dev/null @@ -1,15 +0,0 @@ -/* - * Copyright 2023 Signal Messenger, LLC - * SPDX-License-Identifier: AGPL-3.0-only - */ - -package org.thoughtcrime.securesms.backup.v2.database - -import org.signal.core.util.SqlUtil -import org.signal.core.util.deleteAll -import org.thoughtcrime.securesms.database.StickerTable - -fun StickerTable.clearAllDataForBackupRestore() { - writableDatabase.deleteAll(StickerTable.TABLE_NAME) - SqlUtil.resetAutoIncrementValue(writableDatabase, StickerTable.TABLE_NAME) -} diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/ThreadTableArchiveExtensions.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/ThreadTableArchiveExtensions.kt index 6874572bb1..21b0f08150 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/ThreadTableArchiveExtensions.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/ThreadTableArchiveExtensions.kt @@ -5,7 +5,6 @@ package org.thoughtcrime.securesms.backup.v2.database -import org.signal.core.util.SqlUtil import org.thoughtcrime.securesms.backup.v2.exporters.ChatArchiveExporter import org.thoughtcrime.securesms.database.RecipientTable import org.thoughtcrime.securesms.database.SignalDatabase @@ -35,9 +34,3 @@ fun ThreadTable.getThreadsForBackup(db: SignalDatabase): ChatArchiveExporter { return ChatArchiveExporter(cursor, db) } - -fun ThreadTable.clearAllDataForBackupRestore() { - writableDatabase.delete(ThreadTable.TABLE_NAME, null, null) - SqlUtil.resetAutoIncrementValue(writableDatabase, ThreadTable.TABLE_NAME) - clearCache() -} diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/exporters/CallLinkArchiveExporter.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/exporters/CallLinkArchiveExporter.kt index 90c0fb4da3..0da7cd2ca4 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/exporters/CallLinkArchiveExporter.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/exporters/CallLinkArchiveExporter.kt @@ -6,7 +6,6 @@ package org.thoughtcrime.securesms.backup.v2.database import android.database.Cursor -import okio.ByteString import okio.ByteString.Companion.toByteString import org.signal.ringrtc.CallLinkState import org.thoughtcrime.securesms.backup.v2.ArchiveRecipient @@ -32,8 +31,8 @@ class CallLinkArchiveExporter(private val cursor: Cursor) : Iterator CallTable.Event.GENERIC_GROUP_CALL AdHocCall.State.UNKNOWN_STATE -> CallTable.Event.GENERIC_GROUP_CALL } + val peer = importState.remoteToLocalRecipientId[call.recipientId] ?: run { + Log.w(TAG, "Failed to find matching recipientId for peer with remote recipientId ${call.recipientId}! Skipping.") + return + } + SignalDatabase.writableDatabase .insertInto(CallTable.TABLE_NAME) .values( CallTable.CALL_ID to call.callId, - CallTable.PEER to importState.remoteToLocalRecipientId[call.recipientId]!!.serialize(), + CallTable.PEER to peer.serialize(), CallTable.TYPE to CallTable.Type.serialize(CallTable.Type.AD_HOC_CALL), CallTable.DIRECTION to CallTable.Direction.serialize(CallTable.Direction.OUTGOING), CallTable.EVENT to CallTable.Event.serialize(event), diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/importer/CallLinkArchiveImporter.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/importer/CallLinkArchiveImporter.kt index c1d7d1bad5..8cfc1ee2ac 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/importer/CallLinkArchiveImporter.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/importer/CallLinkArchiveImporter.kt @@ -5,6 +5,8 @@ package org.thoughtcrime.securesms.backup.v2.importer +import org.signal.core.util.isEmpty +import org.signal.core.util.logging.Log import org.signal.ringrtc.CallLinkRootKey import org.signal.ringrtc.CallLinkState import org.thoughtcrime.securesms.backup.v2.ArchiveCallLink @@ -21,13 +23,21 @@ import java.time.Instant * Handles the importing of [ArchiveCallLink] models into the local database. */ object CallLinkArchiveImporter { + + private val TAG = Log.tag(CallLinkArchiveImporter::class) + fun import(callLink: ArchiveCallLink): RecipientId? { - val rootKey: CallLinkRootKey - try { - rootKey = CallLinkRootKey(callLink.rootKey.toByteArray()) + val rootKey: CallLinkRootKey = try { + CallLinkRootKey(callLink.rootKey.toByteArray()) } catch (e: Exception) { + if (callLink.rootKey.isEmpty()) { + Log.w(TAG, "Missing root key!") + } else { + Log.w(TAG, "Failed to parse a non-empty root key!") + } return null } + return SignalDatabase.callLinks.insertCallLink( CallLinkTable.CallLink( recipientId = RecipientId.UNKNOWN, diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/importer/ChatItemArchiveImporter.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/importer/ChatItemArchiveImporter.kt index b633cd7e2b..20b89080b4 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/importer/ChatItemArchiveImporter.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/importer/ChatItemArchiveImporter.kt @@ -875,7 +875,7 @@ class ChatItemArchiveImporter( private fun ContentValues.addQuote(quote: Quote) { this.put(MessageTable.QUOTE_ID, quote.targetSentTimestamp ?: MessageTable.QUOTE_TARGET_MISSING_ID) - this.put(MessageTable.QUOTE_AUTHOR, importState.remoteToLocalRecipientId[quote.authorId]!!.serialize()) + this.put(MessageTable.QUOTE_AUTHOR, importState.requireLocalRecipientId(quote.authorId).serialize()) this.put(MessageTable.QUOTE_BODY, quote.text?.body) this.put(MessageTable.QUOTE_TYPE, quote.type.toLocalQuoteType()) this.put(MessageTable.QUOTE_BODY_RANGES, quote.text?.bodyRanges?.toLocalBodyRanges()?.encode()) diff --git a/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/backup/InternalBackupPlaygroundViewModel.kt b/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/backup/InternalBackupPlaygroundViewModel.kt index 30a9ba3b8f..c57309afed 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/backup/InternalBackupPlaygroundViewModel.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/backup/InternalBackupPlaygroundViewModel.kt @@ -536,6 +536,7 @@ class InternalBackupPlaygroundViewModel : ViewModel() { AppDependencies.jobManager.cancelAllInQueue("ArchiveAttachmentJobs_0") AppDependencies.jobManager.cancelAllInQueue("ArchiveAttachmentJobs_1") AppDependencies.jobManager.cancelAllInQueue("ArchiveThumbnailUploadJob") + AppDependencies.jobManager.cancelAllInQueue("BackupRestoreJob") } fun fetchRemoteBackupAndWritePlaintext(outputStream: OutputStream?) { diff --git a/app/src/main/java/org/thoughtcrime/securesms/database/NameCollisionTables.kt b/app/src/main/java/org/thoughtcrime/securesms/database/NameCollisionTables.kt index de0ca4ff64..79787dfc11 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/database/NameCollisionTables.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/database/NameCollisionTables.kt @@ -55,16 +55,12 @@ class NameCollisionTables( private val PROFILE_CHANGE_TIMEOUT = 1.days - fun createTables(db: SQLiteDatabase) { - db.execSQL(NameCollisionTable.CREATE_TABLE) - db.execSQL(NameCollisionMembershipTable.CREATE_TABLE) - } + val CREATE_TABLE = arrayOf( + NameCollisionTable.CREATE_TABLE, + NameCollisionMembershipTable.CREATE_TABLE + ) - fun createIndexes(db: SQLiteDatabase) { - NameCollisionMembershipTable.CREATE_INDEXES.forEach { - db.execSQL(it) - } - } + val CREATE_INDEXES = NameCollisionMembershipTable.CREATE_INDEXES } /** diff --git a/app/src/main/java/org/thoughtcrime/securesms/database/SQLiteDatabase.java b/app/src/main/java/org/thoughtcrime/securesms/database/SQLiteDatabase.java index 428ac0efd6..37ea3a7159 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/database/SQLiteDatabase.java +++ b/app/src/main/java/org/thoughtcrime/securesms/database/SQLiteDatabase.java @@ -14,6 +14,7 @@ import androidx.sqlite.db.SupportSQLiteQuery; import net.zetetic.database.sqlcipher.SQLiteStatement; import net.zetetic.database.sqlcipher.SQLiteTransactionListener; +import org.signal.core.util.logging.Log; import org.signal.core.util.tracing.Tracer; import java.io.IOException; diff --git a/app/src/main/java/org/thoughtcrime/securesms/database/SearchTable.kt b/app/src/main/java/org/thoughtcrime/securesms/database/SearchTable.kt index dc9a0b1769..f3c3a45de9 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/database/SearchTable.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/database/SearchTable.kt @@ -180,10 +180,11 @@ class SearchTable(context: Context, databaseHelper: SignalDatabase) : DatabaseTa writableDatabase.withinTransaction { db -> db.execSQL( """ - INSERT INTO $FTS_TABLE_NAME ($ID, $BODY) + INSERT INTO $FTS_TABLE_NAME ($ID, $BODY, $THREAD_ID) SELECT ${MessageTable.ID}, - ${MessageTable.BODY} + ${MessageTable.BODY}, + ${MessageTable.THREAD_ID} FROM ${MessageTable.TABLE_NAME} WHERE diff --git a/app/src/main/java/org/thoughtcrime/securesms/database/SignalDatabase.kt b/app/src/main/java/org/thoughtcrime/securesms/database/SignalDatabase.kt index 0d581c71b8..a2ea24a7f6 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/database/SignalDatabase.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/database/SignalDatabase.kt @@ -113,7 +113,7 @@ open class SignalDatabase(private val context: Application, databaseSecret: Data db.execSQL(CallLinkTable.CREATE_TABLE) db.execSQL(CallTable.CREATE_TABLE) db.execSQL(KyberPreKeyTable.CREATE_TABLE) - NameCollisionTables.createTables(db) + executeStatements(db, NameCollisionTables.CREATE_TABLE) db.execSQL(InAppPaymentTable.CREATE_TABLE) db.execSQL(InAppPaymentSubscriberTable.CREATE_TABLE) executeStatements(db, SearchTable.CREATE_TABLE) @@ -144,12 +144,11 @@ open class SignalDatabase(private val context: Application, databaseSecret: Data executeStatements(db, ReactionTable.CREATE_INDEXES) executeStatements(db, KyberPreKeyTable.CREATE_INDEXES) executeStatements(db, ChatFolderTables.CREATE_INDEXES) + executeStatements(db, NameCollisionTables.CREATE_INDEXES) executeStatements(db, SearchTable.CREATE_TRIGGERS) executeStatements(db, MessageSendLogTables.CREATE_TRIGGERS) - NameCollisionTables.createIndexes(db) - DistributionListTables.insertInitialDistributionListAtCreationTime(db) ChatFolderTables.insertInitialChatFoldersAtCreationTime(db) diff --git a/app/src/main/java/org/thoughtcrime/securesms/database/SqlCipherErrorHandler.kt b/app/src/main/java/org/thoughtcrime/securesms/database/SqlCipherErrorHandler.kt index 322a769bee..5956e73ec5 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/database/SqlCipherErrorHandler.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/database/SqlCipherErrorHandler.kt @@ -165,16 +165,16 @@ class SqlCipherErrorHandler(private val databaseName: String) : DatabaseErrorHan } private fun attemptToClearFullTextSearchIndex(db: SQLiteDatabase) { - try { - try { - db.reopenReadWrite() - } catch (e: Exception) { - Log.w(TAG, "Failed to re-open as read-write!", e) - } - SignalDatabase.messageSearch.fullyResetTables(db, useTransaction = false) - } catch (e: Throwable) { - Log.w(TAG, "Failed to clear full text search index.", e) - } +// try { +// try { +// db.reopenReadWrite() +// } catch (e: Exception) { +// Log.w(TAG, "Failed to re-open as read-write!", e) +// } +// SignalDatabase.messageSearch.fullyResetTables(db, useTransaction = false) +// } catch (e: Throwable) { +// Log.w(TAG, "Failed to clear full text search index.", e) +// } } private sealed class DiagnosticResults(val logs: String) { diff --git a/app/src/main/java/org/thoughtcrime/securesms/jobmanager/JobManager.java b/app/src/main/java/org/thoughtcrime/securesms/jobmanager/JobManager.java index 5b9b18f9b2..c75ca04690 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/jobmanager/JobManager.java +++ b/app/src/main/java/org/thoughtcrime/securesms/jobmanager/JobManager.java @@ -64,7 +64,9 @@ public class JobManager implements ConstraintObserver.Notifier { public JobManager(@NonNull Application application, @NonNull Configuration configuration) { this.application = application; this.configuration = configuration; - this.executor = new FilteredExecutor(configuration.getExecutorFactory().newSingleThreadExecutor("signal-JobManager"), ThreadUtil::isMainThread); + this.executor = new FilteredExecutor(configuration.getExecutorFactory().newSingleThreadExecutor("signal-JobManager"), () -> { + return ThreadUtil.isMainThread() || Thread.currentThread().getName().equals("Instr: org.thoughtcrime.securesms.testing.SignalTestRunner"); + }); this.jobTracker = configuration.getJobTracker(); this.jobController = new JobController(application, configuration.getJobStorage(), diff --git a/app/src/main/java/org/thoughtcrime/securesms/jobs/BackupRestoreJob.kt b/app/src/main/java/org/thoughtcrime/securesms/jobs/BackupRestoreJob.kt index 1235c22a76..393574ed85 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/jobs/BackupRestoreJob.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/jobs/BackupRestoreJob.kt @@ -39,6 +39,7 @@ class BackupRestoreJob private constructor(parameters: Parameters) : BaseJob(par .addConstraint(NetworkConstraint.KEY) .setMaxAttempts(Parameters.UNLIMITED) .setMaxInstancesForFactory(1) + .setQueue("BackupRestoreJob") .build() ) @@ -85,6 +86,10 @@ class BackupRestoreJob private constructor(parameters: Parameters) : BaseJob(par throw IOException() } + if (isCanceled) { + return + } + controller.update( title = context.getString(R.string.BackupProgressService_title), progress = 0f, @@ -93,7 +98,7 @@ class BackupRestoreJob private constructor(parameters: Parameters) : BaseJob(par val self = Recipient.self() val selfData = BackupRepository.SelfData(self.aci.get(), self.pni.get(), self.e164.get(), ProfileKey(self.profileKey)) - BackupRepository.import(length = tempBackupFile.length(), inputStreamFactory = tempBackupFile::inputStream, selfData = selfData, plaintext = false) + BackupRepository.import(length = tempBackupFile.length(), inputStreamFactory = tempBackupFile::inputStream, selfData = selfData, plaintext = false, cancellationSignal = { isCanceled }) SignalStore.backup.restoreState = RestoreState.RESTORING_MEDIA } diff --git a/core-util-jvm/src/main/java/org/signal/core/util/ProtoExtensions.kt b/core-util-jvm/src/main/java/org/signal/core/util/ProtoExtensions.kt index 2bd6cf61d1..5f8b6e3bae 100644 --- a/core-util-jvm/src/main/java/org/signal/core/util/ProtoExtensions.kt +++ b/core-util-jvm/src/main/java/org/signal/core/util/ProtoExtensions.kt @@ -20,6 +20,10 @@ import java.util.LinkedList private const val TAG = "ProtoExtension" +fun ByteString?.isEmpty(): Boolean { + return this == null || this.size == 0 +} + fun ByteString?.isNotEmpty(): Boolean { return this != null && this.size > 0 } diff --git a/core-util/src/main/java/org/signal/core/util/SQLiteDatabaseExtensions.kt b/core-util/src/main/java/org/signal/core/util/SQLiteDatabaseExtensions.kt index 7d53d6c0b0..a68afc5d12 100644 --- a/core-util/src/main/java/org/signal/core/util/SQLiteDatabaseExtensions.kt +++ b/core-util/src/main/java/org/signal/core/util/SQLiteDatabaseExtensions.kt @@ -6,6 +6,12 @@ import android.database.sqlite.SQLiteDatabase import androidx.core.content.contentValuesOf import androidx.sqlite.db.SupportSQLiteDatabase import androidx.sqlite.db.SupportSQLiteQueryBuilder +import org.signal.core.util.SqlUtil.ForeignKeyViolation +import org.signal.core.util.logging.Log +import kotlin.time.Duration +import kotlin.time.Duration.Companion.seconds + +private val TAG = "SQLiteDatabaseExtensions" /** * Begins a transaction on the `this` database, runs the provided [block] providing the `this` value as it's argument @@ -46,7 +52,11 @@ fun SupportSQLiteDatabase.getAllTables(): List { * Returns a list of objects that represent the table definitions in the database. Basically the table name and then the SQL that was used to create it. */ fun SupportSQLiteDatabase.getAllTableDefinitions(): List { - return this.query("SELECT name, sql FROM sqlite_schema WHERE type = 'table' AND sql NOT NULL AND name != 'sqlite_sequence'") + return this + .select("name", "sql") + .from("sqlite_schema") + .where("type = ? AND sql NOT NULL AND name != ?", "table", "sqlite_sequence") + .run() .readToList { cursor -> CreateStatement( name = cursor.requireNonNullString("name"), @@ -61,7 +71,11 @@ fun SupportSQLiteDatabase.getAllTableDefinitions(): List { * Returns a list of objects that represent the index definitions in the database. Basically the index name and then the SQL that was used to create it. */ fun SupportSQLiteDatabase.getAllIndexDefinitions(): List { - return this.query("SELECT name, sql FROM sqlite_schema WHERE type = 'index' AND sql NOT NULL") + return this + .select("name", "sql") + .from("sqlite_schema") + .where("type = ? AND sql NOT NULL", "index") + .run() .readToList { cursor -> CreateStatement( name = cursor.requireNonNullString("name"), @@ -71,6 +85,24 @@ fun SupportSQLiteDatabase.getAllIndexDefinitions(): List { .sortedBy { it.name } } +/** + * Retrieves the names of all triggers, sorted alphabetically. + */ +fun SupportSQLiteDatabase.getAllTriggerDefinitions(): List { + return this + .select("name", "sql") + .from("sqlite_schema") + .where("type = ? AND sql NOT NULL", "trigger") + .run() + .readToList { + CreateStatement( + name = it.requireNonNullString("name"), + statement = it.requireNonNullString("sql") + ) + } + .sortedBy { it.name } +} + fun SupportSQLiteDatabase.getForeignKeys(): List { return SqlUtil.getAllTables(this) .map { table -> @@ -93,6 +125,24 @@ fun SupportSQLiteDatabase.areForeignKeyConstraintsEnabled(): Boolean { } } +/** + * Provides a list of all foreign key violations present. + * If a [targetTable] is specified, results will be limited to that table specifically. + * Otherwise, the check will be performed across all tables. + */ +@JvmOverloads +fun SupportSQLiteDatabase.getForeignKeyViolations(targetTable: String? = null): List { + return SqlUtil.getForeignKeyViolations(this, targetTable) +} + +/** + * For tables that have an autoincrementing primary key, this will reset the key to start back at 1. + * IMPORTANT: This is quite dangerous! Only do this if you're effectively resetting the entire database. + */ +fun SupportSQLiteDatabase.resetAutoIncrementValue(targetTable: String) { + SqlUtil.resetAutoIncrementValue(this, targetTable) +} + /** * Does a full WAL checkpoint (TRUNCATE mode, where the log is for sure flushed and the log is zero'd out). * Will try up to [maxAttempts] times. Can technically fail if the database is too active and the checkpoint @@ -132,6 +182,22 @@ fun SupportSQLiteDatabase.getIndexes(): List { } } +fun SupportSQLiteDatabase.forceForeignKeyConstraintsEnabled(enabled: Boolean, timeout: Duration = 10.seconds) { + val startTime = System.currentTimeMillis() + while (true) { + try { + this.setForeignKeyConstraintsEnabled(enabled) + break + } catch (e: IllegalStateException) { + if (System.currentTimeMillis() - startTime > timeout.inWholeMilliseconds) { + throw IllegalStateException("Failed to force foreign keys to '$enabled' within the timeout of $timeout", e) + } + Log.w(TAG, "Failed to set foreign keys because we're in a transaction. Waiting 100ms then trying again.") + ThreadUtil.sleep(100) + } + } +} + /** * Checks if a row exists that matches the query. */