Get a big backupV2 import fully working.

app/src/androidTest/java/org/thoughtcrime/securesms/backup/v2/ArchiveImportExportTests.kt

@@ -14,7 +14,6 @@ import org.junit.Before
 import org.junit.Test
 import org.junit.runner.RunWith
 import org.signal.core.util.Base64
-import org.signal.core.util.SqlUtil
 import org.signal.core.util.logging.Log
 import org.signal.core.util.readFully
 import org.signal.libsignal.messagebackup.ComparableBackup
@@ -22,12 +21,9 @@ import org.signal.libsignal.messagebackup.MessageBackup
 import org.signal.libsignal.zkgroup.profiles.ProfileKey
 import org.thoughtcrime.securesms.backup.v2.proto.Frame
 import org.thoughtcrime.securesms.backup.v2.stream.PlainTextBackupReader
-import org.thoughtcrime.securesms.database.DistributionListTables
 import org.thoughtcrime.securesms.database.KeyValueDatabase
-import org.thoughtcrime.securesms.database.SignalDatabase
 import org.thoughtcrime.securesms.dependencies.AppDependencies
 import org.thoughtcrime.securesms.keyvalue.SignalStore
-import org.thoughtcrime.securesms.recipients.RecipientId
 import org.whispersystems.signalservice.api.kbs.MasterKey
 import org.whispersystems.signalservice.api.push.ServiceId
 import java.io.ByteArrayInputStream
@@ -263,21 +259,7 @@ class ArchiveImportExportTests {
   }
 
   private fun resetAllData() {
-    // Need to delete these first to prevent foreign key crash
+    // All the main database stuff is reset as a normal part of importing
-    SignalDatabase.rawDatabase.execSQL("DELETE FROM ${DistributionListTables.ListTable.TABLE_NAME}")
-    SignalDatabase.rawDatabase.execSQL("DELETE FROM ${DistributionListTables.MembershipTable.TABLE_NAME}")
-
-    SqlUtil.getAllTables(SignalDatabase.rawDatabase)
-      .filterNot { it.contains("sqlite") || it.contains("fts") || it.startsWith("emoji_search_") } // If we delete these we'll corrupt the DB
-      .sorted()
-      .forEach { table ->
-        SignalDatabase.rawDatabase.execSQL("DELETE FROM $table")
-        SqlUtil.resetAutoIncrementValue(SignalDatabase.rawDatabase, table)
-      }
-
-    AppDependencies.recipientCache.clear()
-    AppDependencies.recipientCache.clearSelf()
-    RecipientId.clearCache()
 
     KeyValueDatabase.getInstance(AppDependencies.application).clear()
     SignalStore.resetCache()

app/src/main/java/org/thoughtcrime/securesms/backup/v2/BackupRepository.kt

@@ -11,9 +11,15 @@ import kotlinx.coroutines.withContext
 import org.greenrobot.eventbus.EventBus
 import org.signal.core.util.Base64
 import org.signal.core.util.EventTimer
+import org.signal.core.util.Stopwatch
 import org.signal.core.util.concurrent.LimitedWorker
 import org.signal.core.util.concurrent.SignalExecutors
+import org.signal.core.util.forceForeignKeyConstraintsEnabled
 import org.signal.core.util.fullWalCheckpoint
+import org.signal.core.util.getAllIndexDefinitions
+import org.signal.core.util.getAllTableDefinitions
+import org.signal.core.util.getAllTriggerDefinitions
+import org.signal.core.util.getForeignKeyViolations
 import org.signal.core.util.logging.Log
 import org.signal.core.util.stream.NonClosingOutputStream
 import org.signal.core.util.withinTransaction
@@ -27,8 +33,6 @@ import org.thoughtcrime.securesms.attachments.Attachment
 import org.thoughtcrime.securesms.attachments.AttachmentId
 import org.thoughtcrime.securesms.attachments.Cdn
 import org.thoughtcrime.securesms.attachments.DatabaseAttachment
-import org.thoughtcrime.securesms.backup.v2.database.clearAllDataForBackup
-import org.thoughtcrime.securesms.backup.v2.database.clearAllDataForBackupRestore
 import org.thoughtcrime.securesms.backup.v2.importer.ChatItemArchiveImporter
 import org.thoughtcrime.securesms.backup.v2.processor.AccountDataArchiveProcessor
 import org.thoughtcrime.securesms.backup.v2.processor.AdHocCallArchiveProcessor
@@ -49,6 +53,7 @@ import org.thoughtcrime.securesms.crypto.AttachmentSecretProvider
 import org.thoughtcrime.securesms.crypto.DatabaseSecretProvider
 import org.thoughtcrime.securesms.database.AttachmentTable
 import org.thoughtcrime.securesms.database.KeyValueDatabase
+import org.thoughtcrime.securesms.database.SearchTable
 import org.thoughtcrime.securesms.database.SignalDatabase
 import org.thoughtcrime.securesms.database.model.InAppPaymentSubscriberRecord
 import org.thoughtcrime.securesms.dependencies.AppDependencies
@@ -375,11 +380,11 @@ object BackupRepository {
     }
 
     return frameReader.use { reader ->
-      import(backupKey, reader, selfData)
+      import(backupKey, reader, selfData, cancellationSignal = { false })
     }
   }
 
-  fun import(length: Long, inputStreamFactory: () -> InputStream, selfData: SelfData, plaintext: Boolean = false): ImportResult {
+  fun import(length: Long, inputStreamFactory: () -> InputStream, selfData: SelfData, plaintext: Boolean = false, cancellationSignal: () -> Boolean = { false }): ImportResult {
     val backupKey = SignalStore.svr.getOrCreateMasterKey().deriveBackupKey()
 
     val frameReader = if (plaintext) {
@@ -394,79 +399,139 @@ object BackupRepository {
     }
 
     return frameReader.use { reader ->
-      import(backupKey, reader, selfData)
+      import(backupKey, reader, selfData, cancellationSignal)
     }
   }
 
   private fun import(
     backupKey: BackupKey,
     frameReader: BackupImportReader,
-    selfData: SelfData
+    selfData: SelfData,
+    cancellationSignal: () -> Boolean
   ): ImportResult {
+    val stopwatch = Stopwatch("import")
     val eventTimer = EventTimer()
 
     val header = frameReader.getHeader()
     if (header == null) {
-      Log.e(TAG, "Backup is missing header!")
+      Log.e(TAG, "[import] Backup is missing header!")
       return ImportResult.Failure
     } else if (header.version > VERSION) {
-      Log.e(TAG, "Backup version is newer than we understand: ${header.version}")
+      Log.e(TAG, "[import] Backup version is newer than we understand: ${header.version}")
       return ImportResult.Failure
     }
 
-    SignalDatabase.rawDatabase.withinTransaction {
+    try {
-      SignalDatabase.recipients.clearAllDataForBackupRestore()
+      // Removing all the data from the various tables is *very* expensive (i.e. can take *several* minutes) if we don't do some pre-work.
-      SignalDatabase.distributionLists.clearAllDataForBackupRestore()
+      // SQLite optimizes deletes if there's no foreign keys, triggers, or WHERE clause, so that's the environment we're gonna create.
-      SignalDatabase.threads.clearAllDataForBackupRestore()
+
-      SignalDatabase.messages.clearAllDataForBackupRestore()
+      Log.d(TAG, "[import] Disabling foreign keys...")
-      SignalDatabase.attachments.clearAllDataForBackupRestore()
+      SignalDatabase.rawDatabase.forceForeignKeyConstraintsEnabled(false)
-      SignalDatabase.stickers.clearAllDataForBackupRestore()
+
-      SignalDatabase.reactions.clearAllDataForBackupRestore()
+      Log.d(TAG, "[import] Acquiring transaction...")
-      SignalDatabase.inAppPayments.clearAllDataForBackupRestore()
+      SignalDatabase.rawDatabase.beginTransaction()
-      SignalDatabase.chatColors.clearAllDataForBackupRestore()
+
-      SignalDatabase.calls.clearAllDataForBackup()
+      Log.d(TAG, "[import] Inside transaction.")
-      SignalDatabase.callLinks.clearAllDataForBackup()
+      stopwatch.split("get-transaction")
+
+      Log.d(TAG, "[import] --- Dropping all indices ---")
+      val indexMetadata = SignalDatabase.rawDatabase.getAllIndexDefinitions()
+      for (index in indexMetadata) {
+        Log.d(TAG, "[import] Dropping index ${index.name}...")
+        SignalDatabase.rawDatabase.execSQL("DROP INDEX IF EXISTS ${index.name}")
+      }
+      stopwatch.split("drop-indices")
+
+      if (cancellationSignal()) {
+        return ImportResult.Failure
+      }
+
+      Log.d(TAG, "[import] --- Dropping all triggers ---")
+      val triggerMetadata = SignalDatabase.rawDatabase.getAllTriggerDefinitions()
+      for (trigger in triggerMetadata) {
+        Log.d(TAG, "[import] Dropping trigger ${trigger.name}...")
+        SignalDatabase.rawDatabase.execSQL("DROP TRIGGER IF EXISTS ${trigger.name}")
+      }
+      stopwatch.split("drop-triggers")
+
+      if (cancellationSignal()) {
+        return ImportResult.Failure
+      }
+
+      Log.d(TAG, "[import] --- Recreating all tables ---")
+      val tableMetadata = SignalDatabase.rawDatabase.getAllTableDefinitions().filter { !it.name.startsWith(SearchTable.FTS_TABLE_NAME + "_") }
+      for (table in tableMetadata) {
+        Log.d(TAG, "[import] Dropping table ${table.name}...")
+        SignalDatabase.rawDatabase.execSQL("DROP TABLE IF EXISTS ${table.name}")
+
+        Log.d(TAG, "[import] Creating table ${table.name}...")
+        SignalDatabase.rawDatabase.execSQL(table.statement)
+      }
+
+      RecipientId.clearCache()
+      AppDependencies.recipientCache.clear()
+      AppDependencies.recipientCache.clearSelf()
+
+      stopwatch.split("drop-data")
+
+      if (cancellationSignal()) {
+        return ImportResult.Failure
+      }
 
       // Add back self after clearing data
       val selfId: RecipientId = SignalDatabase.recipients.getAndPossiblyMerge(selfData.aci, selfData.pni, selfData.e164, pniVerified = true, changeSelf = true)
       SignalDatabase.recipients.setProfileKey(selfId, selfData.profileKey)
       SignalDatabase.recipients.setProfileSharing(selfId, true)
 
-      eventTimer.emit("setup")
       val importState = ImportState(backupKey)
       val chatItemInserter: ChatItemArchiveImporter = ChatItemArchiveProcessor.beginImport(importState)
 
+      Log.d(TAG, "[import] Beginning to read frames.")
       val totalLength = frameReader.getStreamLength()
+      var frameCount = 0
       for (frame in frameReader) {
         when {
           frame.account != null -> {
             AccountDataArchiveProcessor.import(frame.account, selfId, importState)
             eventTimer.emit("account")
+            frameCount++
           }
 
           frame.recipient != null -> {
             RecipientArchiveProcessor.import(frame.recipient, importState)
             eventTimer.emit("recipient")
+            frameCount++
           }
 
           frame.chat != null -> {
             ChatArchiveProcessor.import(frame.chat, importState)
             eventTimer.emit("chat")
+            frameCount++
           }
 
           frame.adHocCall != null -> {
             AdHocCallArchiveProcessor.import(frame.adHocCall, importState)
             eventTimer.emit("call")
+            frameCount++
           }
 
           frame.stickerPack != null -> {
             StickerArchiveProcessor.import(frame.stickerPack)
             eventTimer.emit("sticker-pack")
+            frameCount++
           }
 
           frame.chatItem != null -> {
             chatItemInserter.import(frame.chatItem)
             eventTimer.emit("chatItem")
+            frameCount++
+
+            if (frameCount % 1000 == 0) {
+              if (cancellationSignal()) {
+                return ImportResult.Failure
+              }
+              Log.d(TAG, "Imported $frameCount frames so far.")
+            }
             // TODO if there's stuff in the stream after chatItems, we need to flush the inserter before going to the next phase
           }
 
@@ -479,16 +544,50 @@ object BackupRepository {
         eventTimer.emit("chatItem")
       }
 
+      stopwatch.split("frames")
+
+      Log.d(TAG, "[import] Rebuilding FTS index...")
+      SignalDatabase.messageSearch.rebuildIndex()
+
+      Log.d(TAG, "[import] --- Recreating indices ---")
+      for (index in indexMetadata) {
+        Log.d(TAG, "[import] Creating index ${index.name}...")
+        SignalDatabase.rawDatabase.execSQL(index.statement)
+      }
+      stopwatch.split("recreate-indices")
+
+      Log.d(TAG, "[import] --- Recreating triggers ---")
+      for (trigger in triggerMetadata) {
+        Log.d(TAG, "[import] Creating trigger ${trigger.name}...")
+        SignalDatabase.rawDatabase.execSQL(trigger.statement)
+      }
+      stopwatch.split("recreate-triggers")
+
+      Log.d(TAG, "[import] Updating threads...")
       importState.chatIdToLocalThreadId.values.forEach {
         SignalDatabase.threads.update(it, unarchive = false, allowDeletion = false)
       }
+      stopwatch.split("thread-updates")
+
+      val foreignKeyViolations = SignalDatabase.rawDatabase.getForeignKeyViolations()
+      if (foreignKeyViolations.isNotEmpty()) {
+        throw IllegalStateException("Foreign key check failed! Violations: $foreignKeyViolations")
+      }
+      stopwatch.split("fk-check")
+
+      SignalDatabase.rawDatabase.setTransactionSuccessful()
+    } finally {
+      if (SignalDatabase.rawDatabase.inTransaction()) {
+        SignalDatabase.rawDatabase.endTransaction()
+      }
+
+      Log.d(TAG, "[import] Re-enabling foreign keys...")
+      SignalDatabase.rawDatabase.forceForeignKeyConstraintsEnabled(true)
     }
 
     AppDependencies.recipientCache.clear()
     AppDependencies.recipientCache.warmUp()
 
-    Log.d(TAG, "import() ${eventTimer.stop().summary}")
-
     val groupJobs = SignalDatabase.groups.getGroups().use { groups ->
       groups
         .asSequence()
@@ -502,6 +601,10 @@ object BackupRepository {
         .toList()
     }
     AppDependencies.jobManager.addAll(groupJobs)
+    stopwatch.split("group-jobs")
+
+    Log.d(TAG, "[import] Finished! ${eventTimer.stop().summary}")
+    stopwatch.stop(TAG)
 
     return ImportResult.Success(backupTime = header.backupTimeMs)
   }
@@ -1091,6 +1194,10 @@ class ImportState(val backupKey: BackupKey) {
   val chatIdToLocalRecipientId: MutableMap<Long, RecipientId> = hashMapOf()
   val chatIdToBackupRecipientId: MutableMap<Long, Long> = hashMapOf()
   val remoteToLocalColorId: MutableMap<Long, Long> = hashMapOf()
+
+  fun requireLocalRecipientId(remoteId: Long): RecipientId {
+    return remoteToLocalRecipientId[remoteId] ?: throw IllegalArgumentException("There is no local recipientId for remote recipientId $remoteId!")
+  }
 }
 
 class BackupMetadata(

app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/AttachmentTableArchiveExtensions.kt

@@ -5,17 +5,10 @@
 
 package org.thoughtcrime.securesms.backup.v2.database
 
-import org.signal.core.util.SqlUtil
-import org.signal.core.util.deleteAll
 import org.thoughtcrime.securesms.attachments.Attachment
 import org.thoughtcrime.securesms.attachments.AttachmentId
 import org.thoughtcrime.securesms.database.AttachmentTable
 
-fun AttachmentTable.clearAllDataForBackupRestore() {
-  writableDatabase.deleteAll(AttachmentTable.TABLE_NAME)
-  SqlUtil.resetAutoIncrementValue(writableDatabase, AttachmentTable.TABLE_NAME)
-}
-
 fun AttachmentTable.restoreWallpaperAttachment(attachment: Attachment): AttachmentId? {
   return insertAttachmentsForMessage(AttachmentTable.WALLPAPER_MESSAGE_ID, listOf(attachment), emptyList()).values.firstOrNull()
 }

app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/CallLinkTableArchiveExtensions.kt

@@ -5,8 +5,6 @@
 
 package org.thoughtcrime.securesms.backup.v2.database
 
-import org.signal.core.util.SqlUtil
-import org.signal.core.util.deleteAll
 import org.signal.core.util.select
 import org.thoughtcrime.securesms.database.CallLinkTable
 
@@ -14,12 +12,8 @@ fun CallLinkTable.getCallLinksForBackup(): CallLinkArchiveExporter {
   val cursor = readableDatabase
     .select()
     .from(CallLinkTable.TABLE_NAME)
+    .where("${CallLinkTable.ROOT_KEY} NOT NULL")
     .run()
 
   return CallLinkArchiveExporter(cursor)
 }
-
-fun CallLinkTable.clearAllDataForBackup() {
-  writableDatabase.deleteAll(CallLinkTable.TABLE_NAME)
-  SqlUtil.resetAutoIncrementValue(writableDatabase, CallLinkTable.TABLE_NAME)
-}

app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/CallTableArchiveExtensions.kt

@@ -5,8 +5,6 @@
 
 package org.thoughtcrime.securesms.backup.v2.database
 
-import org.signal.core.util.SqlUtil
-import org.signal.core.util.deleteAll
 import org.signal.core.util.select
 import org.thoughtcrime.securesms.database.CallTable
 
@@ -19,8 +17,3 @@ fun CallTable.getAdhocCallsForBackup(): AdHocCallArchiveExporter {
       .run()
   )
 }
-
-fun CallTable.clearAllDataForBackup() {
-  writableDatabase.deleteAll(CallTable.TABLE_NAME)
-  SqlUtil.resetAutoIncrementValue(writableDatabase, CallTable.TABLE_NAME)
-}

app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/ChatColorsTableArchiveExtensions.kt

@@ -1,15 +0,0 @@
-/*
- * Copyright 2024 Signal Messenger, LLC
- * SPDX-License-Identifier: AGPL-3.0-only
- */
-
-package org.thoughtcrime.securesms.backup.v2.database
-
-import org.signal.core.util.SqlUtil
-import org.signal.core.util.deleteAll
-import org.thoughtcrime.securesms.database.ChatColorsTable
-
-fun ChatColorsTable.clearAllDataForBackupRestore() {
-  writableDatabase.deleteAll(ChatColorsTable.TABLE_NAME)
-  SqlUtil.resetAutoIncrementValue(writableDatabase, ChatColorsTable.TABLE_NAME)
-}

app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/DistributionListTablesArchiveExtensions.kt

@@ -5,8 +5,6 @@
 
 package org.thoughtcrime.securesms.backup.v2.database
 
-import org.signal.core.util.SqlUtil
-import org.signal.core.util.deleteAll
 import org.signal.core.util.select
 import org.signal.core.util.withinTransaction
 import org.thoughtcrime.securesms.backup.v2.exporters.DistributionListArchiveExporter
@@ -39,10 +37,3 @@ fun DistributionListTables.getMembersForBackup(id: DistributionListId): List<Rec
     DistributionListPrivacyMode.ALL_EXCEPT -> rawMembers
   }
 }
-
-fun DistributionListTables.clearAllDataForBackupRestore() {
-  writableDatabase.deleteAll(DistributionListTables.ListTable.TABLE_NAME)
-  writableDatabase.deleteAll(DistributionListTables.MembershipTable.TABLE_NAME)
-  SqlUtil.resetAutoIncrementValue(writableDatabase, DistributionListTables.ListTable.TABLE_NAME)
-  SqlUtil.resetAutoIncrementValue(writableDatabase, DistributionListTables.MembershipTable.TABLE_NAME)
-}

app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/InAppPaymentTableArchiveExtensions.kt

@@ -1,15 +0,0 @@
-/*
- * Copyright 2024 Signal Messenger, LLC
- * SPDX-License-Identifier: AGPL-3.0-only
- */
-
-package org.thoughtcrime.securesms.backup.v2.database
-
-import org.signal.core.util.SqlUtil
-import org.signal.core.util.deleteAll
-import org.thoughtcrime.securesms.database.InAppPaymentTable
-
-fun InAppPaymentTable.clearAllDataForBackupRestore() {
-  writableDatabase.deleteAll(InAppPaymentTable.TABLE_NAME)
-  SqlUtil.resetAutoIncrementValue(writableDatabase, InAppPaymentTable.TABLE_NAME)
-}

app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/MessageTableArchiveExtensions.kt

@@ -5,7 +5,6 @@
 
 package org.thoughtcrime.securesms.backup.v2.database
 
-import org.signal.core.util.SqlUtil
 import org.signal.core.util.select
 import org.thoughtcrime.securesms.backup.v2.ImportState
 import org.thoughtcrime.securesms.backup.v2.exporters.ChatItemArchiveExporter
@@ -78,8 +77,3 @@ fun MessageTable.getMessagesForBackup(db: SignalDatabase, backupTime: Long, medi
 fun MessageTable.createChatItemInserter(importState: ImportState): ChatItemArchiveImporter {
   return ChatItemArchiveImporter(writableDatabase, importState, 500)
 }
-
-fun MessageTable.clearAllDataForBackupRestore() {
-  writableDatabase.delete(MessageTable.TABLE_NAME, null, null)
-  SqlUtil.resetAutoIncrementValue(writableDatabase, MessageTable.TABLE_NAME)
-}

app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/ReactionTableArchiveExtensions.kt

@@ -1,15 +0,0 @@
-/*
- * Copyright 2023 Signal Messenger, LLC
- * SPDX-License-Identifier: AGPL-3.0-only
- */
-
-package org.thoughtcrime.securesms.backup.v2.database
-
-import org.signal.core.util.SqlUtil
-import org.signal.core.util.deleteAll
-import org.thoughtcrime.securesms.database.ReactionTable
-
-fun ReactionTable.clearAllDataForBackupRestore() {
-  writableDatabase.deleteAll(ReactionTable.TABLE_NAME)
-  SqlUtil.resetAutoIncrementValue(writableDatabase, ReactionTable.TABLE_NAME)
-}

app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/RecipientTableArchiveExtensions.kt

@@ -7,8 +7,6 @@ package org.thoughtcrime.securesms.backup.v2.database
 
 import android.content.ContentValues
 import org.signal.core.util.Base64
-import org.signal.core.util.SqlUtil
-import org.signal.core.util.deleteAll
 import org.signal.core.util.logging.Log
 import org.signal.core.util.nullIfBlank
 import org.signal.core.util.select
@@ -20,7 +18,6 @@ import org.thoughtcrime.securesms.backup.v2.proto.AccountData
 import org.thoughtcrime.securesms.database.GroupTable
 import org.thoughtcrime.securesms.database.RecipientTable
 import org.thoughtcrime.securesms.database.model.databaseprotos.RecipientExtras
-import org.thoughtcrime.securesms.dependencies.AppDependencies
 import org.thoughtcrime.securesms.keyvalue.SignalStore
 import org.thoughtcrime.securesms.profiles.ProfileName
 import org.thoughtcrime.securesms.recipients.RecipientId
@@ -123,15 +120,6 @@ fun RecipientTable.restoreSelfFromBackup(accountData: AccountData, selfId: Recip
     .run()
 }
 
-fun RecipientTable.clearAllDataForBackupRestore() {
-  writableDatabase.deleteAll(RecipientTable.TABLE_NAME)
-  SqlUtil.resetAutoIncrementValue(writableDatabase, RecipientTable.TABLE_NAME)
-
-  RecipientId.clearCache()
-  AppDependencies.recipientCache.clear()
-  AppDependencies.recipientCache.clearSelf()
-}
-
 fun RecipientTable.restoreReleaseNotes(): RecipientId {
   val releaseChannelId: RecipientId = insertReleaseChannelRecipient()
   SignalStore.releaseChannel.setReleaseChannelRecipientId(releaseChannelId)

app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/StickerTableArchiveExtensions.kt

@@ -1,15 +0,0 @@
-/*
- * Copyright 2023 Signal Messenger, LLC
- * SPDX-License-Identifier: AGPL-3.0-only
- */
-
-package org.thoughtcrime.securesms.backup.v2.database
-
-import org.signal.core.util.SqlUtil
-import org.signal.core.util.deleteAll
-import org.thoughtcrime.securesms.database.StickerTable
-
-fun StickerTable.clearAllDataForBackupRestore() {
-  writableDatabase.deleteAll(StickerTable.TABLE_NAME)
-  SqlUtil.resetAutoIncrementValue(writableDatabase, StickerTable.TABLE_NAME)
-}

app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/ThreadTableArchiveExtensions.kt

@@ -5,7 +5,6 @@
 
 package org.thoughtcrime.securesms.backup.v2.database
 
-import org.signal.core.util.SqlUtil
 import org.thoughtcrime.securesms.backup.v2.exporters.ChatArchiveExporter
 import org.thoughtcrime.securesms.database.RecipientTable
 import org.thoughtcrime.securesms.database.SignalDatabase
@@ -35,9 +34,3 @@ fun ThreadTable.getThreadsForBackup(db: SignalDatabase): ChatArchiveExporter {
 
   return ChatArchiveExporter(cursor, db)
 }
-
-fun ThreadTable.clearAllDataForBackupRestore() {
-  writableDatabase.delete(ThreadTable.TABLE_NAME, null, null)
-  SqlUtil.resetAutoIncrementValue(writableDatabase, ThreadTable.TABLE_NAME)
-  clearCache()
-}

app/src/main/java/org/thoughtcrime/securesms/backup/v2/exporters/CallLinkArchiveExporter.kt

@@ -6,7 +6,6 @@
 package org.thoughtcrime.securesms.backup.v2.database
 
 import android.database.Cursor
-import okio.ByteString
 import okio.ByteString.Companion.toByteString
 import org.signal.ringrtc.CallLinkState
 import org.thoughtcrime.securesms.backup.v2.ArchiveRecipient
@@ -32,8 +31,8 @@ class CallLinkArchiveExporter(private val cursor: Cursor) : Iterator<ArchiveReci
     return ArchiveRecipient(
       id = callLink.recipientId.toLong(),
       callLink = CallLink(
-        rootKey = callLink.credentials?.linkKeyBytes?.toByteString() ?: ByteString.EMPTY,
+        rootKey = callLink.credentials!!.linkKeyBytes.toByteString(),
-        adminKey = callLink.credentials?.adminPassBytes?.toByteString(),
+        adminKey = callLink.credentials.adminPassBytes?.toByteString(),
         name = callLink.state.name,
         expirationMs = try {
           callLink.state.expiration.toEpochMilli()

app/src/main/java/org/thoughtcrime/securesms/backup/v2/importer/AdHodCallArchiveImporter.kt

@@ -6,6 +6,7 @@
 package org.thoughtcrime.securesms.backup.v2.importer
 
 import org.signal.core.util.insertInto
+import org.signal.core.util.logging.Log
 import org.thoughtcrime.securesms.backup.v2.ImportState
 import org.thoughtcrime.securesms.backup.v2.proto.AdHocCall
 import org.thoughtcrime.securesms.database.CallTable
@@ -15,17 +16,25 @@ import org.thoughtcrime.securesms.database.SignalDatabase
  * Handles the importing of [AdHocCall] models into the local database.
  */
 object AdHodCallArchiveImporter {
+
+  private val TAG = Log.tag(AdHodCallArchiveImporter::class)
+
   fun import(call: AdHocCall, importState: ImportState) {
     val event = when (call.state) {
       AdHocCall.State.GENERIC -> CallTable.Event.GENERIC_GROUP_CALL
       AdHocCall.State.UNKNOWN_STATE -> CallTable.Event.GENERIC_GROUP_CALL
     }
 
+    val peer = importState.remoteToLocalRecipientId[call.recipientId] ?: run {
+      Log.w(TAG, "Failed to find matching recipientId for peer with remote recipientId ${call.recipientId}! Skipping.")
+      return
+    }
+
     SignalDatabase.writableDatabase
       .insertInto(CallTable.TABLE_NAME)
       .values(
         CallTable.CALL_ID to call.callId,
-        CallTable.PEER to importState.remoteToLocalRecipientId[call.recipientId]!!.serialize(),
+        CallTable.PEER to peer.serialize(),
         CallTable.TYPE to CallTable.Type.serialize(CallTable.Type.AD_HOC_CALL),
         CallTable.DIRECTION to CallTable.Direction.serialize(CallTable.Direction.OUTGOING),
         CallTable.EVENT to CallTable.Event.serialize(event),

app/src/main/java/org/thoughtcrime/securesms/backup/v2/importer/CallLinkArchiveImporter.kt

@@ -5,6 +5,8 @@
 
 package org.thoughtcrime.securesms.backup.v2.importer
 
+import org.signal.core.util.isEmpty
+import org.signal.core.util.logging.Log
 import org.signal.ringrtc.CallLinkRootKey
 import org.signal.ringrtc.CallLinkState
 import org.thoughtcrime.securesms.backup.v2.ArchiveCallLink
@@ -21,13 +23,21 @@ import java.time.Instant
  * Handles the importing of [ArchiveCallLink] models into the local database.
  */
 object CallLinkArchiveImporter {
+
+  private val TAG = Log.tag(CallLinkArchiveImporter::class)
+
   fun import(callLink: ArchiveCallLink): RecipientId? {
-    val rootKey: CallLinkRootKey
+    val rootKey: CallLinkRootKey = try {
-    try {
+      CallLinkRootKey(callLink.rootKey.toByteArray())
-      rootKey = CallLinkRootKey(callLink.rootKey.toByteArray())
     } catch (e: Exception) {
+      if (callLink.rootKey.isEmpty()) {
+        Log.w(TAG, "Missing root key!")
+      } else {
+        Log.w(TAG, "Failed to parse a non-empty root key!")
+      }
       return null
     }
+
     return SignalDatabase.callLinks.insertCallLink(
       CallLinkTable.CallLink(
         recipientId = RecipientId.UNKNOWN,

app/src/main/java/org/thoughtcrime/securesms/backup/v2/importer/ChatItemArchiveImporter.kt

@@ -875,7 +875,7 @@ class ChatItemArchiveImporter(
 
   private fun ContentValues.addQuote(quote: Quote) {
     this.put(MessageTable.QUOTE_ID, quote.targetSentTimestamp ?: MessageTable.QUOTE_TARGET_MISSING_ID)
-    this.put(MessageTable.QUOTE_AUTHOR, importState.remoteToLocalRecipientId[quote.authorId]!!.serialize())
+    this.put(MessageTable.QUOTE_AUTHOR, importState.requireLocalRecipientId(quote.authorId).serialize())
     this.put(MessageTable.QUOTE_BODY, quote.text?.body)
     this.put(MessageTable.QUOTE_TYPE, quote.type.toLocalQuoteType())
     this.put(MessageTable.QUOTE_BODY_RANGES, quote.text?.bodyRanges?.toLocalBodyRanges()?.encode())

app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/backup/InternalBackupPlaygroundViewModel.kt

@@ -536,6 +536,7 @@ class InternalBackupPlaygroundViewModel : ViewModel() {
     AppDependencies.jobManager.cancelAllInQueue("ArchiveAttachmentJobs_0")
     AppDependencies.jobManager.cancelAllInQueue("ArchiveAttachmentJobs_1")
     AppDependencies.jobManager.cancelAllInQueue("ArchiveThumbnailUploadJob")
+    AppDependencies.jobManager.cancelAllInQueue("BackupRestoreJob")
   }
 
   fun fetchRemoteBackupAndWritePlaintext(outputStream: OutputStream?) {

app/src/main/java/org/thoughtcrime/securesms/database/NameCollisionTables.kt

@@ -55,16 +55,12 @@ class NameCollisionTables(
 
     private val PROFILE_CHANGE_TIMEOUT = 1.days
 
-    fun createTables(db: SQLiteDatabase) {
+    val CREATE_TABLE = arrayOf(
-      db.execSQL(NameCollisionTable.CREATE_TABLE)
+      NameCollisionTable.CREATE_TABLE,
-      db.execSQL(NameCollisionMembershipTable.CREATE_TABLE)
+      NameCollisionMembershipTable.CREATE_TABLE
-    }
+    )
 
-    fun createIndexes(db: SQLiteDatabase) {
+    val CREATE_INDEXES = NameCollisionMembershipTable.CREATE_INDEXES
-      NameCollisionMembershipTable.CREATE_INDEXES.forEach {
-        db.execSQL(it)
-      }
-    }
   }
 
   /**

app/src/main/java/org/thoughtcrime/securesms/database/SQLiteDatabase.java

@@ -14,6 +14,7 @@ import androidx.sqlite.db.SupportSQLiteQuery;
 import net.zetetic.database.sqlcipher.SQLiteStatement;
 import net.zetetic.database.sqlcipher.SQLiteTransactionListener;
 
+import org.signal.core.util.logging.Log;
 import org.signal.core.util.tracing.Tracer;
 
 import java.io.IOException;

app/src/main/java/org/thoughtcrime/securesms/database/SearchTable.kt

@@ -180,10 +180,11 @@ class SearchTable(context: Context, databaseHelper: SignalDatabase) : DatabaseTa
         writableDatabase.withinTransaction { db ->
           db.execSQL(
             """
-            INSERT INTO $FTS_TABLE_NAME ($ID, $BODY)
+            INSERT INTO $FTS_TABLE_NAME ($ID, $BODY, $THREAD_ID)
                 SELECT
                   ${MessageTable.ID},
-                  ${MessageTable.BODY}
+                  ${MessageTable.BODY},
+                  ${MessageTable.THREAD_ID}
                 FROM
                   ${MessageTable.TABLE_NAME}
                 WHERE

app/src/main/java/org/thoughtcrime/securesms/database/SignalDatabase.kt

@@ -113,7 +113,7 @@ open class SignalDatabase(private val context: Application, databaseSecret: Data
     db.execSQL(CallLinkTable.CREATE_TABLE)
     db.execSQL(CallTable.CREATE_TABLE)
     db.execSQL(KyberPreKeyTable.CREATE_TABLE)
-    NameCollisionTables.createTables(db)
+    executeStatements(db, NameCollisionTables.CREATE_TABLE)
     db.execSQL(InAppPaymentTable.CREATE_TABLE)
     db.execSQL(InAppPaymentSubscriberTable.CREATE_TABLE)
     executeStatements(db, SearchTable.CREATE_TABLE)
@@ -144,12 +144,11 @@ open class SignalDatabase(private val context: Application, databaseSecret: Data
     executeStatements(db, ReactionTable.CREATE_INDEXES)
     executeStatements(db, KyberPreKeyTable.CREATE_INDEXES)
     executeStatements(db, ChatFolderTables.CREATE_INDEXES)
+    executeStatements(db, NameCollisionTables.CREATE_INDEXES)
 
     executeStatements(db, SearchTable.CREATE_TRIGGERS)
     executeStatements(db, MessageSendLogTables.CREATE_TRIGGERS)
 
-    NameCollisionTables.createIndexes(db)
-
     DistributionListTables.insertInitialDistributionListAtCreationTime(db)
     ChatFolderTables.insertInitialChatFoldersAtCreationTime(db)
 

app/src/main/java/org/thoughtcrime/securesms/database/SqlCipherErrorHandler.kt

@@ -165,16 +165,16 @@ class SqlCipherErrorHandler(private val databaseName: String) : DatabaseErrorHan
   }
 
   private fun attemptToClearFullTextSearchIndex(db: SQLiteDatabase) {
-    try {
+//    try {
-      try {
+//      try {
-        db.reopenReadWrite()
+//        db.reopenReadWrite()
-      } catch (e: Exception) {
+//      } catch (e: Exception) {
-        Log.w(TAG, "Failed to re-open as read-write!", e)
+//        Log.w(TAG, "Failed to re-open as read-write!", e)
-      }
+//      }
-      SignalDatabase.messageSearch.fullyResetTables(db, useTransaction = false)
+//      SignalDatabase.messageSearch.fullyResetTables(db, useTransaction = false)
-    } catch (e: Throwable) {
+//    } catch (e: Throwable) {
-      Log.w(TAG, "Failed to clear full text search index.", e)
+//      Log.w(TAG, "Failed to clear full text search index.", e)
-    }
+//    }
   }
 
   private sealed class DiagnosticResults(val logs: String) {

app/src/main/java/org/thoughtcrime/securesms/jobmanager/JobManager.java

@@ -64,7 +64,9 @@ public class JobManager implements ConstraintObserver.Notifier {
   public JobManager(@NonNull Application application, @NonNull Configuration configuration) {
     this.application   = application;
     this.configuration = configuration;
-    this.executor      = new FilteredExecutor(configuration.getExecutorFactory().newSingleThreadExecutor("signal-JobManager"), ThreadUtil::isMainThread);
+    this.executor      = new FilteredExecutor(configuration.getExecutorFactory().newSingleThreadExecutor("signal-JobManager"), () -> {
+      return ThreadUtil.isMainThread() || Thread.currentThread().getName().equals("Instr: org.thoughtcrime.securesms.testing.SignalTestRunner");
+    });
     this.jobTracker    = configuration.getJobTracker();
     this.jobController = new JobController(application,
                                            configuration.getJobStorage(),

app/src/main/java/org/thoughtcrime/securesms/jobs/BackupRestoreJob.kt

@@ -39,6 +39,7 @@ class BackupRestoreJob private constructor(parameters: Parameters) : BaseJob(par
       .addConstraint(NetworkConstraint.KEY)
       .setMaxAttempts(Parameters.UNLIMITED)
       .setMaxInstancesForFactory(1)
+      .setQueue("BackupRestoreJob")
       .build()
   )
 
@@ -85,6 +86,10 @@ class BackupRestoreJob private constructor(parameters: Parameters) : BaseJob(par
       throw IOException()
     }
 
+    if (isCanceled) {
+      return
+    }
+
     controller.update(
       title = context.getString(R.string.BackupProgressService_title),
       progress = 0f,
@@ -93,7 +98,7 @@ class BackupRestoreJob private constructor(parameters: Parameters) : BaseJob(par
 
     val self = Recipient.self()
     val selfData = BackupRepository.SelfData(self.aci.get(), self.pni.get(), self.e164.get(), ProfileKey(self.profileKey))
-    BackupRepository.import(length = tempBackupFile.length(), inputStreamFactory = tempBackupFile::inputStream, selfData = selfData, plaintext = false)
+    BackupRepository.import(length = tempBackupFile.length(), inputStreamFactory = tempBackupFile::inputStream, selfData = selfData, plaintext = false, cancellationSignal = { isCanceled })
 
     SignalStore.backup.restoreState = RestoreState.RESTORING_MEDIA
   }

core-util-jvm/src/main/java/org/signal/core/util/ProtoExtensions.kt

@@ -20,6 +20,10 @@ import java.util.LinkedList
 
 private const val TAG = "ProtoExtension"
 
+fun ByteString?.isEmpty(): Boolean {
+  return this == null || this.size == 0
+}
+
 fun ByteString?.isNotEmpty(): Boolean {
   return this != null && this.size > 0
 }

core-util/src/main/java/org/signal/core/util/SQLiteDatabaseExtensions.kt

@@ -6,6 +6,12 @@ import android.database.sqlite.SQLiteDatabase
 import androidx.core.content.contentValuesOf
 import androidx.sqlite.db.SupportSQLiteDatabase
 import androidx.sqlite.db.SupportSQLiteQueryBuilder
+import org.signal.core.util.SqlUtil.ForeignKeyViolation
+import org.signal.core.util.logging.Log
+import kotlin.time.Duration
+import kotlin.time.Duration.Companion.seconds
+
+private val TAG = "SQLiteDatabaseExtensions"
 
 /**
  * Begins a transaction on the `this` database, runs the provided [block] providing the `this` value as it's argument
@@ -46,7 +52,11 @@ fun SupportSQLiteDatabase.getAllTables(): List<String> {
  * Returns a list of objects that represent the table definitions in the database. Basically the table name and then the SQL that was used to create it.
  */
 fun SupportSQLiteDatabase.getAllTableDefinitions(): List<CreateStatement> {
-  return this.query("SELECT name, sql FROM sqlite_schema WHERE type = 'table' AND sql NOT NULL AND name != 'sqlite_sequence'")
+  return this
+    .select("name", "sql")
+    .from("sqlite_schema")
+    .where("type = ? AND sql NOT NULL AND name != ?", "table", "sqlite_sequence")
+    .run()
     .readToList { cursor ->
       CreateStatement(
         name = cursor.requireNonNullString("name"),
@@ -61,7 +71,11 @@ fun SupportSQLiteDatabase.getAllTableDefinitions(): List<CreateStatement> {
  * Returns a list of objects that represent the index definitions in the database. Basically the index name and then the SQL that was used to create it.
  */
 fun SupportSQLiteDatabase.getAllIndexDefinitions(): List<CreateStatement> {
-  return this.query("SELECT name, sql FROM sqlite_schema WHERE type = 'index' AND sql NOT NULL")
+  return this
+    .select("name", "sql")
+    .from("sqlite_schema")
+    .where("type = ? AND sql NOT NULL", "index")
+    .run()
     .readToList { cursor ->
       CreateStatement(
         name = cursor.requireNonNullString("name"),
@@ -71,6 +85,24 @@ fun SupportSQLiteDatabase.getAllIndexDefinitions(): List<CreateStatement> {
     .sortedBy { it.name }
 }
 
+/**
+ * Retrieves the names of all triggers, sorted alphabetically.
+ */
+fun SupportSQLiteDatabase.getAllTriggerDefinitions(): List<CreateStatement> {
+  return this
+    .select("name", "sql")
+    .from("sqlite_schema")
+    .where("type = ? AND sql NOT NULL", "trigger")
+    .run()
+    .readToList {
+      CreateStatement(
+        name = it.requireNonNullString("name"),
+        statement = it.requireNonNullString("sql")
+      )
+    }
+    .sortedBy { it.name }
+}
+
 fun SupportSQLiteDatabase.getForeignKeys(): List<ForeignKeyConstraint> {
   return SqlUtil.getAllTables(this)
     .map { table ->
@@ -93,6 +125,24 @@ fun SupportSQLiteDatabase.areForeignKeyConstraintsEnabled(): Boolean {
   }
 }
 
+/**
+ * Provides a list of all foreign key violations present.
+ * If a [targetTable] is specified, results will be limited to that table specifically.
+ * Otherwise, the check will be performed across all tables.
+ */
+@JvmOverloads
+fun SupportSQLiteDatabase.getForeignKeyViolations(targetTable: String? = null): List<ForeignKeyViolation> {
+  return SqlUtil.getForeignKeyViolations(this, targetTable)
+}
+
+/**
+ * For tables that have an autoincrementing primary key, this will reset the key to start back at 1.
+ * IMPORTANT: This is quite dangerous! Only do this if you're effectively resetting the entire database.
+ */
+fun SupportSQLiteDatabase.resetAutoIncrementValue(targetTable: String) {
+  SqlUtil.resetAutoIncrementValue(this, targetTable)
+}
+
 /**
  * Does a full WAL checkpoint (TRUNCATE mode, where the log is for sure flushed and the log is zero'd out).
  * Will try up to [maxAttempts] times. Can technically fail if the database is too active and the checkpoint
@@ -132,6 +182,22 @@ fun SupportSQLiteDatabase.getIndexes(): List<Index> {
   }
 }
 
+fun SupportSQLiteDatabase.forceForeignKeyConstraintsEnabled(enabled: Boolean, timeout: Duration = 10.seconds) {
+  val startTime = System.currentTimeMillis()
+  while (true) {
+    try {
+      this.setForeignKeyConstraintsEnabled(enabled)
+      break
+    } catch (e: IllegalStateException) {
+      if (System.currentTimeMillis() - startTime > timeout.inWholeMilliseconds) {
+        throw IllegalStateException("Failed to force foreign keys to '$enabled' within the timeout of $timeout", e)
+      }
+      Log.w(TAG, "Failed to set foreign keys because we're in a transaction. Waiting 100ms then trying again.")
+      ThreadUtil.sleep(100)
+    }
+  }
+}
+
 /**
  * Checks if a row exists that matches the query.
  */