Get a big backupV2 import fully working.

2025-12-20 11:08:31 +00:00 · 2024-10-10 16:01:48 -04:00
parent 0d878ca70a
commit a90df1e262
27 changed files with 261 additions and 193 deletions
--- a/app/src/androidTest/java/org/thoughtcrime/securesms/backup/v2/ArchiveImportExportTests.kt
+++ b/app/src/androidTest/java/org/thoughtcrime/securesms/backup/v2/ArchiveImportExportTests.kt
@@ -14,7 +14,6 @@ import org.junit.Before
 import org.junit.Test
 import org.junit.runner.RunWith
 import org.signal.core.util.Base64
-import org.signal.core.util.SqlUtil
 import org.signal.core.util.logging.Log
 import org.signal.core.util.readFully
 import org.signal.libsignal.messagebackup.ComparableBackup
@@ -22,12 +21,9 @@ import org.signal.libsignal.messagebackup.MessageBackup
 import org.signal.libsignal.zkgroup.profiles.ProfileKey
 import org.thoughtcrime.securesms.backup.v2.proto.Frame
 import org.thoughtcrime.securesms.backup.v2.stream.PlainTextBackupReader
-import org.thoughtcrime.securesms.database.DistributionListTables
 import org.thoughtcrime.securesms.database.KeyValueDatabase
-import org.thoughtcrime.securesms.database.SignalDatabase
 import org.thoughtcrime.securesms.dependencies.AppDependencies
 import org.thoughtcrime.securesms.keyvalue.SignalStore
-import org.thoughtcrime.securesms.recipients.RecipientId
 import org.whispersystems.signalservice.api.kbs.MasterKey
 import org.whispersystems.signalservice.api.push.ServiceId
 import java.io.ByteArrayInputStream
@@ -263,21 +259,7 @@ class ArchiveImportExportTests {
  }

  private fun resetAllData() {
-    // Need to delete these first to prevent foreign key crash
-    SignalDatabase.rawDatabase.execSQL("DELETE FROM ${DistributionListTables.ListTable.TABLE_NAME}")
-    SignalDatabase.rawDatabase.execSQL("DELETE FROM ${DistributionListTables.MembershipTable.TABLE_NAME}")
-
-    SqlUtil.getAllTables(SignalDatabase.rawDatabase)
-      .filterNot { it.contains("sqlite") || it.contains("fts") || it.startsWith("emoji_search_") } // If we delete these we'll corrupt the DB
-      .sorted()
-      .forEach { table ->
-        SignalDatabase.rawDatabase.execSQL("DELETE FROM $table")
-        SqlUtil.resetAutoIncrementValue(SignalDatabase.rawDatabase, table)
-      }
-
-    AppDependencies.recipientCache.clear()
-    AppDependencies.recipientCache.clearSelf()
-    RecipientId.clearCache()
+    // All the main database stuff is reset as a normal part of importing

    KeyValueDatabase.getInstance(AppDependencies.application).clear()
    SignalStore.resetCache()
--- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/BackupRepository.kt
+++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/BackupRepository.kt
@@ -11,9 +11,15 @@ import kotlinx.coroutines.withContext
 import org.greenrobot.eventbus.EventBus
 import org.signal.core.util.Base64
 import org.signal.core.util.EventTimer
+import org.signal.core.util.Stopwatch
 import org.signal.core.util.concurrent.LimitedWorker
 import org.signal.core.util.concurrent.SignalExecutors
+import org.signal.core.util.forceForeignKeyConstraintsEnabled
 import org.signal.core.util.fullWalCheckpoint
+import org.signal.core.util.getAllIndexDefinitions
+import org.signal.core.util.getAllTableDefinitions
+import org.signal.core.util.getAllTriggerDefinitions
+import org.signal.core.util.getForeignKeyViolations
 import org.signal.core.util.logging.Log
 import org.signal.core.util.stream.NonClosingOutputStream
 import org.signal.core.util.withinTransaction
@@ -27,8 +33,6 @@ import org.thoughtcrime.securesms.attachments.Attachment
 import org.thoughtcrime.securesms.attachments.AttachmentId
 import org.thoughtcrime.securesms.attachments.Cdn
 import org.thoughtcrime.securesms.attachments.DatabaseAttachment
-import org.thoughtcrime.securesms.backup.v2.database.clearAllDataForBackup
-import org.thoughtcrime.securesms.backup.v2.database.clearAllDataForBackupRestore
 import org.thoughtcrime.securesms.backup.v2.importer.ChatItemArchiveImporter
 import org.thoughtcrime.securesms.backup.v2.processor.AccountDataArchiveProcessor
 import org.thoughtcrime.securesms.backup.v2.processor.AdHocCallArchiveProcessor
@@ -49,6 +53,7 @@ import org.thoughtcrime.securesms.crypto.AttachmentSecretProvider
 import org.thoughtcrime.securesms.crypto.DatabaseSecretProvider
 import org.thoughtcrime.securesms.database.AttachmentTable
 import org.thoughtcrime.securesms.database.KeyValueDatabase
+import org.thoughtcrime.securesms.database.SearchTable
 import org.thoughtcrime.securesms.database.SignalDatabase
 import org.thoughtcrime.securesms.database.model.InAppPaymentSubscriberRecord
 import org.thoughtcrime.securesms.dependencies.AppDependencies
@@ -375,11 +380,11 @@ object BackupRepository {
    }

    return frameReader.use { reader ->
-      import(backupKey, reader, selfData)
+      import(backupKey, reader, selfData, cancellationSignal = { false })
    }
  }

-  fun import(length: Long, inputStreamFactory: () -> InputStream, selfData: SelfData, plaintext: Boolean = false): ImportResult {
+  fun import(length: Long, inputStreamFactory: () -> InputStream, selfData: SelfData, plaintext: Boolean = false, cancellationSignal: () -> Boolean = { false }): ImportResult {
    val backupKey = SignalStore.svr.getOrCreateMasterKey().deriveBackupKey()

    val frameReader = if (plaintext) {
@@ -394,79 +399,139 @@ object BackupRepository {
    }

    return frameReader.use { reader ->
-      import(backupKey, reader, selfData)
+      import(backupKey, reader, selfData, cancellationSignal)
    }
  }

  private fun import(
    backupKey: BackupKey,
    frameReader: BackupImportReader,
-    selfData: SelfData
+    selfData: SelfData,
+    cancellationSignal: () -> Boolean
  ): ImportResult {
+    val stopwatch = Stopwatch("import")
    val eventTimer = EventTimer()

    val header = frameReader.getHeader()
    if (header == null) {
-      Log.e(TAG, "Backup is missing header!")
+      Log.e(TAG, "[import] Backup is missing header!")
      return ImportResult.Failure
    } else if (header.version > VERSION) {
-      Log.e(TAG, "Backup version is newer than we understand: ${header.version}")
+      Log.e(TAG, "[import] Backup version is newer than we understand: ${header.version}")
      return ImportResult.Failure
    }

-    SignalDatabase.rawDatabase.withinTransaction {
-      SignalDatabase.recipients.clearAllDataForBackupRestore()
-      SignalDatabase.distributionLists.clearAllDataForBackupRestore()
-      SignalDatabase.threads.clearAllDataForBackupRestore()
-      SignalDatabase.messages.clearAllDataForBackupRestore()
-      SignalDatabase.attachments.clearAllDataForBackupRestore()
-      SignalDatabase.stickers.clearAllDataForBackupRestore()
-      SignalDatabase.reactions.clearAllDataForBackupRestore()
-      SignalDatabase.inAppPayments.clearAllDataForBackupRestore()
-      SignalDatabase.chatColors.clearAllDataForBackupRestore()
-      SignalDatabase.calls.clearAllDataForBackup()
-      SignalDatabase.callLinks.clearAllDataForBackup()
+    try {
+      // Removing all the data from the various tables is *very* expensive (i.e. can take *several* minutes) if we don't do some pre-work.
+      // SQLite optimizes deletes if there's no foreign keys, triggers, or WHERE clause, so that's the environment we're gonna create.
+
+      Log.d(TAG, "[import] Disabling foreign keys...")
+      SignalDatabase.rawDatabase.forceForeignKeyConstraintsEnabled(false)
+
+      Log.d(TAG, "[import] Acquiring transaction...")
+      SignalDatabase.rawDatabase.beginTransaction()
+
+      Log.d(TAG, "[import] Inside transaction.")
+      stopwatch.split("get-transaction")
+
+      Log.d(TAG, "[import] --- Dropping all indices ---")
+      val indexMetadata = SignalDatabase.rawDatabase.getAllIndexDefinitions()
+      for (index in indexMetadata) {
+        Log.d(TAG, "[import] Dropping index ${index.name}...")
+        SignalDatabase.rawDatabase.execSQL("DROP INDEX IF EXISTS ${index.name}")
+      }
+      stopwatch.split("drop-indices")
+
+      if (cancellationSignal()) {
+        return ImportResult.Failure
+      }
+
+      Log.d(TAG, "[import] --- Dropping all triggers ---")
+      val triggerMetadata = SignalDatabase.rawDatabase.getAllTriggerDefinitions()
+      for (trigger in triggerMetadata) {
+        Log.d(TAG, "[import] Dropping trigger ${trigger.name}...")
+        SignalDatabase.rawDatabase.execSQL("DROP TRIGGER IF EXISTS ${trigger.name}")
+      }
+      stopwatch.split("drop-triggers")
+
+      if (cancellationSignal()) {
+        return ImportResult.Failure
+      }
+
+      Log.d(TAG, "[import] --- Recreating all tables ---")
+      val tableMetadata = SignalDatabase.rawDatabase.getAllTableDefinitions().filter { !it.name.startsWith(SearchTable.FTS_TABLE_NAME + "_") }
+      for (table in tableMetadata) {
+        Log.d(TAG, "[import] Dropping table ${table.name}...")
+        SignalDatabase.rawDatabase.execSQL("DROP TABLE IF EXISTS ${table.name}")
+
+        Log.d(TAG, "[import] Creating table ${table.name}...")
+        SignalDatabase.rawDatabase.execSQL(table.statement)
+      }
+
+      RecipientId.clearCache()
+      AppDependencies.recipientCache.clear()
+      AppDependencies.recipientCache.clearSelf()
+
+      stopwatch.split("drop-data")
+
+      if (cancellationSignal()) {
+        return ImportResult.Failure
+      }

      // Add back self after clearing data
      val selfId: RecipientId = SignalDatabase.recipients.getAndPossiblyMerge(selfData.aci, selfData.pni, selfData.e164, pniVerified = true, changeSelf = true)
      SignalDatabase.recipients.setProfileKey(selfId, selfData.profileKey)
      SignalDatabase.recipients.setProfileSharing(selfId, true)

-      eventTimer.emit("setup")
      val importState = ImportState(backupKey)
      val chatItemInserter: ChatItemArchiveImporter = ChatItemArchiveProcessor.beginImport(importState)

+      Log.d(TAG, "[import] Beginning to read frames.")
      val totalLength = frameReader.getStreamLength()
+      var frameCount = 0
      for (frame in frameReader) {
        when {
          frame.account != null -> {
            AccountDataArchiveProcessor.import(frame.account, selfId, importState)
            eventTimer.emit("account")
+            frameCount++
          }

          frame.recipient != null -> {
            RecipientArchiveProcessor.import(frame.recipient, importState)
            eventTimer.emit("recipient")
+            frameCount++
          }

          frame.chat != null -> {
            ChatArchiveProcessor.import(frame.chat, importState)
            eventTimer.emit("chat")
+            frameCount++
          }

          frame.adHocCall != null -> {
            AdHocCallArchiveProcessor.import(frame.adHocCall, importState)
            eventTimer.emit("call")
+            frameCount++
          }

          frame.stickerPack != null -> {
            StickerArchiveProcessor.import(frame.stickerPack)
            eventTimer.emit("sticker-pack")
+            frameCount++
          }

          frame.chatItem != null -> {
            chatItemInserter.import(frame.chatItem)
            eventTimer.emit("chatItem")
+            frameCount++
+
+            if (frameCount % 1000 == 0) {
+              if (cancellationSignal()) {
+                return ImportResult.Failure
+              }
+              Log.d(TAG, "Imported $frameCount frames so far.")
+            }
            // TODO if there's stuff in the stream after chatItems, we need to flush the inserter before going to the next phase
          }

@@ -479,16 +544,50 @@ object BackupRepository {
        eventTimer.emit("chatItem")
      }

+      stopwatch.split("frames")
+
+      Log.d(TAG, "[import] Rebuilding FTS index...")
+      SignalDatabase.messageSearch.rebuildIndex()
+
+      Log.d(TAG, "[import] --- Recreating indices ---")
+      for (index in indexMetadata) {
+        Log.d(TAG, "[import] Creating index ${index.name}...")
+        SignalDatabase.rawDatabase.execSQL(index.statement)
+      }
+      stopwatch.split("recreate-indices")
+
+      Log.d(TAG, "[import] --- Recreating triggers ---")
+      for (trigger in triggerMetadata) {
+        Log.d(TAG, "[import] Creating trigger ${trigger.name}...")
+        SignalDatabase.rawDatabase.execSQL(trigger.statement)
+      }
+      stopwatch.split("recreate-triggers")
+
+      Log.d(TAG, "[import] Updating threads...")
      importState.chatIdToLocalThreadId.values.forEach {
        SignalDatabase.threads.update(it, unarchive = false, allowDeletion = false)
      }
+      stopwatch.split("thread-updates")
+
+      val foreignKeyViolations = SignalDatabase.rawDatabase.getForeignKeyViolations()
+      if (foreignKeyViolations.isNotEmpty()) {
+        throw IllegalStateException("Foreign key check failed! Violations: $foreignKeyViolations")
+      }
+      stopwatch.split("fk-check")
+
+      SignalDatabase.rawDatabase.setTransactionSuccessful()
+    } finally {
+      if (SignalDatabase.rawDatabase.inTransaction()) {
+        SignalDatabase.rawDatabase.endTransaction()
+      }
+
+      Log.d(TAG, "[import] Re-enabling foreign keys...")
+      SignalDatabase.rawDatabase.forceForeignKeyConstraintsEnabled(true)
    }

    AppDependencies.recipientCache.clear()
    AppDependencies.recipientCache.warmUp()

-    Log.d(TAG, "import() ${eventTimer.stop().summary}")
-
    val groupJobs = SignalDatabase.groups.getGroups().use { groups ->
      groups
        .asSequence()
@@ -502,6 +601,10 @@ object BackupRepository {
        .toList()
    }
    AppDependencies.jobManager.addAll(groupJobs)
+    stopwatch.split("group-jobs")
+
+    Log.d(TAG, "[import] Finished! ${eventTimer.stop().summary}")
+    stopwatch.stop(TAG)

    return ImportResult.Success(backupTime = header.backupTimeMs)
  }
@@ -1091,6 +1194,10 @@ class ImportState(val backupKey: BackupKey) {
  val chatIdToLocalRecipientId: MutableMap<Long, RecipientId> = hashMapOf()
  val chatIdToBackupRecipientId: MutableMap<Long, Long> = hashMapOf()
  val remoteToLocalColorId: MutableMap<Long, Long> = hashMapOf()
+
+  fun requireLocalRecipientId(remoteId: Long): RecipientId {
+    return remoteToLocalRecipientId[remoteId] ?: throw IllegalArgumentException("There is no local recipientId for remote recipientId $remoteId!")
+  }
 }

 class BackupMetadata(
--- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/AttachmentTableArchiveExtensions.kt
+++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/AttachmentTableArchiveExtensions.kt
@@ -5,17 +5,10 @@

 package org.thoughtcrime.securesms.backup.v2.database

-import org.signal.core.util.SqlUtil
-import org.signal.core.util.deleteAll
 import org.thoughtcrime.securesms.attachments.Attachment
 import org.thoughtcrime.securesms.attachments.AttachmentId
 import org.thoughtcrime.securesms.database.AttachmentTable

-fun AttachmentTable.clearAllDataForBackupRestore() {
-  writableDatabase.deleteAll(AttachmentTable.TABLE_NAME)
-  SqlUtil.resetAutoIncrementValue(writableDatabase, AttachmentTable.TABLE_NAME)
-}
-
 fun AttachmentTable.restoreWallpaperAttachment(attachment: Attachment): AttachmentId? {
  return insertAttachmentsForMessage(AttachmentTable.WALLPAPER_MESSAGE_ID, listOf(attachment), emptyList()).values.firstOrNull()
 }
--- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/CallLinkTableArchiveExtensions.kt
+++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/CallLinkTableArchiveExtensions.kt
@@ -5,8 +5,6 @@

 package org.thoughtcrime.securesms.backup.v2.database

-import org.signal.core.util.SqlUtil
-import org.signal.core.util.deleteAll
 import org.signal.core.util.select
 import org.thoughtcrime.securesms.database.CallLinkTable

@@ -14,12 +12,8 @@ fun CallLinkTable.getCallLinksForBackup(): CallLinkArchiveExporter {
  val cursor = readableDatabase
    .select()
    .from(CallLinkTable.TABLE_NAME)
+    .where("${CallLinkTable.ROOT_KEY} NOT NULL")
    .run()

  return CallLinkArchiveExporter(cursor)
 }
-
-fun CallLinkTable.clearAllDataForBackup() {
-  writableDatabase.deleteAll(CallLinkTable.TABLE_NAME)
-  SqlUtil.resetAutoIncrementValue(writableDatabase, CallLinkTable.TABLE_NAME)
-}
--- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/CallTableArchiveExtensions.kt
+++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/CallTableArchiveExtensions.kt
@@ -5,8 +5,6 @@

 package org.thoughtcrime.securesms.backup.v2.database

-import org.signal.core.util.SqlUtil
-import org.signal.core.util.deleteAll
 import org.signal.core.util.select
 import org.thoughtcrime.securesms.database.CallTable

@@ -19,8 +17,3 @@ fun CallTable.getAdhocCallsForBackup(): AdHocCallArchiveExporter {
      .run()
  )
 }
-
-fun CallTable.clearAllDataForBackup() {
-  writableDatabase.deleteAll(CallTable.TABLE_NAME)
-  SqlUtil.resetAutoIncrementValue(writableDatabase, CallTable.TABLE_NAME)
-}
--- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/ChatColorsTableArchiveExtensions.kt
+++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/ChatColorsTableArchiveExtensions.kt
@@ -1,15 +0,0 @@
-/*
- * Copyright 2024 Signal Messenger, LLC
- * SPDX-License-Identifier: AGPL-3.0-only
- */
-
-package org.thoughtcrime.securesms.backup.v2.database
-
-import org.signal.core.util.SqlUtil
-import org.signal.core.util.deleteAll
-import org.thoughtcrime.securesms.database.ChatColorsTable
-
-fun ChatColorsTable.clearAllDataForBackupRestore() {
-  writableDatabase.deleteAll(ChatColorsTable.TABLE_NAME)
-  SqlUtil.resetAutoIncrementValue(writableDatabase, ChatColorsTable.TABLE_NAME)
-}
--- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/DistributionListTablesArchiveExtensions.kt
+++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/DistributionListTablesArchiveExtensions.kt
@@ -5,8 +5,6 @@

 package org.thoughtcrime.securesms.backup.v2.database

-import org.signal.core.util.SqlUtil
-import org.signal.core.util.deleteAll
 import org.signal.core.util.select
 import org.signal.core.util.withinTransaction
 import org.thoughtcrime.securesms.backup.v2.exporters.DistributionListArchiveExporter
@@ -39,10 +37,3 @@ fun DistributionListTables.getMembersForBackup(id: DistributionListId): List<Rec
    DistributionListPrivacyMode.ALL_EXCEPT -> rawMembers
  }
 }
-
-fun DistributionListTables.clearAllDataForBackupRestore() {
-  writableDatabase.deleteAll(DistributionListTables.ListTable.TABLE_NAME)
-  writableDatabase.deleteAll(DistributionListTables.MembershipTable.TABLE_NAME)
-  SqlUtil.resetAutoIncrementValue(writableDatabase, DistributionListTables.ListTable.TABLE_NAME)
-  SqlUtil.resetAutoIncrementValue(writableDatabase, DistributionListTables.MembershipTable.TABLE_NAME)
-}
--- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/InAppPaymentTableArchiveExtensions.kt
+++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/InAppPaymentTableArchiveExtensions.kt
@@ -1,15 +0,0 @@
-/*
- * Copyright 2024 Signal Messenger, LLC
- * SPDX-License-Identifier: AGPL-3.0-only
- */
-
-package org.thoughtcrime.securesms.backup.v2.database
-
-import org.signal.core.util.SqlUtil
-import org.signal.core.util.deleteAll
-import org.thoughtcrime.securesms.database.InAppPaymentTable
-
-fun InAppPaymentTable.clearAllDataForBackupRestore() {
-  writableDatabase.deleteAll(InAppPaymentTable.TABLE_NAME)
-  SqlUtil.resetAutoIncrementValue(writableDatabase, InAppPaymentTable.TABLE_NAME)
-}
--- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/MessageTableArchiveExtensions.kt
+++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/MessageTableArchiveExtensions.kt
@@ -5,7 +5,6 @@

 package org.thoughtcrime.securesms.backup.v2.database

-import org.signal.core.util.SqlUtil
 import org.signal.core.util.select
 import org.thoughtcrime.securesms.backup.v2.ImportState
 import org.thoughtcrime.securesms.backup.v2.exporters.ChatItemArchiveExporter
@@ -78,8 +77,3 @@ fun MessageTable.getMessagesForBackup(db: SignalDatabase, backupTime: Long, medi
 fun MessageTable.createChatItemInserter(importState: ImportState): ChatItemArchiveImporter {
  return ChatItemArchiveImporter(writableDatabase, importState, 500)
 }
-
-fun MessageTable.clearAllDataForBackupRestore() {
-  writableDatabase.delete(MessageTable.TABLE_NAME, null, null)
-  SqlUtil.resetAutoIncrementValue(writableDatabase, MessageTable.TABLE_NAME)
-}
--- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/ReactionTableArchiveExtensions.kt
+++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/ReactionTableArchiveExtensions.kt
@@ -1,15 +0,0 @@
-/*
- * Copyright 2023 Signal Messenger, LLC
- * SPDX-License-Identifier: AGPL-3.0-only
- */
-
-package org.thoughtcrime.securesms.backup.v2.database
-
-import org.signal.core.util.SqlUtil
-import org.signal.core.util.deleteAll
-import org.thoughtcrime.securesms.database.ReactionTable
-
-fun ReactionTable.clearAllDataForBackupRestore() {
-  writableDatabase.deleteAll(ReactionTable.TABLE_NAME)
-  SqlUtil.resetAutoIncrementValue(writableDatabase, ReactionTable.TABLE_NAME)
-}
--- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/RecipientTableArchiveExtensions.kt
+++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/RecipientTableArchiveExtensions.kt
@@ -7,8 +7,6 @@ package org.thoughtcrime.securesms.backup.v2.database

 import android.content.ContentValues
 import org.signal.core.util.Base64
-import org.signal.core.util.SqlUtil
-import org.signal.core.util.deleteAll
 import org.signal.core.util.logging.Log
 import org.signal.core.util.nullIfBlank
 import org.signal.core.util.select
@@ -20,7 +18,6 @@ import org.thoughtcrime.securesms.backup.v2.proto.AccountData
 import org.thoughtcrime.securesms.database.GroupTable
 import org.thoughtcrime.securesms.database.RecipientTable
 import org.thoughtcrime.securesms.database.model.databaseprotos.RecipientExtras
-import org.thoughtcrime.securesms.dependencies.AppDependencies
 import org.thoughtcrime.securesms.keyvalue.SignalStore
 import org.thoughtcrime.securesms.profiles.ProfileName
 import org.thoughtcrime.securesms.recipients.RecipientId
@@ -123,15 +120,6 @@ fun RecipientTable.restoreSelfFromBackup(accountData: AccountData, selfId: Recip
    .run()
 }

-fun RecipientTable.clearAllDataForBackupRestore() {
-  writableDatabase.deleteAll(RecipientTable.TABLE_NAME)
-  SqlUtil.resetAutoIncrementValue(writableDatabase, RecipientTable.TABLE_NAME)
-
-  RecipientId.clearCache()
-  AppDependencies.recipientCache.clear()
-  AppDependencies.recipientCache.clearSelf()
-}
-
 fun RecipientTable.restoreReleaseNotes(): RecipientId {
  val releaseChannelId: RecipientId = insertReleaseChannelRecipient()
  SignalStore.releaseChannel.setReleaseChannelRecipientId(releaseChannelId)
--- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/StickerTableArchiveExtensions.kt
+++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/StickerTableArchiveExtensions.kt
@@ -1,15 +0,0 @@
-/*
- * Copyright 2023 Signal Messenger, LLC
- * SPDX-License-Identifier: AGPL-3.0-only
- */
-
-package org.thoughtcrime.securesms.backup.v2.database
-
-import org.signal.core.util.SqlUtil
-import org.signal.core.util.deleteAll
-import org.thoughtcrime.securesms.database.StickerTable
-
-fun StickerTable.clearAllDataForBackupRestore() {
-  writableDatabase.deleteAll(StickerTable.TABLE_NAME)
-  SqlUtil.resetAutoIncrementValue(writableDatabase, StickerTable.TABLE_NAME)
-}
--- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/ThreadTableArchiveExtensions.kt
+++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/ThreadTableArchiveExtensions.kt
@@ -5,7 +5,6 @@

 package org.thoughtcrime.securesms.backup.v2.database

-import org.signal.core.util.SqlUtil
 import org.thoughtcrime.securesms.backup.v2.exporters.ChatArchiveExporter
 import org.thoughtcrime.securesms.database.RecipientTable
 import org.thoughtcrime.securesms.database.SignalDatabase
@@ -35,9 +34,3 @@ fun ThreadTable.getThreadsForBackup(db: SignalDatabase): ChatArchiveExporter {

  return ChatArchiveExporter(cursor, db)
 }
-
-fun ThreadTable.clearAllDataForBackupRestore() {
-  writableDatabase.delete(ThreadTable.TABLE_NAME, null, null)
-  SqlUtil.resetAutoIncrementValue(writableDatabase, ThreadTable.TABLE_NAME)
-  clearCache()
-}
--- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/exporters/CallLinkArchiveExporter.kt
+++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/exporters/CallLinkArchiveExporter.kt
@@ -6,7 +6,6 @@
 package org.thoughtcrime.securesms.backup.v2.database

 import android.database.Cursor
-import okio.ByteString
 import okio.ByteString.Companion.toByteString
 import org.signal.ringrtc.CallLinkState
 import org.thoughtcrime.securesms.backup.v2.ArchiveRecipient
@@ -32,8 +31,8 @@ class CallLinkArchiveExporter(private val cursor: Cursor) : Iterator<ArchiveReci
    return ArchiveRecipient(
      id = callLink.recipientId.toLong(),
      callLink = CallLink(
-        rootKey = callLink.credentials?.linkKeyBytes?.toByteString() ?: ByteString.EMPTY,
-        adminKey = callLink.credentials?.adminPassBytes?.toByteString(),
+        rootKey = callLink.credentials!!.linkKeyBytes.toByteString(),
+        adminKey = callLink.credentials.adminPassBytes?.toByteString(),
        name = callLink.state.name,
        expirationMs = try {
          callLink.state.expiration.toEpochMilli()
--- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/importer/AdHodCallArchiveImporter.kt
+++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/importer/AdHodCallArchiveImporter.kt
@@ -6,6 +6,7 @@
 package org.thoughtcrime.securesms.backup.v2.importer

 import org.signal.core.util.insertInto
+import org.signal.core.util.logging.Log
 import org.thoughtcrime.securesms.backup.v2.ImportState
 import org.thoughtcrime.securesms.backup.v2.proto.AdHocCall
 import org.thoughtcrime.securesms.database.CallTable
@@ -15,17 +16,25 @@ import org.thoughtcrime.securesms.database.SignalDatabase
 * Handles the importing of [AdHocCall] models into the local database.
 */
 object AdHodCallArchiveImporter {
+
+  private val TAG = Log.tag(AdHodCallArchiveImporter::class)
+
  fun import(call: AdHocCall, importState: ImportState) {
    val event = when (call.state) {
      AdHocCall.State.GENERIC -> CallTable.Event.GENERIC_GROUP_CALL
      AdHocCall.State.UNKNOWN_STATE -> CallTable.Event.GENERIC_GROUP_CALL
    }

+    val peer = importState.remoteToLocalRecipientId[call.recipientId] ?: run {
+      Log.w(TAG, "Failed to find matching recipientId for peer with remote recipientId ${call.recipientId}! Skipping.")
+      return
+    }
+
    SignalDatabase.writableDatabase
      .insertInto(CallTable.TABLE_NAME)
      .values(
        CallTable.CALL_ID to call.callId,
-        CallTable.PEER to importState.remoteToLocalRecipientId[call.recipientId]!!.serialize(),
+        CallTable.PEER to peer.serialize(),
        CallTable.TYPE to CallTable.Type.serialize(CallTable.Type.AD_HOC_CALL),
        CallTable.DIRECTION to CallTable.Direction.serialize(CallTable.Direction.OUTGOING),
        CallTable.EVENT to CallTable.Event.serialize(event),
--- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/importer/CallLinkArchiveImporter.kt
+++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/importer/CallLinkArchiveImporter.kt
@@ -5,6 +5,8 @@

 package org.thoughtcrime.securesms.backup.v2.importer

+import org.signal.core.util.isEmpty
+import org.signal.core.util.logging.Log
 import org.signal.ringrtc.CallLinkRootKey
 import org.signal.ringrtc.CallLinkState
 import org.thoughtcrime.securesms.backup.v2.ArchiveCallLink
@@ -21,13 +23,21 @@ import java.time.Instant
 * Handles the importing of [ArchiveCallLink] models into the local database.
 */
 object CallLinkArchiveImporter {
+
+  private val TAG = Log.tag(CallLinkArchiveImporter::class)
+
  fun import(callLink: ArchiveCallLink): RecipientId? {
-    val rootKey: CallLinkRootKey
-    try {
-      rootKey = CallLinkRootKey(callLink.rootKey.toByteArray())
+    val rootKey: CallLinkRootKey = try {
+      CallLinkRootKey(callLink.rootKey.toByteArray())
    } catch (e: Exception) {
+      if (callLink.rootKey.isEmpty()) {
+        Log.w(TAG, "Missing root key!")
+      } else {
+        Log.w(TAG, "Failed to parse a non-empty root key!")
+      }
      return null
    }
+
    return SignalDatabase.callLinks.insertCallLink(
      CallLinkTable.CallLink(
        recipientId = RecipientId.UNKNOWN,
--- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/importer/ChatItemArchiveImporter.kt
+++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/importer/ChatItemArchiveImporter.kt
@@ -875,7 +875,7 @@ class ChatItemArchiveImporter(

  private fun ContentValues.addQuote(quote: Quote) {
    this.put(MessageTable.QUOTE_ID, quote.targetSentTimestamp ?: MessageTable.QUOTE_TARGET_MISSING_ID)
-    this.put(MessageTable.QUOTE_AUTHOR, importState.remoteToLocalRecipientId[quote.authorId]!!.serialize())
+    this.put(MessageTable.QUOTE_AUTHOR, importState.requireLocalRecipientId(quote.authorId).serialize())
    this.put(MessageTable.QUOTE_BODY, quote.text?.body)
    this.put(MessageTable.QUOTE_TYPE, quote.type.toLocalQuoteType())
    this.put(MessageTable.QUOTE_BODY_RANGES, quote.text?.bodyRanges?.toLocalBodyRanges()?.encode())
--- a/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/backup/InternalBackupPlaygroundViewModel.kt
+++ b/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/backup/InternalBackupPlaygroundViewModel.kt
@@ -536,6 +536,7 @@ class InternalBackupPlaygroundViewModel : ViewModel() {
    AppDependencies.jobManager.cancelAllInQueue("ArchiveAttachmentJobs_0")
    AppDependencies.jobManager.cancelAllInQueue("ArchiveAttachmentJobs_1")
    AppDependencies.jobManager.cancelAllInQueue("ArchiveThumbnailUploadJob")
+    AppDependencies.jobManager.cancelAllInQueue("BackupRestoreJob")
  }

  fun fetchRemoteBackupAndWritePlaintext(outputStream: OutputStream?) {
--- a/app/src/main/java/org/thoughtcrime/securesms/database/NameCollisionTables.kt
+++ b/app/src/main/java/org/thoughtcrime/securesms/database/NameCollisionTables.kt
@@ -55,16 +55,12 @@ class NameCollisionTables(

    private val PROFILE_CHANGE_TIMEOUT = 1.days

-    fun createTables(db: SQLiteDatabase) {
-      db.execSQL(NameCollisionTable.CREATE_TABLE)
-      db.execSQL(NameCollisionMembershipTable.CREATE_TABLE)
-    }
+    val CREATE_TABLE = arrayOf(
+      NameCollisionTable.CREATE_TABLE,
+      NameCollisionMembershipTable.CREATE_TABLE
+    )

-    fun createIndexes(db: SQLiteDatabase) {
-      NameCollisionMembershipTable.CREATE_INDEXES.forEach {
-        db.execSQL(it)
-      }
-    }
+    val CREATE_INDEXES = NameCollisionMembershipTable.CREATE_INDEXES
  }

  /**
--- a/app/src/main/java/org/thoughtcrime/securesms/database/SQLiteDatabase.java
+++ b/app/src/main/java/org/thoughtcrime/securesms/database/SQLiteDatabase.java
@@ -14,6 +14,7 @@ import androidx.sqlite.db.SupportSQLiteQuery;
 import net.zetetic.database.sqlcipher.SQLiteStatement;
 import net.zetetic.database.sqlcipher.SQLiteTransactionListener;

+import org.signal.core.util.logging.Log;
 import org.signal.core.util.tracing.Tracer;

 import java.io.IOException;
--- a/app/src/main/java/org/thoughtcrime/securesms/database/SearchTable.kt
+++ b/app/src/main/java/org/thoughtcrime/securesms/database/SearchTable.kt
@@ -180,10 +180,11 @@ class SearchTable(context: Context, databaseHelper: SignalDatabase) : DatabaseTa
        writableDatabase.withinTransaction { db ->
          db.execSQL(
            """
-            INSERT INTO $FTS_TABLE_NAME ($ID, $BODY)
+            INSERT INTO $FTS_TABLE_NAME ($ID, $BODY, $THREAD_ID)
                SELECT
                  ${MessageTable.ID},
-                  ${MessageTable.BODY}
+                  ${MessageTable.BODY},
+                  ${MessageTable.THREAD_ID}
                FROM
                  ${MessageTable.TABLE_NAME}
                WHERE
--- a/app/src/main/java/org/thoughtcrime/securesms/database/SignalDatabase.kt
+++ b/app/src/main/java/org/thoughtcrime/securesms/database/SignalDatabase.kt
@@ -113,7 +113,7 @@ open class SignalDatabase(private val context: Application, databaseSecret: Data
    db.execSQL(CallLinkTable.CREATE_TABLE)
    db.execSQL(CallTable.CREATE_TABLE)
    db.execSQL(KyberPreKeyTable.CREATE_TABLE)
-    NameCollisionTables.createTables(db)
+    executeStatements(db, NameCollisionTables.CREATE_TABLE)
    db.execSQL(InAppPaymentTable.CREATE_TABLE)
    db.execSQL(InAppPaymentSubscriberTable.CREATE_TABLE)
    executeStatements(db, SearchTable.CREATE_TABLE)
@@ -144,12 +144,11 @@ open class SignalDatabase(private val context: Application, databaseSecret: Data
    executeStatements(db, ReactionTable.CREATE_INDEXES)
    executeStatements(db, KyberPreKeyTable.CREATE_INDEXES)
    executeStatements(db, ChatFolderTables.CREATE_INDEXES)
+    executeStatements(db, NameCollisionTables.CREATE_INDEXES)

    executeStatements(db, SearchTable.CREATE_TRIGGERS)
    executeStatements(db, MessageSendLogTables.CREATE_TRIGGERS)

-    NameCollisionTables.createIndexes(db)
-
    DistributionListTables.insertInitialDistributionListAtCreationTime(db)
    ChatFolderTables.insertInitialChatFoldersAtCreationTime(db)

--- a/app/src/main/java/org/thoughtcrime/securesms/database/SqlCipherErrorHandler.kt
+++ b/app/src/main/java/org/thoughtcrime/securesms/database/SqlCipherErrorHandler.kt
@@ -165,16 +165,16 @@ class SqlCipherErrorHandler(private val databaseName: String) : DatabaseErrorHan
  }

  private fun attemptToClearFullTextSearchIndex(db: SQLiteDatabase) {
-    try {
-      try {
-        db.reopenReadWrite()
-      } catch (e: Exception) {
-        Log.w(TAG, "Failed to re-open as read-write!", e)
-      }
-      SignalDatabase.messageSearch.fullyResetTables(db, useTransaction = false)
-    } catch (e: Throwable) {
-      Log.w(TAG, "Failed to clear full text search index.", e)
-    }
+//    try {
+//      try {
+//        db.reopenReadWrite()
+//      } catch (e: Exception) {
+//        Log.w(TAG, "Failed to re-open as read-write!", e)
+//      }
+//      SignalDatabase.messageSearch.fullyResetTables(db, useTransaction = false)
+//    } catch (e: Throwable) {
+//      Log.w(TAG, "Failed to clear full text search index.", e)
+//    }
  }

  private sealed class DiagnosticResults(val logs: String) {
--- a/app/src/main/java/org/thoughtcrime/securesms/jobmanager/JobManager.java
+++ b/app/src/main/java/org/thoughtcrime/securesms/jobmanager/JobManager.java
@@ -64,7 +64,9 @@ public class JobManager implements ConstraintObserver.Notifier {
  public JobManager(@NonNull Application application, @NonNull Configuration configuration) {
    this.application   = application;
    this.configuration = configuration;
-    this.executor      = new FilteredExecutor(configuration.getExecutorFactory().newSingleThreadExecutor("signal-JobManager"), ThreadUtil::isMainThread);
+    this.executor      = new FilteredExecutor(configuration.getExecutorFactory().newSingleThreadExecutor("signal-JobManager"), () -> {
+      return ThreadUtil.isMainThread() || Thread.currentThread().getName().equals("Instr: org.thoughtcrime.securesms.testing.SignalTestRunner");
+    });
    this.jobTracker    = configuration.getJobTracker();
    this.jobController = new JobController(application,
                                           configuration.getJobStorage(),
--- a/app/src/main/java/org/thoughtcrime/securesms/jobs/BackupRestoreJob.kt
+++ b/app/src/main/java/org/thoughtcrime/securesms/jobs/BackupRestoreJob.kt
@@ -39,6 +39,7 @@ class BackupRestoreJob private constructor(parameters: Parameters) : BaseJob(par
      .addConstraint(NetworkConstraint.KEY)
      .setMaxAttempts(Parameters.UNLIMITED)
      .setMaxInstancesForFactory(1)
+      .setQueue("BackupRestoreJob")
      .build()
  )

@@ -85,6 +86,10 @@ class BackupRestoreJob private constructor(parameters: Parameters) : BaseJob(par
      throw IOException()
    }

+    if (isCanceled) {
+      return
+    }
+
    controller.update(
      title = context.getString(R.string.BackupProgressService_title),
      progress = 0f,
@@ -93,7 +98,7 @@ class BackupRestoreJob private constructor(parameters: Parameters) : BaseJob(par

    val self = Recipient.self()
    val selfData = BackupRepository.SelfData(self.aci.get(), self.pni.get(), self.e164.get(), ProfileKey(self.profileKey))
-    BackupRepository.import(length = tempBackupFile.length(), inputStreamFactory = tempBackupFile::inputStream, selfData = selfData, plaintext = false)
+    BackupRepository.import(length = tempBackupFile.length(), inputStreamFactory = tempBackupFile::inputStream, selfData = selfData, plaintext = false, cancellationSignal = { isCanceled })

    SignalStore.backup.restoreState = RestoreState.RESTORING_MEDIA
  }
--- a/core-util-jvm/src/main/java/org/signal/core/util/ProtoExtensions.kt
+++ b/core-util-jvm/src/main/java/org/signal/core/util/ProtoExtensions.kt
@@ -20,6 +20,10 @@ import java.util.LinkedList

 private const val TAG = "ProtoExtension"

+fun ByteString?.isEmpty(): Boolean {
+  return this == null || this.size == 0
+}
+
 fun ByteString?.isNotEmpty(): Boolean {
  return this != null && this.size > 0
 }
--- a/core-util/src/main/java/org/signal/core/util/SQLiteDatabaseExtensions.kt
+++ b/core-util/src/main/java/org/signal/core/util/SQLiteDatabaseExtensions.kt
@@ -6,6 +6,12 @@ import android.database.sqlite.SQLiteDatabase
 import androidx.core.content.contentValuesOf
 import androidx.sqlite.db.SupportSQLiteDatabase
 import androidx.sqlite.db.SupportSQLiteQueryBuilder
+import org.signal.core.util.SqlUtil.ForeignKeyViolation
+import org.signal.core.util.logging.Log
+import kotlin.time.Duration
+import kotlin.time.Duration.Companion.seconds
+
+private val TAG = "SQLiteDatabaseExtensions"

 /**
 * Begins a transaction on the `this` database, runs the provided [block] providing the `this` value as it's argument
@@ -46,7 +52,11 @@ fun SupportSQLiteDatabase.getAllTables(): List<String> {
 * Returns a list of objects that represent the table definitions in the database. Basically the table name and then the SQL that was used to create it.
 */
 fun SupportSQLiteDatabase.getAllTableDefinitions(): List<CreateStatement> {
-  return this.query("SELECT name, sql FROM sqlite_schema WHERE type = 'table' AND sql NOT NULL AND name != 'sqlite_sequence'")
+  return this
+    .select("name", "sql")
+    .from("sqlite_schema")
+    .where("type = ? AND sql NOT NULL AND name != ?", "table", "sqlite_sequence")
+    .run()
    .readToList { cursor ->
      CreateStatement(
        name = cursor.requireNonNullString("name"),
@@ -61,7 +71,11 @@ fun SupportSQLiteDatabase.getAllTableDefinitions(): List<CreateStatement> {
 * Returns a list of objects that represent the index definitions in the database. Basically the index name and then the SQL that was used to create it.
 */
 fun SupportSQLiteDatabase.getAllIndexDefinitions(): List<CreateStatement> {
-  return this.query("SELECT name, sql FROM sqlite_schema WHERE type = 'index' AND sql NOT NULL")
+  return this
+    .select("name", "sql")
+    .from("sqlite_schema")
+    .where("type = ? AND sql NOT NULL", "index")
+    .run()
    .readToList { cursor ->
      CreateStatement(
        name = cursor.requireNonNullString("name"),
@@ -71,6 +85,24 @@ fun SupportSQLiteDatabase.getAllIndexDefinitions(): List<CreateStatement> {
    .sortedBy { it.name }
 }

+/**
+ * Retrieves the names of all triggers, sorted alphabetically.
+ */
+fun SupportSQLiteDatabase.getAllTriggerDefinitions(): List<CreateStatement> {
+  return this
+    .select("name", "sql")
+    .from("sqlite_schema")
+    .where("type = ? AND sql NOT NULL", "trigger")
+    .run()
+    .readToList {
+      CreateStatement(
+        name = it.requireNonNullString("name"),
+        statement = it.requireNonNullString("sql")
+      )
+    }
+    .sortedBy { it.name }
+}
+
 fun SupportSQLiteDatabase.getForeignKeys(): List<ForeignKeyConstraint> {
  return SqlUtil.getAllTables(this)
    .map { table ->
@@ -93,6 +125,24 @@ fun SupportSQLiteDatabase.areForeignKeyConstraintsEnabled(): Boolean {
  }
 }

+/**
+ * Provides a list of all foreign key violations present.
+ * If a [targetTable] is specified, results will be limited to that table specifically.
+ * Otherwise, the check will be performed across all tables.
+ */
+@JvmOverloads
+fun SupportSQLiteDatabase.getForeignKeyViolations(targetTable: String? = null): List<ForeignKeyViolation> {
+  return SqlUtil.getForeignKeyViolations(this, targetTable)
+}
+
+/**
+ * For tables that have an autoincrementing primary key, this will reset the key to start back at 1.
+ * IMPORTANT: This is quite dangerous! Only do this if you're effectively resetting the entire database.
+ */
+fun SupportSQLiteDatabase.resetAutoIncrementValue(targetTable: String) {
+  SqlUtil.resetAutoIncrementValue(this, targetTable)
+}
+
 /**
 * Does a full WAL checkpoint (TRUNCATE mode, where the log is for sure flushed and the log is zero'd out).
 * Will try up to [maxAttempts] times. Can technically fail if the database is too active and the checkpoint
@@ -132,6 +182,22 @@ fun SupportSQLiteDatabase.getIndexes(): List<Index> {
  }
 }

+fun SupportSQLiteDatabase.forceForeignKeyConstraintsEnabled(enabled: Boolean, timeout: Duration = 10.seconds) {
+  val startTime = System.currentTimeMillis()
+  while (true) {
+    try {
+      this.setForeignKeyConstraintsEnabled(enabled)
+      break
+    } catch (e: IllegalStateException) {
+      if (System.currentTimeMillis() - startTime > timeout.inWholeMilliseconds) {
+        throw IllegalStateException("Failed to force foreign keys to '$enabled' within the timeout of $timeout", e)
+      }
+      Log.w(TAG, "Failed to set foreign keys because we're in a transaction. Waiting 100ms then trying again.")
+      ThreadUtil.sleep(100)
+    }
+  }
+}
+
 /**
 * Checks if a row exists that matches the query.
 */