Replace manual FTS5 fix with SQLite secure-delete flag.

We used to workaround this by manually optimizing the search index, but secure-delete does that for us with less work.

app/src/main/java/org/thoughtcrime/securesms/database/MessageTable.kt

@@ -112,7 +112,6 @@ import org.thoughtcrime.securesms.database.model.databaseprotos.SessionSwitchove
 import org.thoughtcrime.securesms.database.model.databaseprotos.ThreadMergeEvent
 import org.thoughtcrime.securesms.dependencies.AppDependencies
 import org.thoughtcrime.securesms.groups.GroupMigrationMembershipChange
-import org.thoughtcrime.securesms.jobs.OptimizeMessageSearchIndexJob
 import org.thoughtcrime.securesms.jobs.ThreadUpdateJob
 import org.thoughtcrime.securesms.jobs.TrimThreadJob
 import org.thoughtcrime.securesms.keyvalue.SignalStore
@@ -1651,7 +1650,7 @@ open class MessageTable(context: Context?, databaseHelper: SignalDatabase) : Dat
         .readToList { RecipientId.from(it.requireLong(FROM_RECIPIENT_ID)) }
         .forEach { id -> AppDependencies.databaseObserver.notifyStoryObservers(id) }
 
-      val deletedStoryCount = db.select(ID)
+      db.select(ID)
         .from(TABLE_NAME)
         .where(storiesBeforeTimestampWhere, sharedArgs)
         .run()
@@ -1662,12 +1661,6 @@ open class MessageTable(context: Context?, databaseHelper: SignalDatabase) : Dat
 
           cursor.count
         }
-
-      if (deletedStoryCount > 0) {
-        OptimizeMessageSearchIndexJob.enqueue()
-      }
-
-      deletedStoryCount
     }
   }
 
@@ -1710,7 +1703,7 @@ open class MessageTable(context: Context?, databaseHelper: SignalDatabase) : Dat
 
       AppDependencies.databaseObserver.notifyStoryObservers(recipientId)
 
-      val deletedStoryCount = db.select(ID)
+      db.select(ID)
         .from(TABLE_NAME)
         .where(storesInRecipientThread, sharedArgs)
         .run()
@@ -1721,12 +1714,6 @@ open class MessageTable(context: Context?, databaseHelper: SignalDatabase) : Dat
 
           cursor.count
         }
-
-      if (deletedStoryCount > 0) {
-        OptimizeMessageSearchIndexJob.enqueue()
-      }
-
-      deletedStoryCount
     }
   }
 
@@ -2107,7 +2094,6 @@ open class MessageTable(context: Context?, databaseHelper: SignalDatabase) : Dat
       threads.update(threadId, false)
     }
 
-    OptimizeMessageSearchIndexJob.enqueue()
     AppDependencies.databaseObserver.notifyMessageUpdateObservers(MessageId(messageId))
     AppDependencies.databaseObserver.notifyConversationListListeners()
 
@@ -3306,7 +3292,6 @@ open class MessageTable(context: Context?, databaseHelper: SignalDatabase) : Dat
       notifyConversationListeners(threadId)
       notifyStickerListeners()
       notifyStickerPackListeners()
-      OptimizeMessageSearchIndexJob.enqueue()
     }
 
     return threadDeleted
@@ -3526,7 +3511,6 @@ open class MessageTable(context: Context?, databaseHelper: SignalDatabase) : Dat
     notifyConversationListListeners()
     notifyStickerListeners()
     notifyStickerPackListeners()
-    OptimizeMessageSearchIndexJob.enqueue()
 
     return unhandled
   }
@@ -3548,8 +3532,6 @@ open class MessageTable(context: Context?, databaseHelper: SignalDatabase) : Dat
     mentions.deleteAllMentions()
     writableDatabase.deleteAll(TABLE_NAME)
     calls.updateCallEventDeletionTimestamps()
-
-    OptimizeMessageSearchIndexJob.enqueue()
   }
 
   fun getNearestExpiringViewOnceMessage(): ViewOnceExpirationInfo? {

app/src/main/java/org/thoughtcrime/securesms/database/SearchTable.kt

@@ -7,9 +7,7 @@ import android.text.TextUtils
 import net.zetetic.database.sqlcipher.SQLiteDatabase
 import org.intellij.lang.annotations.Language
 import org.signal.core.util.SqlUtil
-import org.signal.core.util.ThreadUtil
 import org.signal.core.util.logging.Log
-import org.signal.core.util.withinTransaction
 import org.thoughtcrime.securesms.jobs.RebuildMessageSearchIndexJob
 
 /**
@@ -36,7 +34,10 @@ class SearchTable(context: Context, databaseHelper: SignalDatabase) : DatabaseTa
       // We've taken the default of tokenize value of "unicode61 categories 'L* N* Co'" and added the Sc (currency) and So (emoji) categories to allow searching for those characters.
       // https://www.sqlite.org/fts5.html#tokenizers
       // https://www.compart.com/en/unicode/category
-      """CREATE VIRTUAL TABLE $FTS_TABLE_NAME USING fts5($BODY, $THREAD_ID UNINDEXED, content=${MessageTable.TABLE_NAME}, content_rowid=${MessageTable.ID}, tokenize = "unicode61 categories 'L* N* Co Sc So'")"""
+      """CREATE VIRTUAL TABLE $FTS_TABLE_NAME USING fts5($BODY, $THREAD_ID UNINDEXED, content=${MessageTable.TABLE_NAME}, content_rowid=${MessageTable.ID}, tokenize = "unicode61 categories 'L* N* Co Sc So'")""",
+
+      // Not technically a `CREATE` statement, but it's part of table creation. FTS5 just has weird configuration syntax. See https://www.sqlite.org/fts5.html#the_secure_delete_configuration_option
+      """INSERT INTO $FTS_TABLE_NAME ($FTS_TABLE_NAME, rank) VALUES('secure-delete', 1);"""
     )
 
     private const val TRIGGER_AFTER_INSERT = "message_ai"
@@ -165,73 +166,6 @@ class SearchTable(context: Context, databaseHelper: SignalDatabase) : DatabaseTa
     }
   }
 
-  /**
-   * This performs the same thing as the `optimize` command in SQLite, but broken into iterative stages to avoid locking up the database for too long.
-   * If what's going on in this method seems weird, that's because it is, but please read the sqlite docs -- we're following their algorithm:
-   * https://www.sqlite.org/fts5.html#the_optimize_command
-   *
-   * Note that in order for the [SqlUtil.getTotalChanges] call to work, we have to be within a transaction, or else the connection pool screws everything up
-   * (the stats are on a per-connection basis).
-   *
-   * There's this double-batching mechanism happening here to strike a balance between making individual transactions short while also not hammering the
-   * database with a ton of independent transactions.
-   *
-   * To give you some ballpark numbers, on a large database (~400k messages), it takes ~75 iterations to fully optimize everything.
-   */
-  fun optimizeIndex(timeout: Long): Boolean {
-    val pageSize = 64 // chosen through experimentation
-    val batchSize = 10 // chosen through experimentation
-    val noChangeThreshold = 2 // if less changes occurred than this, operation is considered no-op (see sqlite docs ref'd in kdoc)
-
-    val startTime = System.currentTimeMillis()
-    var totalIterations = 0
-    var totalBatches = 0
-    var actualWorkTime = 0L
-    var finished = false
-
-    while (!finished) {
-      var batchIterations = 0
-      val batchStartTime = System.currentTimeMillis()
-
-      writableDatabase.withinTransaction { db ->
-        // Note the negative page size -- see sqlite docs ref'd in kdoc
-        db.execSQL("INSERT INTO $FTS_TABLE_NAME ($FTS_TABLE_NAME, rank) values ('merge', -$pageSize)")
-        var previousCount = SqlUtil.getTotalChanges(db)
-
-        val iterativeStatement = db.compileStatement("INSERT INTO $FTS_TABLE_NAME ($FTS_TABLE_NAME, rank) values ('merge', $pageSize)")
-        iterativeStatement.execute()
-        var count = SqlUtil.getTotalChanges(db)
-
-        while (batchIterations < batchSize && count - previousCount >= noChangeThreshold) {
-          previousCount = count
-          iterativeStatement.execute()
-
-          count = SqlUtil.getTotalChanges(db)
-          batchIterations++
-        }
-
-        if (count - previousCount < noChangeThreshold) {
-          finished = true
-        }
-      }
-
-      totalIterations += batchIterations
-      totalBatches++
-      actualWorkTime += System.currentTimeMillis() - batchStartTime
-
-      if (actualWorkTime >= timeout) {
-        Log.w(TAG, "Timed out during optimization! We did $totalIterations iterations across $totalBatches batches, taking ${System.currentTimeMillis() - startTime} ms. Bailed out to avoid database lockup.")
-        return false
-      }
-
-      // We want to sleep in between batches to give other db operations a chance to run
-      ThreadUtil.sleep(50)
-    }
-
-    Log.d(TAG, "Took ${System.currentTimeMillis() - startTime} ms and $totalIterations iterations across $totalBatches batches to optimize. Of that time, $actualWorkTime ms were spent actually working (~${actualWorkTime / totalBatches} ms/batch). The rest was spent sleeping.")
-    return true
-  }
-
   /**
    * Drops all tables and recreates them.
    */

app/src/main/java/org/thoughtcrime/securesms/database/ThreadTable.kt

@@ -51,7 +51,6 @@ import org.thoughtcrime.securesms.dependencies.AppDependencies
 import org.thoughtcrime.securesms.groups.BadGroupIdException
 import org.thoughtcrime.securesms.groups.GroupId
 import org.thoughtcrime.securesms.jobs.MultiDeviceDeleteSyncJob
-import org.thoughtcrime.securesms.jobs.OptimizeMessageSearchIndexJob
 import org.thoughtcrime.securesms.keyvalue.SignalStore
 import org.thoughtcrime.securesms.mms.SlideDeck
 import org.thoughtcrime.securesms.mms.StickerSlide
@@ -363,7 +362,6 @@ class ThreadTable(context: Context, databaseHelper: SignalDatabase) : DatabaseTa
 
     notifyAttachmentListeners()
     notifyStickerPackListeners()
-    OptimizeMessageSearchIndexJob.enqueue()
   }
 
   fun trimThread(
@@ -397,7 +395,6 @@ class ThreadTable(context: Context, databaseHelper: SignalDatabase) : DatabaseTa
 
     notifyAttachmentListeners()
     notifyStickerPackListeners()
-    OptimizeMessageSearchIndexJob.enqueue()
   }
 
   private fun trimThreadInternal(
@@ -1188,8 +1185,6 @@ class ThreadTable(context: Context, databaseHelper: SignalDatabase) : DatabaseTa
     AppDependencies.databaseObserver.notifyConversationDeleteListeners(selectedConversations)
 
     ConversationUtil.clearShortcuts(context, recipientIds)
-
-    OptimizeMessageSearchIndexJob.enqueue()
   }
 
   @SuppressLint("DiscouragedApi")

app/src/main/java/org/thoughtcrime/securesms/database/helpers/SignalDatabaseMigrations.kt

@@ -97,6 +97,7 @@ import org.thoughtcrime.securesms.database.helpers.migration.V236_FixInAppSubscr
 import org.thoughtcrime.securesms.database.helpers.migration.V237_ResetGroupForceUpdateTimestamps
 import org.thoughtcrime.securesms.database.helpers.migration.V238_AddGroupSendEndorsementsColumns
 import org.thoughtcrime.securesms.database.helpers.migration.V239_MessageFullTextSearchEmojiSupport
+import org.thoughtcrime.securesms.database.helpers.migration.V240_MessageFullTextSearchSecureDelete
 
 /**
  * Contains all of the database migrations for [SignalDatabase]. Broken into a separate file for cleanliness.
@@ -196,10 +197,11 @@ object SignalDatabaseMigrations {
     236 to V236_FixInAppSubscriberCurrencyIfAble,
     237 to V237_ResetGroupForceUpdateTimestamps,
     238 to V238_AddGroupSendEndorsementsColumns,
-    239 to V239_MessageFullTextSearchEmojiSupport
+    239 to V239_MessageFullTextSearchEmojiSupport,
+    240 to V240_MessageFullTextSearchSecureDelete
   )
 
-  const val DATABASE_VERSION = 239
+  const val DATABASE_VERSION = 240
 
   @JvmStatic
   fun migrate(context: Application, db: SQLiteDatabase, oldVersion: Int, newVersion: Int) {

app/src/main/java/org/thoughtcrime/securesms/database/helpers/migration/V240_MessageFullTextSearchSecureDelete.kt

@@ -0,0 +1,23 @@
+/*
+ * Copyright 2024 Signal Messenger, LLC
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+package org.thoughtcrime.securesms.database.helpers.migration
+
+import android.app.Application
+import net.zetetic.database.sqlcipher.SQLiteDatabase
+
+/**
+ * Sets the 'secure-delete' flag on the message_fts table.
+ * https://www.sqlite.org/fts5.html#the_secure_delete_configuration_option
+ */
+@Suppress("ClassName")
+object V240_MessageFullTextSearchSecureDelete : SignalDatabaseMigration {
+
+  const val FTS_TABLE_NAME = "message_fts"
+
+  override fun migrate(context: Application, db: SQLiteDatabase, oldVersion: Int, newVersion: Int) {
+    db.execSQL("""INSERT INTO $FTS_TABLE_NAME ($FTS_TABLE_NAME, rank) VALUES('secure-delete', 1);""")
+  }
+}

app/src/main/java/org/thoughtcrime/securesms/jobs/JobManagerFactories.java

@@ -61,7 +61,6 @@ import org.thoughtcrime.securesms.migrations.EmojiSearchIndexCheckMigrationJob;
 import org.thoughtcrime.securesms.migrations.IdentityTableCleanupMigrationJob;
 import org.thoughtcrime.securesms.migrations.LegacyMigrationJob;
 import org.thoughtcrime.securesms.migrations.MigrationCompleteJob;
-import org.thoughtcrime.securesms.migrations.OptimizeMessageSearchIndexMigrationJob;
 import org.thoughtcrime.securesms.migrations.PassingMigrationJob;
 import org.thoughtcrime.securesms.migrations.PinOptOutMigration;
 import org.thoughtcrime.securesms.migrations.PinReminderMigrationJob;
@@ -181,7 +180,6 @@ public final class JobManagerFactories {
       put(MultiDeviceViewOnceOpenJob.KEY,            new MultiDeviceViewOnceOpenJob.Factory());
       put(MultiDeviceViewedUpdateJob.KEY,            new MultiDeviceViewedUpdateJob.Factory());
       put(NullMessageSendJob.KEY,                    new NullMessageSendJob.Factory());
-      put(OptimizeMessageSearchIndexJob.KEY,         new OptimizeMessageSearchIndexJob.Factory());
       put(PaymentLedgerUpdateJob.KEY,                new PaymentLedgerUpdateJob.Factory());
       put(PaymentNotificationSendJob.KEY,            new PaymentNotificationSendJob.Factory());
       put(PaymentNotificationSendJobV2.KEY,          new PaymentNotificationSendJobV2.Factory());
@@ -271,7 +269,6 @@ public final class JobManagerFactories {
       put(IdentityTableCleanupMigrationJob.KEY,      new IdentityTableCleanupMigrationJob.Factory());
       put(LegacyMigrationJob.KEY,                    new LegacyMigrationJob.Factory());
       put(MigrationCompleteJob.KEY,                  new MigrationCompleteJob.Factory());
-      put(OptimizeMessageSearchIndexMigrationJob.KEY,new OptimizeMessageSearchIndexMigrationJob.Factory());
       put(PinOptOutMigration.KEY,                    new PinOptOutMigration.Factory());
       put(PinReminderMigrationJob.KEY,               new PinReminderMigrationJob.Factory());
       put(PniAccountInitializationMigrationJob.KEY,  new PniAccountInitializationMigrationJob.Factory());
@@ -342,6 +339,8 @@ public final class JobManagerFactories {
       put("SmsSentJob",                              new FailingJob.Factory());
       put("MmsSendJobV2",                            new FailingJob.Factory());
       put("AttachmentUploadJobV2",                   new FailingJob.Factory());
+      put("OptimizeMessageSearchIndexJob",           new FailingJob.Factory());
+      put("OptimizeMessageSearchIndexMigrationJob",  new PassingMigrationJob.Factory());
     }};
   }
 

app/src/main/java/org/thoughtcrime/securesms/jobs/OptimizeMessageSearchIndexJob.kt

@@ -1,58 +0,0 @@
-package org.thoughtcrime.securesms.jobs
-
-import org.signal.core.util.logging.Log
-import org.thoughtcrime.securesms.database.SignalDatabase
-import org.thoughtcrime.securesms.dependencies.AppDependencies
-import org.thoughtcrime.securesms.jobmanager.Job
-import org.thoughtcrime.securesms.keyvalue.SignalStore
-import org.thoughtcrime.securesms.transport.RetryLaterException
-import java.lang.Exception
-import kotlin.time.Duration.Companion.seconds
-
-/**
- * Optimizes the message search index incrementally.
- */
-class OptimizeMessageSearchIndexJob private constructor(parameters: Parameters) : BaseJob(parameters) {
-
-  companion object {
-    const val KEY = "OptimizeMessageSearchIndexJob"
-
-    private val TAG = Log.tag(OptimizeMessageSearchIndexJob::class.java)
-
-    @JvmStatic
-    fun enqueue() {
-      AppDependencies.jobManager.add(OptimizeMessageSearchIndexJob())
-    }
-  }
-
-  constructor() : this(
-    Parameters.Builder()
-      .setQueue("OptimizeMessageSearchIndexJob")
-      .setMaxAttempts(5)
-      .setMaxInstancesForQueue(2)
-      .build()
-  )
-
-  override fun serialize(): ByteArray? = null
-  override fun getFactoryKey() = KEY
-  override fun onFailure() = Unit
-  override fun onShouldRetry(e: Exception) = e is RetryLaterException
-  override fun getNextRunAttemptBackoff(pastAttemptCount: Int, exception: Exception): Long = 30.seconds.inWholeMilliseconds
-
-  override fun onRun() {
-    if (!SignalStore.registration.isRegistrationComplete || SignalStore.account.aci == null) {
-      Log.w(TAG, "Registration not finished yet! Skipping.")
-      return
-    }
-
-    val success = SignalDatabase.messageSearch.optimizeIndex(5.seconds.inWholeMilliseconds)
-
-    if (!success) {
-      throw RetryLaterException()
-    }
-  }
-
-  class Factory : Job.Factory<OptimizeMessageSearchIndexJob> {
-    override fun create(parameters: Parameters, serializedData: ByteArray?) = OptimizeMessageSearchIndexJob(parameters)
-  }
-}

app/src/main/java/org/thoughtcrime/securesms/migrations/ApplicationMigrations.java

@@ -116,7 +116,7 @@ public class ApplicationMigrations {
     static final int SMS_MMS_MERGE                 = 71;
     static final int REBUILD_MESSAGE_FTS_INDEX     = 72;
     static final int UPDATE_SMS_JOBS               = 73;
-    static final int OPTIMIZE_MESSAGE_FTS_INDEX    = 74;
+//    static final int OPTIMIZE_MESSAGE_FTS_INDEX    = 74;
     static final int REACTION_DATABASE_MIGRATION   = 75;
     static final int REBUILD_MESSAGE_FTS_INDEX_2   = 76;
     static final int GLIDE_CACHE_CLEAR             = 77;
@@ -547,9 +547,9 @@ public class ApplicationMigrations {
       jobs.put(Version.UPDATE_SMS_JOBS, new UpdateSmsJobsMigrationJob());
     }
 
-    if (lastSeenVersion < Version.OPTIMIZE_MESSAGE_FTS_INDEX) {
-      jobs.put(Version.OPTIMIZE_MESSAGE_FTS_INDEX, new OptimizeMessageSearchIndexMigrationJob());
-    }
+//    if (lastSeenVersion < Version.OPTIMIZE_MESSAGE_FTS_INDEX) {
+//      jobs.put(Version.OPTIMIZE_MESSAGE_FTS_INDEX, new OptimizeMessageSearchIndexMigrationJob());
+//    }
 
     if (lastSeenVersion < Version.REACTION_DATABASE_MIGRATION) {
       jobs.put(Version.REACTION_DATABASE_MIGRATION, new DatabaseMigrationJob());

app/src/main/java/org/thoughtcrime/securesms/migrations/OptimizeMessageSearchIndexMigrationJob.kt

@@ -1,34 +0,0 @@
-package org.thoughtcrime.securesms.migrations
-
-import org.signal.core.util.logging.Log
-import org.thoughtcrime.securesms.jobmanager.Job
-import org.thoughtcrime.securesms.jobs.OptimizeMessageSearchIndexJob
-
-/**
- * Kicks off a job to optimize the message search index.
- */
-internal class OptimizeMessageSearchIndexMigrationJob(
-  parameters: Parameters = Parameters.Builder().build()
-) : MigrationJob(parameters) {
-
-  companion object {
-    val TAG = Log.tag(OptimizeMessageSearchIndexMigrationJob::class.java)
-    const val KEY = "OptimizeMessageSearchIndexMigrationJob"
-  }
-
-  override fun getFactoryKey(): String = KEY
-
-  override fun isUiBlocking(): Boolean = false
-
-  override fun performMigration() {
-    OptimizeMessageSearchIndexJob.enqueue()
-  }
-
-  override fun shouldRetry(e: Exception): Boolean = false
-
-  class Factory : Job.Factory<OptimizeMessageSearchIndexMigrationJob> {
-    override fun create(parameters: Parameters, serializedData: ByteArray?): OptimizeMessageSearchIndexMigrationJob {
-      return OptimizeMessageSearchIndexMigrationJob(parameters)
-    }
-  }
-}