Improve reliability of rebuilding the search index.

app/src/main/java/org/thoughtcrime/securesms/database/SearchTable.kt

@@ -3,6 +3,7 @@ package org.thoughtcrime.securesms.database
 import android.annotation.SuppressLint
 import android.content.Context
 import android.database.Cursor
+import android.database.sqlite.SQLiteException
 import android.text.TextUtils
 import net.zetetic.database.sqlcipher.SQLiteDatabase
 import org.intellij.lang.annotations.Language
@@ -141,34 +142,43 @@ class SearchTable(context: Context, databaseHelper: SignalDatabase) : DatabaseTa
    *
    * Warning: This is a potentially extremely-costly operation! It can take 10+ seconds on large installs and/or slow devices.
    * Be smart about where you call this.
+   *
+   * @return True if the rebuild was successful, otherwise false.
    */
-  fun rebuildIndex(batchSize: Long = 10_000L) {
+  fun rebuildIndex(batchSize: Long = 10_000L): Boolean {
-    val maxId: Long = SignalDatabase.messages.getNextId()
+    try {
+      val maxId: Long = SignalDatabase.messages.getNextId()
 
-    if (!SqlUtil.tableExists(readableDatabase, FTS_TABLE_NAME)) {
+      Log.i(TAG, "Re-indexing. Operating on ID's 1-$maxId in steps of $batchSize.")
-      Log.w(TAG, "FTS table does not exist. Rebuilding.")
+
-      fullyResetTables()
+      for (i in 1..maxId step batchSize) {
-      return
+        Log.i(TAG, "Reindexing ID's [$i, ${i + batchSize})")
+
+        writableDatabase.withinTransaction { db ->
+          db.execSQL(
+            """
+            INSERT INTO $FTS_TABLE_NAME ($ID, $BODY)
+                SELECT
+                  ${MessageTable.ID},
+                  ${MessageTable.BODY}
+                FROM
+                  ${MessageTable.TABLE_NAME}
+                WHERE
+                  ${MessageTable.ID} >= $i AND
+                  ${MessageTable.ID} < ${i + batchSize}
+            """
+          )
+        }
+      }
+    } catch (e: SQLiteException) {
+      Log.w(TAG, "Failed to rebuild index!", e)
+      return false
+    } catch (e: IllegalStateException) {
+      Log.w(TAG, "Failed to rebuild index!", e)
+      return false
     }
 
-    Log.i(TAG, "Re-indexing. Operating on ID's 1-$maxId in steps of $batchSize.")
+    return true
-
-    for (i in 1..maxId step batchSize) {
-      Log.i(TAG, "Reindexing ID's [$i, ${i + batchSize})")
-      writableDatabase.execSQL(
-        """
-        INSERT INTO $FTS_TABLE_NAME ($ID, $BODY) 
-            SELECT 
-              ${MessageTable.ID}, 
-              ${MessageTable.BODY}
-            FROM 
-              ${MessageTable.TABLE_NAME} 
-            WHERE 
-              ${MessageTable.ID} >= $i AND
-              ${MessageTable.ID} < ${i + batchSize}
-        """
-      )
-    }
   }
 
   /**
@@ -249,11 +259,16 @@ class SearchTable(context: Context, databaseHelper: SignalDatabase) : DatabaseTa
 
     try {
       Log.w(TAG, "[fullyResetTables] Dropping tables and triggers...")
-      db.execSQL("DROP TABLE IF EXISTS $FTS_TABLE_NAME")
+
-      db.execSQL("DROP TABLE IF EXISTS ${FTS_TABLE_NAME}_config")
+      // Due to issues we've had in the past, the delete sequence here is very particular. It mimics the "safe drop" process in the SQLite source code
-      db.execSQL("DROP TABLE IF EXISTS ${FTS_TABLE_NAME}_content")
+      // that prevents weird vtable constructor issues when dropping potentially-corrupt tables. https://sqlite.org/src/info/4db9258a78?ln=1549-1592
-      db.execSQL("DROP TABLE IF EXISTS ${FTS_TABLE_NAME}_data")
+      if (SqlUtil.tableExists(db, FTS_TABLE_NAME)) {
-      db.execSQL("DROP TABLE IF EXISTS ${FTS_TABLE_NAME}_idx")
+        db.execSQL("DELETE FROM ${FTS_TABLE_NAME}_data")
+        db.execSQL("DELETE FROM ${FTS_TABLE_NAME}_config")
+        db.execSQL("INSERT INTO ${FTS_TABLE_NAME}_data VALUES(10, X'0000000000')")
+        db.execSQL("INSERT INTO ${FTS_TABLE_NAME}_config VALUES('version', 4)")
+        db.execSQL("DROP TABLE $FTS_TABLE_NAME")
+      }
       db.execSQL("DROP TRIGGER IF EXISTS $TRIGGER_AFTER_INSERT")
       db.execSQL("DROP TRIGGER IF EXISTS $TRIGGER_AFTER_DELETE")
       db.execSQL("DROP TRIGGER IF EXISTS $TRIGGER_AFTER_UPDATE")

app/src/main/java/org/thoughtcrime/securesms/jobs/RebuildMessageSearchIndexJob.kt

@@ -7,7 +7,6 @@ import org.thoughtcrime.securesms.jobmanager.Job
 import org.thoughtcrime.securesms.jobmanager.impl.DataRestoreConstraint
 import org.thoughtcrime.securesms.transport.RetryLaterException
 import java.lang.Exception
-import java.lang.IllegalStateException
 import kotlin.time.Duration.Companion.seconds
 
 class RebuildMessageSearchIndexJob private constructor(params: Parameters) : BaseJob(params) {
@@ -37,10 +36,11 @@ class RebuildMessageSearchIndexJob private constructor(params: Parameters) : Bas
   override fun onFailure() = Unit
 
   override fun onRun() {
-    try {
+    val success = SignalDatabase.messageSearch.rebuildIndex()
-      SignalDatabase.messageSearch.rebuildIndex()
+
-    } catch (e: IllegalStateException) {
+    if (!success) {
-      throw RetryLaterException(e)
+      Log.w(TAG, "Failed to rebuild search index. Resetting tables. That will enqueue another copy of this job as a side-effect.")
+      SignalDatabase.messageSearch.fullyResetTables()
     }
   }
 

app/src/main/java/org/thoughtcrime/securesms/migrations/RebuildMessageSearchIndexMigrationJob.kt

@@ -22,7 +22,14 @@ internal class RebuildMessageSearchIndexMigrationJob(
 
   override fun performMigration() {
     val startTime = System.currentTimeMillis()
-    SignalDatabase.messageSearch.rebuildIndex()
+
+    val success = SignalDatabase.messageSearch.rebuildIndex()
+
+    if (!success) {
+      Log.w(TAG, "Failed to rebuild search index. Resetting tables. That will enqueue a job to reset the index as a side-effect.")
+      SignalDatabase.messageSearch.fullyResetTables()
+    }
+
     Log.d(TAG, "It took ${System.currentTimeMillis() - startTime} ms to rebuild the search index.")
   }