diff --git a/app/src/main/java/org/thoughtcrime/securesms/database/MessageTable.kt b/app/src/main/java/org/thoughtcrime/securesms/database/MessageTable.kt index f18f4f56af..7efd427c7b 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/database/MessageTable.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/database/MessageTable.kt @@ -112,6 +112,7 @@ import org.thoughtcrime.securesms.database.model.databaseprotos.SessionSwitchove import org.thoughtcrime.securesms.database.model.databaseprotos.ThreadMergeEvent import org.thoughtcrime.securesms.dependencies.AppDependencies import org.thoughtcrime.securesms.groups.GroupMigrationMembershipChange +import org.thoughtcrime.securesms.jobs.OptimizeMessageSearchIndexJob import org.thoughtcrime.securesms.jobs.ThreadUpdateJob import org.thoughtcrime.securesms.jobs.TrimThreadJob import org.thoughtcrime.securesms.keyvalue.SignalStore @@ -1653,7 +1654,7 @@ open class MessageTable(context: Context?, databaseHelper: SignalDatabase) : Dat .readToList { RecipientId.from(it.requireLong(FROM_RECIPIENT_ID)) } .forEach { id -> AppDependencies.databaseObserver.notifyStoryObservers(id) } - db.select(ID) + val deletedStoryCount = db.select(ID) .from(TABLE_NAME) .where(storiesBeforeTimestampWhere, sharedArgs) .run() @@ -1664,6 +1665,12 @@ open class MessageTable(context: Context?, databaseHelper: SignalDatabase) : Dat cursor.count } + + if (deletedStoryCount > 0) { + OptimizeMessageSearchIndexJob.enqueue() + } + + deletedStoryCount } } @@ -1706,7 +1713,7 @@ open class MessageTable(context: Context?, databaseHelper: SignalDatabase) : Dat AppDependencies.databaseObserver.notifyStoryObservers(recipientId) - db.select(ID) + val deletedStoryCount = db.select(ID) .from(TABLE_NAME) .where(storesInRecipientThread, sharedArgs) .run() @@ -1717,6 +1724,12 @@ open class MessageTable(context: Context?, databaseHelper: SignalDatabase) : Dat cursor.count } + + if (deletedStoryCount > 0) { + OptimizeMessageSearchIndexJob.enqueue() + } + + deletedStoryCount } } @@ -2097,6 +2110,7 @@ open class MessageTable(context: Context?, databaseHelper: SignalDatabase) : Dat threads.update(threadId, false) } + OptimizeMessageSearchIndexJob.enqueue() AppDependencies.databaseObserver.notifyMessageUpdateObservers(MessageId(messageId)) AppDependencies.databaseObserver.notifyConversationListListeners() @@ -3299,6 +3313,7 @@ open class MessageTable(context: Context?, databaseHelper: SignalDatabase) : Dat notifyConversationListeners(threadId) notifyStickerListeners() notifyStickerPackListeners() + OptimizeMessageSearchIndexJob.enqueue() } return threadDeleted @@ -3518,6 +3533,7 @@ open class MessageTable(context: Context?, databaseHelper: SignalDatabase) : Dat notifyConversationListListeners() notifyStickerListeners() notifyStickerPackListeners() + OptimizeMessageSearchIndexJob.enqueue() return unhandled } @@ -3539,6 +3555,8 @@ open class MessageTable(context: Context?, databaseHelper: SignalDatabase) : Dat mentions.deleteAllMentions() writableDatabase.deleteAll(TABLE_NAME) calls.updateCallEventDeletionTimestamps() + + OptimizeMessageSearchIndexJob.enqueue() } fun getNearestExpiringViewOnceMessage(): ViewOnceExpirationInfo? { diff --git a/app/src/main/java/org/thoughtcrime/securesms/database/SearchTable.kt b/app/src/main/java/org/thoughtcrime/securesms/database/SearchTable.kt index 7a1215544c..da4f1ebbee 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/database/SearchTable.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/database/SearchTable.kt @@ -7,7 +7,9 @@ import android.text.TextUtils import net.zetetic.database.sqlcipher.SQLiteDatabase import org.intellij.lang.annotations.Language import org.signal.core.util.SqlUtil +import org.signal.core.util.ThreadUtil import org.signal.core.util.logging.Log +import org.signal.core.util.withinTransaction import org.thoughtcrime.securesms.jobs.RebuildMessageSearchIndexJob /** @@ -34,10 +36,7 @@ class SearchTable(context: Context, databaseHelper: SignalDatabase) : DatabaseTa // We've taken the default of tokenize value of "unicode61 categories 'L* N* Co'" and added the Sc (currency) and So (emoji) categories to allow searching for those characters. // https://www.sqlite.org/fts5.html#tokenizers // https://www.compart.com/en/unicode/category - """CREATE VIRTUAL TABLE $FTS_TABLE_NAME USING fts5($BODY, $THREAD_ID UNINDEXED, content=${MessageTable.TABLE_NAME}, content_rowid=${MessageTable.ID}, tokenize = "unicode61 categories 'L* N* Co Sc So'")""", - - // Not technically a `CREATE` statement, but it's part of table creation. FTS5 just has weird configuration syntax. See https://www.sqlite.org/fts5.html#the_secure_delete_configuration_option - """INSERT INTO $FTS_TABLE_NAME ($FTS_TABLE_NAME, rank) VALUES('secure-delete', 1);""" + """CREATE VIRTUAL TABLE $FTS_TABLE_NAME USING fts5($BODY, $THREAD_ID UNINDEXED, content=${MessageTable.TABLE_NAME}, content_rowid=${MessageTable.ID}, tokenize = "unicode61 categories 'L* N* Co Sc So'")""" ) private const val TRIGGER_AFTER_INSERT = "message_ai" @@ -172,6 +171,73 @@ class SearchTable(context: Context, databaseHelper: SignalDatabase) : DatabaseTa } } + /** + * This performs the same thing as the `optimize` command in SQLite, but broken into iterative stages to avoid locking up the database for too long. + * If what's going on in this method seems weird, that's because it is, but please read the sqlite docs -- we're following their algorithm: + * https://www.sqlite.org/fts5.html#the_optimize_command + * + * Note that in order for the [SqlUtil.getTotalChanges] call to work, we have to be within a transaction, or else the connection pool screws everything up + * (the stats are on a per-connection basis). + * + * There's this double-batching mechanism happening here to strike a balance between making individual transactions short while also not hammering the + * database with a ton of independent transactions. + * + * To give you some ballpark numbers, on a large database (~400k messages), it takes ~75 iterations to fully optimize everything. + */ + fun optimizeIndex(timeout: Long): Boolean { + val pageSize = 64 // chosen through experimentation + val batchSize = 10 // chosen through experimentation + val noChangeThreshold = 2 // if less changes occurred than this, operation is considered no-op (see sqlite docs ref'd in kdoc) + + val startTime = System.currentTimeMillis() + var totalIterations = 0 + var totalBatches = 0 + var actualWorkTime = 0L + var finished = false + + while (!finished) { + var batchIterations = 0 + val batchStartTime = System.currentTimeMillis() + + writableDatabase.withinTransaction { db -> + // Note the negative page size -- see sqlite docs ref'd in kdoc + db.execSQL("INSERT INTO $FTS_TABLE_NAME ($FTS_TABLE_NAME, rank) values ('merge', -$pageSize)") + var previousCount = SqlUtil.getTotalChanges(db) + + val iterativeStatement = db.compileStatement("INSERT INTO $FTS_TABLE_NAME ($FTS_TABLE_NAME, rank) values ('merge', $pageSize)") + iterativeStatement.execute() + var count = SqlUtil.getTotalChanges(db) + + while (batchIterations < batchSize && count - previousCount >= noChangeThreshold) { + previousCount = count + iterativeStatement.execute() + + count = SqlUtil.getTotalChanges(db) + batchIterations++ + } + + if (count - previousCount < noChangeThreshold) { + finished = true + } + } + + totalIterations += batchIterations + totalBatches++ + actualWorkTime += System.currentTimeMillis() - batchStartTime + + if (actualWorkTime >= timeout) { + Log.w(TAG, "Timed out during optimization! We did $totalIterations iterations across $totalBatches batches, taking ${System.currentTimeMillis() - startTime} ms. Bailed out to avoid database lockup.") + return false + } + + // We want to sleep in between batches to give other db operations a chance to run + ThreadUtil.sleep(50) + } + + Log.d(TAG, "Took ${System.currentTimeMillis() - startTime} ms and $totalIterations iterations across $totalBatches batches to optimize. Of that time, $actualWorkTime ms were spent actually working (~${actualWorkTime / totalBatches} ms/batch). The rest was spent sleeping.") + return true + } + /** * Drops all tables and recreates them. */ diff --git a/app/src/main/java/org/thoughtcrime/securesms/database/ThreadTable.kt b/app/src/main/java/org/thoughtcrime/securesms/database/ThreadTable.kt index 9e2ffb530f..a16e0024ab 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/database/ThreadTable.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/database/ThreadTable.kt @@ -51,6 +51,7 @@ import org.thoughtcrime.securesms.dependencies.AppDependencies import org.thoughtcrime.securesms.groups.BadGroupIdException import org.thoughtcrime.securesms.groups.GroupId import org.thoughtcrime.securesms.jobs.MultiDeviceDeleteSyncJob +import org.thoughtcrime.securesms.jobs.OptimizeMessageSearchIndexJob import org.thoughtcrime.securesms.keyvalue.SignalStore import org.thoughtcrime.securesms.mms.SlideDeck import org.thoughtcrime.securesms.mms.StickerSlide @@ -362,6 +363,7 @@ class ThreadTable(context: Context, databaseHelper: SignalDatabase) : DatabaseTa notifyAttachmentListeners() notifyStickerPackListeners() + OptimizeMessageSearchIndexJob.enqueue() } fun trimThread( @@ -395,6 +397,7 @@ class ThreadTable(context: Context, databaseHelper: SignalDatabase) : DatabaseTa notifyAttachmentListeners() notifyStickerPackListeners() + OptimizeMessageSearchIndexJob.enqueue() } private fun trimThreadInternal( @@ -1185,6 +1188,8 @@ class ThreadTable(context: Context, databaseHelper: SignalDatabase) : DatabaseTa AppDependencies.databaseObserver.notifyConversationDeleteListeners(selectedConversations) ConversationUtil.clearShortcuts(context, recipientIds) + + OptimizeMessageSearchIndexJob.enqueue() } @SuppressLint("DiscouragedApi") diff --git a/app/src/main/java/org/thoughtcrime/securesms/database/helpers/SignalDatabaseMigrations.kt b/app/src/main/java/org/thoughtcrime/securesms/database/helpers/SignalDatabaseMigrations.kt index 0bc6165a0c..21e71679fe 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/database/helpers/SignalDatabaseMigrations.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/database/helpers/SignalDatabaseMigrations.kt @@ -100,6 +100,7 @@ import org.thoughtcrime.securesms.database.helpers.migration.V239_MessageFullTex import org.thoughtcrime.securesms.database.helpers.migration.V240_MessageFullTextSearchSecureDelete import org.thoughtcrime.securesms.database.helpers.migration.V241_ExpireTimerVersion import org.thoughtcrime.securesms.database.helpers.migration.V242_MessageFullTextSearchEmojiSupportV2 +import org.thoughtcrime.securesms.database.helpers.migration.V243_MessageFullTextSearchDisableSecureDelete /** * Contains all of the database migrations for [SignalDatabase]. Broken into a separate file for cleanliness. @@ -202,10 +203,11 @@ object SignalDatabaseMigrations { 239 to V239_MessageFullTextSearchEmojiSupport, 240 to V240_MessageFullTextSearchSecureDelete, 241 to V241_ExpireTimerVersion, - 242 to V242_MessageFullTextSearchEmojiSupportV2 + 242 to V242_MessageFullTextSearchEmojiSupportV2, + 243 to V243_MessageFullTextSearchDisableSecureDelete ) - const val DATABASE_VERSION = 242 + const val DATABASE_VERSION = 243 @JvmStatic fun migrate(context: Application, db: SQLiteDatabase, oldVersion: Int, newVersion: Int) { diff --git a/app/src/main/java/org/thoughtcrime/securesms/database/helpers/migration/V243_MessageFullTextSearchDisableSecureDelete.kt b/app/src/main/java/org/thoughtcrime/securesms/database/helpers/migration/V243_MessageFullTextSearchDisableSecureDelete.kt new file mode 100644 index 0000000000..cbcf7fc0a2 --- /dev/null +++ b/app/src/main/java/org/thoughtcrime/securesms/database/helpers/migration/V243_MessageFullTextSearchDisableSecureDelete.kt @@ -0,0 +1,21 @@ +/* + * Copyright 2024 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ + +package org.thoughtcrime.securesms.database.helpers.migration + +import android.app.Application +import net.zetetic.database.sqlcipher.SQLiteDatabase +import org.thoughtcrime.securesms.database.helpers.migration.V240_MessageFullTextSearchSecureDelete.FTS_TABLE_NAME + +/** + * This undoes [V240_MessageFullTextSearchSecureDelete] by disabling secure-delete on our FTS table. + * Unfortunately the performance overhead was too high. Thankfully, our old approach, while more + * manual, provides the same safety guarantees, while also allowing us to optimize bulk deletes. + */ +object V243_MessageFullTextSearchDisableSecureDelete : SignalDatabaseMigration { + override fun migrate(context: Application, db: SQLiteDatabase, oldVersion: Int, newVersion: Int) { + db.execSQL("""INSERT INTO $FTS_TABLE_NAME ($FTS_TABLE_NAME, rank) VALUES('secure-delete', 0);""") + } +} diff --git a/app/src/main/java/org/thoughtcrime/securesms/jobs/JobManagerFactories.java b/app/src/main/java/org/thoughtcrime/securesms/jobs/JobManagerFactories.java index 1e1c012681..933f950064 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/jobs/JobManagerFactories.java +++ b/app/src/main/java/org/thoughtcrime/securesms/jobs/JobManagerFactories.java @@ -61,6 +61,7 @@ import org.thoughtcrime.securesms.migrations.EmojiSearchIndexCheckMigrationJob; import org.thoughtcrime.securesms.migrations.IdentityTableCleanupMigrationJob; import org.thoughtcrime.securesms.migrations.LegacyMigrationJob; import org.thoughtcrime.securesms.migrations.MigrationCompleteJob; +import org.thoughtcrime.securesms.migrations.OptimizeMessageSearchIndexMigrationJob; import org.thoughtcrime.securesms.migrations.PassingMigrationJob; import org.thoughtcrime.securesms.migrations.PinOptOutMigration; import org.thoughtcrime.securesms.migrations.PinReminderMigrationJob; @@ -180,6 +181,7 @@ public final class JobManagerFactories { put(MultiDeviceViewOnceOpenJob.KEY, new MultiDeviceViewOnceOpenJob.Factory()); put(MultiDeviceViewedUpdateJob.KEY, new MultiDeviceViewedUpdateJob.Factory()); put(NullMessageSendJob.KEY, new NullMessageSendJob.Factory()); + put(OptimizeMessageSearchIndexJob.KEY, new OptimizeMessageSearchIndexJob.Factory()); put(PaymentLedgerUpdateJob.KEY, new PaymentLedgerUpdateJob.Factory()); put(PaymentNotificationSendJob.KEY, new PaymentNotificationSendJob.Factory()); put(PaymentNotificationSendJobV2.KEY, new PaymentNotificationSendJobV2.Factory()); @@ -269,6 +271,7 @@ public final class JobManagerFactories { put(IdentityTableCleanupMigrationJob.KEY, new IdentityTableCleanupMigrationJob.Factory()); put(LegacyMigrationJob.KEY, new LegacyMigrationJob.Factory()); put(MigrationCompleteJob.KEY, new MigrationCompleteJob.Factory()); + put(OptimizeMessageSearchIndexMigrationJob.KEY,new OptimizeMessageSearchIndexMigrationJob.Factory()); put(PinOptOutMigration.KEY, new PinOptOutMigration.Factory()); put(PinReminderMigrationJob.KEY, new PinReminderMigrationJob.Factory()); put(PniAccountInitializationMigrationJob.KEY, new PniAccountInitializationMigrationJob.Factory()); @@ -339,8 +342,6 @@ public final class JobManagerFactories { put("SmsSentJob", new FailingJob.Factory()); put("MmsSendJobV2", new FailingJob.Factory()); put("AttachmentUploadJobV2", new FailingJob.Factory()); - put("OptimizeMessageSearchIndexJob", new FailingJob.Factory()); - put("OptimizeMessageSearchIndexMigrationJob", new PassingMigrationJob.Factory()); }}; } diff --git a/app/src/main/java/org/thoughtcrime/securesms/jobs/OptimizeMessageSearchIndexJob.kt b/app/src/main/java/org/thoughtcrime/securesms/jobs/OptimizeMessageSearchIndexJob.kt new file mode 100644 index 0000000000..56393ec209 --- /dev/null +++ b/app/src/main/java/org/thoughtcrime/securesms/jobs/OptimizeMessageSearchIndexJob.kt @@ -0,0 +1,58 @@ +package org.thoughtcrime.securesms.jobs + +import org.signal.core.util.logging.Log +import org.thoughtcrime.securesms.database.SignalDatabase +import org.thoughtcrime.securesms.dependencies.AppDependencies +import org.thoughtcrime.securesms.jobmanager.Job +import org.thoughtcrime.securesms.keyvalue.SignalStore +import org.thoughtcrime.securesms.transport.RetryLaterException +import java.lang.Exception +import kotlin.time.Duration.Companion.seconds + +/** + * Optimizes the message search index incrementally. + */ +class OptimizeMessageSearchIndexJob private constructor(parameters: Parameters) : BaseJob(parameters) { + + companion object { + const val KEY = "OptimizeMessageSearchIndexJob" + + private val TAG = Log.tag(OptimizeMessageSearchIndexJob::class.java) + + @JvmStatic + fun enqueue() { + AppDependencies.jobManager.add(OptimizeMessageSearchIndexJob()) + } + } + + constructor() : this( + Parameters.Builder() + .setQueue("OptimizeMessageSearchIndexJob") + .setMaxAttempts(5) + .setMaxInstancesForQueue(2) + .build() + ) + + override fun serialize(): ByteArray? = null + override fun getFactoryKey() = KEY + override fun onFailure() = Unit + override fun onShouldRetry(e: Exception) = e is RetryLaterException + override fun getNextRunAttemptBackoff(pastAttemptCount: Int, exception: Exception): Long = 30.seconds.inWholeMilliseconds + + override fun onRun() { + if (!SignalStore.registration.isRegistrationComplete || SignalStore.account.aci == null) { + Log.w(TAG, "Registration not finished yet! Skipping.") + return + } + + val success = SignalDatabase.messageSearch.optimizeIndex(5.seconds.inWholeMilliseconds) + + if (!success) { + throw RetryLaterException() + } + } + + class Factory : Job.Factory { + override fun create(parameters: Parameters, serializedData: ByteArray?) = OptimizeMessageSearchIndexJob(parameters) + } +} diff --git a/app/src/main/java/org/thoughtcrime/securesms/migrations/ApplicationMigrations.java b/app/src/main/java/org/thoughtcrime/securesms/migrations/ApplicationMigrations.java index 0d55e699f5..65f8e57dcd 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/migrations/ApplicationMigrations.java +++ b/app/src/main/java/org/thoughtcrime/securesms/migrations/ApplicationMigrations.java @@ -116,7 +116,7 @@ public class ApplicationMigrations { static final int SMS_MMS_MERGE = 71; static final int REBUILD_MESSAGE_FTS_INDEX = 72; static final int UPDATE_SMS_JOBS = 73; -// static final int OPTIMIZE_MESSAGE_FTS_INDEX = 74; + static final int OPTIMIZE_MESSAGE_FTS_INDEX = 74; static final int REACTION_DATABASE_MIGRATION = 75; static final int REBUILD_MESSAGE_FTS_INDEX_2 = 76; static final int GLIDE_CACHE_CLEAR = 77; @@ -549,9 +549,9 @@ public class ApplicationMigrations { jobs.put(Version.UPDATE_SMS_JOBS, new UpdateSmsJobsMigrationJob()); } -// if (lastSeenVersion < Version.OPTIMIZE_MESSAGE_FTS_INDEX) { -// jobs.put(Version.OPTIMIZE_MESSAGE_FTS_INDEX, new OptimizeMessageSearchIndexMigrationJob()); -// } + if (lastSeenVersion < Version.OPTIMIZE_MESSAGE_FTS_INDEX) { + jobs.put(Version.OPTIMIZE_MESSAGE_FTS_INDEX, new OptimizeMessageSearchIndexMigrationJob()); + } if (lastSeenVersion < Version.REACTION_DATABASE_MIGRATION) { jobs.put(Version.REACTION_DATABASE_MIGRATION, new DatabaseMigrationJob()); diff --git a/app/src/main/java/org/thoughtcrime/securesms/migrations/OptimizeMessageSearchIndexMigrationJob.kt b/app/src/main/java/org/thoughtcrime/securesms/migrations/OptimizeMessageSearchIndexMigrationJob.kt new file mode 100644 index 0000000000..d4dc7d0cce --- /dev/null +++ b/app/src/main/java/org/thoughtcrime/securesms/migrations/OptimizeMessageSearchIndexMigrationJob.kt @@ -0,0 +1,34 @@ +package org.thoughtcrime.securesms.migrations + +import org.signal.core.util.logging.Log +import org.thoughtcrime.securesms.jobmanager.Job +import org.thoughtcrime.securesms.jobs.OptimizeMessageSearchIndexJob + +/** + * Kicks off a job to optimize the message search index. + */ +internal class OptimizeMessageSearchIndexMigrationJob( + parameters: Parameters = Parameters.Builder().build() +) : MigrationJob(parameters) { + + companion object { + val TAG = Log.tag(OptimizeMessageSearchIndexMigrationJob::class.java) + const val KEY = "OptimizeMessageSearchIndexMigrationJob" + } + + override fun getFactoryKey(): String = KEY + + override fun isUiBlocking(): Boolean = false + + override fun performMigration() { + OptimizeMessageSearchIndexJob.enqueue() + } + + override fun shouldRetry(e: Exception): Boolean = false + + class Factory : Job.Factory { + override fun create(parameters: Parameters, serializedData: ByteArray?): OptimizeMessageSearchIndexMigrationJob { + return OptimizeMessageSearchIndexMigrationJob(parameters) + } + } +}