diff --git a/app/src/main/java/org/thoughtcrime/securesms/database/MessageTable.kt b/app/src/main/java/org/thoughtcrime/securesms/database/MessageTable.kt index 5b624b4c05..076d6a1f8a 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/database/MessageTable.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/database/MessageTable.kt @@ -112,7 +112,6 @@ import org.thoughtcrime.securesms.database.model.databaseprotos.SessionSwitchove import org.thoughtcrime.securesms.database.model.databaseprotos.ThreadMergeEvent import org.thoughtcrime.securesms.dependencies.AppDependencies import org.thoughtcrime.securesms.groups.GroupMigrationMembershipChange -import org.thoughtcrime.securesms.jobs.OptimizeMessageSearchIndexJob import org.thoughtcrime.securesms.jobs.ThreadUpdateJob import org.thoughtcrime.securesms.jobs.TrimThreadJob import org.thoughtcrime.securesms.keyvalue.SignalStore @@ -1651,7 +1650,7 @@ open class MessageTable(context: Context?, databaseHelper: SignalDatabase) : Dat .readToList { RecipientId.from(it.requireLong(FROM_RECIPIENT_ID)) } .forEach { id -> AppDependencies.databaseObserver.notifyStoryObservers(id) } - val deletedStoryCount = db.select(ID) + db.select(ID) .from(TABLE_NAME) .where(storiesBeforeTimestampWhere, sharedArgs) .run() @@ -1662,12 +1661,6 @@ open class MessageTable(context: Context?, databaseHelper: SignalDatabase) : Dat cursor.count } - - if (deletedStoryCount > 0) { - OptimizeMessageSearchIndexJob.enqueue() - } - - deletedStoryCount } } @@ -1710,7 +1703,7 @@ open class MessageTable(context: Context?, databaseHelper: SignalDatabase) : Dat AppDependencies.databaseObserver.notifyStoryObservers(recipientId) - val deletedStoryCount = db.select(ID) + db.select(ID) .from(TABLE_NAME) .where(storesInRecipientThread, sharedArgs) .run() @@ -1721,12 +1714,6 @@ open class MessageTable(context: Context?, databaseHelper: SignalDatabase) : Dat cursor.count } - - if (deletedStoryCount > 0) { - OptimizeMessageSearchIndexJob.enqueue() - } - - deletedStoryCount } } @@ -2107,7 +2094,6 @@ open class MessageTable(context: Context?, databaseHelper: SignalDatabase) : Dat threads.update(threadId, false) } - OptimizeMessageSearchIndexJob.enqueue() AppDependencies.databaseObserver.notifyMessageUpdateObservers(MessageId(messageId)) AppDependencies.databaseObserver.notifyConversationListListeners() @@ -3306,7 +3292,6 @@ open class MessageTable(context: Context?, databaseHelper: SignalDatabase) : Dat notifyConversationListeners(threadId) notifyStickerListeners() notifyStickerPackListeners() - OptimizeMessageSearchIndexJob.enqueue() } return threadDeleted @@ -3526,7 +3511,6 @@ open class MessageTable(context: Context?, databaseHelper: SignalDatabase) : Dat notifyConversationListListeners() notifyStickerListeners() notifyStickerPackListeners() - OptimizeMessageSearchIndexJob.enqueue() return unhandled } @@ -3548,8 +3532,6 @@ open class MessageTable(context: Context?, databaseHelper: SignalDatabase) : Dat mentions.deleteAllMentions() writableDatabase.deleteAll(TABLE_NAME) calls.updateCallEventDeletionTimestamps() - - OptimizeMessageSearchIndexJob.enqueue() } fun getNearestExpiringViewOnceMessage(): ViewOnceExpirationInfo? { diff --git a/app/src/main/java/org/thoughtcrime/securesms/database/SearchTable.kt b/app/src/main/java/org/thoughtcrime/securesms/database/SearchTable.kt index 8932576cdd..6c2db11b19 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/database/SearchTable.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/database/SearchTable.kt @@ -7,9 +7,7 @@ import android.text.TextUtils import net.zetetic.database.sqlcipher.SQLiteDatabase import org.intellij.lang.annotations.Language import org.signal.core.util.SqlUtil -import org.signal.core.util.ThreadUtil import org.signal.core.util.logging.Log -import org.signal.core.util.withinTransaction import org.thoughtcrime.securesms.jobs.RebuildMessageSearchIndexJob /** @@ -36,7 +34,10 @@ class SearchTable(context: Context, databaseHelper: SignalDatabase) : DatabaseTa // We've taken the default of tokenize value of "unicode61 categories 'L* N* Co'" and added the Sc (currency) and So (emoji) categories to allow searching for those characters. // https://www.sqlite.org/fts5.html#tokenizers // https://www.compart.com/en/unicode/category - """CREATE VIRTUAL TABLE $FTS_TABLE_NAME USING fts5($BODY, $THREAD_ID UNINDEXED, content=${MessageTable.TABLE_NAME}, content_rowid=${MessageTable.ID}, tokenize = "unicode61 categories 'L* N* Co Sc So'")""" + """CREATE VIRTUAL TABLE $FTS_TABLE_NAME USING fts5($BODY, $THREAD_ID UNINDEXED, content=${MessageTable.TABLE_NAME}, content_rowid=${MessageTable.ID}, tokenize = "unicode61 categories 'L* N* Co Sc So'")""", + + // Not technically a `CREATE` statement, but it's part of table creation. FTS5 just has weird configuration syntax. See https://www.sqlite.org/fts5.html#the_secure_delete_configuration_option + """INSERT INTO $FTS_TABLE_NAME ($FTS_TABLE_NAME, rank) VALUES('secure-delete', 1);""" ) private const val TRIGGER_AFTER_INSERT = "message_ai" @@ -165,73 +166,6 @@ class SearchTable(context: Context, databaseHelper: SignalDatabase) : DatabaseTa } } - /** - * This performs the same thing as the `optimize` command in SQLite, but broken into iterative stages to avoid locking up the database for too long. - * If what's going on in this method seems weird, that's because it is, but please read the sqlite docs -- we're following their algorithm: - * https://www.sqlite.org/fts5.html#the_optimize_command - * - * Note that in order for the [SqlUtil.getTotalChanges] call to work, we have to be within a transaction, or else the connection pool screws everything up - * (the stats are on a per-connection basis). - * - * There's this double-batching mechanism happening here to strike a balance between making individual transactions short while also not hammering the - * database with a ton of independent transactions. - * - * To give you some ballpark numbers, on a large database (~400k messages), it takes ~75 iterations to fully optimize everything. - */ - fun optimizeIndex(timeout: Long): Boolean { - val pageSize = 64 // chosen through experimentation - val batchSize = 10 // chosen through experimentation - val noChangeThreshold = 2 // if less changes occurred than this, operation is considered no-op (see sqlite docs ref'd in kdoc) - - val startTime = System.currentTimeMillis() - var totalIterations = 0 - var totalBatches = 0 - var actualWorkTime = 0L - var finished = false - - while (!finished) { - var batchIterations = 0 - val batchStartTime = System.currentTimeMillis() - - writableDatabase.withinTransaction { db -> - // Note the negative page size -- see sqlite docs ref'd in kdoc - db.execSQL("INSERT INTO $FTS_TABLE_NAME ($FTS_TABLE_NAME, rank) values ('merge', -$pageSize)") - var previousCount = SqlUtil.getTotalChanges(db) - - val iterativeStatement = db.compileStatement("INSERT INTO $FTS_TABLE_NAME ($FTS_TABLE_NAME, rank) values ('merge', $pageSize)") - iterativeStatement.execute() - var count = SqlUtil.getTotalChanges(db) - - while (batchIterations < batchSize && count - previousCount >= noChangeThreshold) { - previousCount = count - iterativeStatement.execute() - - count = SqlUtil.getTotalChanges(db) - batchIterations++ - } - - if (count - previousCount < noChangeThreshold) { - finished = true - } - } - - totalIterations += batchIterations - totalBatches++ - actualWorkTime += System.currentTimeMillis() - batchStartTime - - if (actualWorkTime >= timeout) { - Log.w(TAG, "Timed out during optimization! We did $totalIterations iterations across $totalBatches batches, taking ${System.currentTimeMillis() - startTime} ms. Bailed out to avoid database lockup.") - return false - } - - // We want to sleep in between batches to give other db operations a chance to run - ThreadUtil.sleep(50) - } - - Log.d(TAG, "Took ${System.currentTimeMillis() - startTime} ms and $totalIterations iterations across $totalBatches batches to optimize. Of that time, $actualWorkTime ms were spent actually working (~${actualWorkTime / totalBatches} ms/batch). The rest was spent sleeping.") - return true - } - /** * Drops all tables and recreates them. */ diff --git a/app/src/main/java/org/thoughtcrime/securesms/database/ThreadTable.kt b/app/src/main/java/org/thoughtcrime/securesms/database/ThreadTable.kt index a16e0024ab..9e2ffb530f 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/database/ThreadTable.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/database/ThreadTable.kt @@ -51,7 +51,6 @@ import org.thoughtcrime.securesms.dependencies.AppDependencies import org.thoughtcrime.securesms.groups.BadGroupIdException import org.thoughtcrime.securesms.groups.GroupId import org.thoughtcrime.securesms.jobs.MultiDeviceDeleteSyncJob -import org.thoughtcrime.securesms.jobs.OptimizeMessageSearchIndexJob import org.thoughtcrime.securesms.keyvalue.SignalStore import org.thoughtcrime.securesms.mms.SlideDeck import org.thoughtcrime.securesms.mms.StickerSlide @@ -363,7 +362,6 @@ class ThreadTable(context: Context, databaseHelper: SignalDatabase) : DatabaseTa notifyAttachmentListeners() notifyStickerPackListeners() - OptimizeMessageSearchIndexJob.enqueue() } fun trimThread( @@ -397,7 +395,6 @@ class ThreadTable(context: Context, databaseHelper: SignalDatabase) : DatabaseTa notifyAttachmentListeners() notifyStickerPackListeners() - OptimizeMessageSearchIndexJob.enqueue() } private fun trimThreadInternal( @@ -1188,8 +1185,6 @@ class ThreadTable(context: Context, databaseHelper: SignalDatabase) : DatabaseTa AppDependencies.databaseObserver.notifyConversationDeleteListeners(selectedConversations) ConversationUtil.clearShortcuts(context, recipientIds) - - OptimizeMessageSearchIndexJob.enqueue() } @SuppressLint("DiscouragedApi") diff --git a/app/src/main/java/org/thoughtcrime/securesms/database/helpers/SignalDatabaseMigrations.kt b/app/src/main/java/org/thoughtcrime/securesms/database/helpers/SignalDatabaseMigrations.kt index a37568082d..0a7c79a597 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/database/helpers/SignalDatabaseMigrations.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/database/helpers/SignalDatabaseMigrations.kt @@ -97,6 +97,7 @@ import org.thoughtcrime.securesms.database.helpers.migration.V236_FixInAppSubscr import org.thoughtcrime.securesms.database.helpers.migration.V237_ResetGroupForceUpdateTimestamps import org.thoughtcrime.securesms.database.helpers.migration.V238_AddGroupSendEndorsementsColumns import org.thoughtcrime.securesms.database.helpers.migration.V239_MessageFullTextSearchEmojiSupport +import org.thoughtcrime.securesms.database.helpers.migration.V240_MessageFullTextSearchSecureDelete /** * Contains all of the database migrations for [SignalDatabase]. Broken into a separate file for cleanliness. @@ -196,10 +197,11 @@ object SignalDatabaseMigrations { 236 to V236_FixInAppSubscriberCurrencyIfAble, 237 to V237_ResetGroupForceUpdateTimestamps, 238 to V238_AddGroupSendEndorsementsColumns, - 239 to V239_MessageFullTextSearchEmojiSupport + 239 to V239_MessageFullTextSearchEmojiSupport, + 240 to V240_MessageFullTextSearchSecureDelete ) - const val DATABASE_VERSION = 239 + const val DATABASE_VERSION = 240 @JvmStatic fun migrate(context: Application, db: SQLiteDatabase, oldVersion: Int, newVersion: Int) { diff --git a/app/src/main/java/org/thoughtcrime/securesms/database/helpers/migration/V240_MessageFullTextSearchSecureDelete.kt b/app/src/main/java/org/thoughtcrime/securesms/database/helpers/migration/V240_MessageFullTextSearchSecureDelete.kt new file mode 100644 index 0000000000..62904e167e --- /dev/null +++ b/app/src/main/java/org/thoughtcrime/securesms/database/helpers/migration/V240_MessageFullTextSearchSecureDelete.kt @@ -0,0 +1,23 @@ +/* + * Copyright 2024 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ + +package org.thoughtcrime.securesms.database.helpers.migration + +import android.app.Application +import net.zetetic.database.sqlcipher.SQLiteDatabase + +/** + * Sets the 'secure-delete' flag on the message_fts table. + * https://www.sqlite.org/fts5.html#the_secure_delete_configuration_option + */ +@Suppress("ClassName") +object V240_MessageFullTextSearchSecureDelete : SignalDatabaseMigration { + + const val FTS_TABLE_NAME = "message_fts" + + override fun migrate(context: Application, db: SQLiteDatabase, oldVersion: Int, newVersion: Int) { + db.execSQL("""INSERT INTO $FTS_TABLE_NAME ($FTS_TABLE_NAME, rank) VALUES('secure-delete', 1);""") + } +} diff --git a/app/src/main/java/org/thoughtcrime/securesms/jobs/JobManagerFactories.java b/app/src/main/java/org/thoughtcrime/securesms/jobs/JobManagerFactories.java index 933f950064..1e1c012681 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/jobs/JobManagerFactories.java +++ b/app/src/main/java/org/thoughtcrime/securesms/jobs/JobManagerFactories.java @@ -61,7 +61,6 @@ import org.thoughtcrime.securesms.migrations.EmojiSearchIndexCheckMigrationJob; import org.thoughtcrime.securesms.migrations.IdentityTableCleanupMigrationJob; import org.thoughtcrime.securesms.migrations.LegacyMigrationJob; import org.thoughtcrime.securesms.migrations.MigrationCompleteJob; -import org.thoughtcrime.securesms.migrations.OptimizeMessageSearchIndexMigrationJob; import org.thoughtcrime.securesms.migrations.PassingMigrationJob; import org.thoughtcrime.securesms.migrations.PinOptOutMigration; import org.thoughtcrime.securesms.migrations.PinReminderMigrationJob; @@ -181,7 +180,6 @@ public final class JobManagerFactories { put(MultiDeviceViewOnceOpenJob.KEY, new MultiDeviceViewOnceOpenJob.Factory()); put(MultiDeviceViewedUpdateJob.KEY, new MultiDeviceViewedUpdateJob.Factory()); put(NullMessageSendJob.KEY, new NullMessageSendJob.Factory()); - put(OptimizeMessageSearchIndexJob.KEY, new OptimizeMessageSearchIndexJob.Factory()); put(PaymentLedgerUpdateJob.KEY, new PaymentLedgerUpdateJob.Factory()); put(PaymentNotificationSendJob.KEY, new PaymentNotificationSendJob.Factory()); put(PaymentNotificationSendJobV2.KEY, new PaymentNotificationSendJobV2.Factory()); @@ -271,7 +269,6 @@ public final class JobManagerFactories { put(IdentityTableCleanupMigrationJob.KEY, new IdentityTableCleanupMigrationJob.Factory()); put(LegacyMigrationJob.KEY, new LegacyMigrationJob.Factory()); put(MigrationCompleteJob.KEY, new MigrationCompleteJob.Factory()); - put(OptimizeMessageSearchIndexMigrationJob.KEY,new OptimizeMessageSearchIndexMigrationJob.Factory()); put(PinOptOutMigration.KEY, new PinOptOutMigration.Factory()); put(PinReminderMigrationJob.KEY, new PinReminderMigrationJob.Factory()); put(PniAccountInitializationMigrationJob.KEY, new PniAccountInitializationMigrationJob.Factory()); @@ -342,6 +339,8 @@ public final class JobManagerFactories { put("SmsSentJob", new FailingJob.Factory()); put("MmsSendJobV2", new FailingJob.Factory()); put("AttachmentUploadJobV2", new FailingJob.Factory()); + put("OptimizeMessageSearchIndexJob", new FailingJob.Factory()); + put("OptimizeMessageSearchIndexMigrationJob", new PassingMigrationJob.Factory()); }}; } diff --git a/app/src/main/java/org/thoughtcrime/securesms/jobs/OptimizeMessageSearchIndexJob.kt b/app/src/main/java/org/thoughtcrime/securesms/jobs/OptimizeMessageSearchIndexJob.kt deleted file mode 100644 index 56393ec209..0000000000 --- a/app/src/main/java/org/thoughtcrime/securesms/jobs/OptimizeMessageSearchIndexJob.kt +++ /dev/null @@ -1,58 +0,0 @@ -package org.thoughtcrime.securesms.jobs - -import org.signal.core.util.logging.Log -import org.thoughtcrime.securesms.database.SignalDatabase -import org.thoughtcrime.securesms.dependencies.AppDependencies -import org.thoughtcrime.securesms.jobmanager.Job -import org.thoughtcrime.securesms.keyvalue.SignalStore -import org.thoughtcrime.securesms.transport.RetryLaterException -import java.lang.Exception -import kotlin.time.Duration.Companion.seconds - -/** - * Optimizes the message search index incrementally. - */ -class OptimizeMessageSearchIndexJob private constructor(parameters: Parameters) : BaseJob(parameters) { - - companion object { - const val KEY = "OptimizeMessageSearchIndexJob" - - private val TAG = Log.tag(OptimizeMessageSearchIndexJob::class.java) - - @JvmStatic - fun enqueue() { - AppDependencies.jobManager.add(OptimizeMessageSearchIndexJob()) - } - } - - constructor() : this( - Parameters.Builder() - .setQueue("OptimizeMessageSearchIndexJob") - .setMaxAttempts(5) - .setMaxInstancesForQueue(2) - .build() - ) - - override fun serialize(): ByteArray? = null - override fun getFactoryKey() = KEY - override fun onFailure() = Unit - override fun onShouldRetry(e: Exception) = e is RetryLaterException - override fun getNextRunAttemptBackoff(pastAttemptCount: Int, exception: Exception): Long = 30.seconds.inWholeMilliseconds - - override fun onRun() { - if (!SignalStore.registration.isRegistrationComplete || SignalStore.account.aci == null) { - Log.w(TAG, "Registration not finished yet! Skipping.") - return - } - - val success = SignalDatabase.messageSearch.optimizeIndex(5.seconds.inWholeMilliseconds) - - if (!success) { - throw RetryLaterException() - } - } - - class Factory : Job.Factory { - override fun create(parameters: Parameters, serializedData: ByteArray?) = OptimizeMessageSearchIndexJob(parameters) - } -} diff --git a/app/src/main/java/org/thoughtcrime/securesms/migrations/ApplicationMigrations.java b/app/src/main/java/org/thoughtcrime/securesms/migrations/ApplicationMigrations.java index 7343b8968f..083e47c363 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/migrations/ApplicationMigrations.java +++ b/app/src/main/java/org/thoughtcrime/securesms/migrations/ApplicationMigrations.java @@ -116,7 +116,7 @@ public class ApplicationMigrations { static final int SMS_MMS_MERGE = 71; static final int REBUILD_MESSAGE_FTS_INDEX = 72; static final int UPDATE_SMS_JOBS = 73; - static final int OPTIMIZE_MESSAGE_FTS_INDEX = 74; +// static final int OPTIMIZE_MESSAGE_FTS_INDEX = 74; static final int REACTION_DATABASE_MIGRATION = 75; static final int REBUILD_MESSAGE_FTS_INDEX_2 = 76; static final int GLIDE_CACHE_CLEAR = 77; @@ -547,9 +547,9 @@ public class ApplicationMigrations { jobs.put(Version.UPDATE_SMS_JOBS, new UpdateSmsJobsMigrationJob()); } - if (lastSeenVersion < Version.OPTIMIZE_MESSAGE_FTS_INDEX) { - jobs.put(Version.OPTIMIZE_MESSAGE_FTS_INDEX, new OptimizeMessageSearchIndexMigrationJob()); - } +// if (lastSeenVersion < Version.OPTIMIZE_MESSAGE_FTS_INDEX) { +// jobs.put(Version.OPTIMIZE_MESSAGE_FTS_INDEX, new OptimizeMessageSearchIndexMigrationJob()); +// } if (lastSeenVersion < Version.REACTION_DATABASE_MIGRATION) { jobs.put(Version.REACTION_DATABASE_MIGRATION, new DatabaseMigrationJob()); diff --git a/app/src/main/java/org/thoughtcrime/securesms/migrations/OptimizeMessageSearchIndexMigrationJob.kt b/app/src/main/java/org/thoughtcrime/securesms/migrations/OptimizeMessageSearchIndexMigrationJob.kt deleted file mode 100644 index d4dc7d0cce..0000000000 --- a/app/src/main/java/org/thoughtcrime/securesms/migrations/OptimizeMessageSearchIndexMigrationJob.kt +++ /dev/null @@ -1,34 +0,0 @@ -package org.thoughtcrime.securesms.migrations - -import org.signal.core.util.logging.Log -import org.thoughtcrime.securesms.jobmanager.Job -import org.thoughtcrime.securesms.jobs.OptimizeMessageSearchIndexJob - -/** - * Kicks off a job to optimize the message search index. - */ -internal class OptimizeMessageSearchIndexMigrationJob( - parameters: Parameters = Parameters.Builder().build() -) : MigrationJob(parameters) { - - companion object { - val TAG = Log.tag(OptimizeMessageSearchIndexMigrationJob::class.java) - const val KEY = "OptimizeMessageSearchIndexMigrationJob" - } - - override fun getFactoryKey(): String = KEY - - override fun isUiBlocking(): Boolean = false - - override fun performMigration() { - OptimizeMessageSearchIndexJob.enqueue() - } - - override fun shouldRetry(e: Exception): Boolean = false - - class Factory : Job.Factory { - override fun create(parameters: Parameters, serializedData: ByteArray?): OptimizeMessageSearchIndexMigrationJob { - return OptimizeMessageSearchIndexMigrationJob(parameters) - } - } -}