mirror of
https://github.com/signalapp/Signal-Android.git
synced 2025-12-22 20:18:36 +00:00
Replace manual FTS5 fix with SQLite secure-delete flag.
We used to workaround this by manually optimizing the search index, but secure-delete does that for us with less work.
This commit is contained in:
Greyson Parrelli
committed by
Nicholas Tinsley
Nicholas Tinsley
parent
2cfd19add6
commit
1aaa833127
@@ -112,7 +112,6 @@ import org.thoughtcrime.securesms.database.model.databaseprotos.SessionSwitchove
|
||||
import org.thoughtcrime.securesms.database.model.databaseprotos.ThreadMergeEvent
|
||||
import org.thoughtcrime.securesms.dependencies.AppDependencies
|
||||
import org.thoughtcrime.securesms.groups.GroupMigrationMembershipChange
|
||||
import org.thoughtcrime.securesms.jobs.OptimizeMessageSearchIndexJob
|
||||
import org.thoughtcrime.securesms.jobs.ThreadUpdateJob
|
||||
import org.thoughtcrime.securesms.jobs.TrimThreadJob
|
||||
import org.thoughtcrime.securesms.keyvalue.SignalStore
|
||||
@@ -1651,7 +1650,7 @@ open class MessageTable(context: Context?, databaseHelper: SignalDatabase) : Dat
|
||||
.readToList { RecipientId.from(it.requireLong(FROM_RECIPIENT_ID)) }
|
||||
.forEach { id -> AppDependencies.databaseObserver.notifyStoryObservers(id) }
|
||||
|
||||
val deletedStoryCount = db.select(ID)
|
||||
db.select(ID)
|
||||
.from(TABLE_NAME)
|
||||
.where(storiesBeforeTimestampWhere, sharedArgs)
|
||||
.run()
|
||||
@@ -1662,12 +1661,6 @@ open class MessageTable(context: Context?, databaseHelper: SignalDatabase) : Dat
|
||||
|
||||
cursor.count
|
||||
}
|
||||
|
||||
if (deletedStoryCount > 0) {
|
||||
OptimizeMessageSearchIndexJob.enqueue()
|
||||
}
|
||||
|
||||
deletedStoryCount
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1710,7 +1703,7 @@ open class MessageTable(context: Context?, databaseHelper: SignalDatabase) : Dat
|
||||
|
||||
AppDependencies.databaseObserver.notifyStoryObservers(recipientId)
|
||||
|
||||
val deletedStoryCount = db.select(ID)
|
||||
db.select(ID)
|
||||
.from(TABLE_NAME)
|
||||
.where(storesInRecipientThread, sharedArgs)
|
||||
.run()
|
||||
@@ -1721,12 +1714,6 @@ open class MessageTable(context: Context?, databaseHelper: SignalDatabase) : Dat
|
||||
|
||||
cursor.count
|
||||
}
|
||||
|
||||
if (deletedStoryCount > 0) {
|
||||
OptimizeMessageSearchIndexJob.enqueue()
|
||||
}
|
||||
|
||||
deletedStoryCount
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2107,7 +2094,6 @@ open class MessageTable(context: Context?, databaseHelper: SignalDatabase) : Dat
|
||||
threads.update(threadId, false)
|
||||
}
|
||||
|
||||
OptimizeMessageSearchIndexJob.enqueue()
|
||||
AppDependencies.databaseObserver.notifyMessageUpdateObservers(MessageId(messageId))
|
||||
AppDependencies.databaseObserver.notifyConversationListListeners()
|
||||
|
||||
@@ -3306,7 +3292,6 @@ open class MessageTable(context: Context?, databaseHelper: SignalDatabase) : Dat
|
||||
notifyConversationListeners(threadId)
|
||||
notifyStickerListeners()
|
||||
notifyStickerPackListeners()
|
||||
OptimizeMessageSearchIndexJob.enqueue()
|
||||
}
|
||||
|
||||
return threadDeleted
|
||||
@@ -3526,7 +3511,6 @@ open class MessageTable(context: Context?, databaseHelper: SignalDatabase) : Dat
|
||||
notifyConversationListListeners()
|
||||
notifyStickerListeners()
|
||||
notifyStickerPackListeners()
|
||||
OptimizeMessageSearchIndexJob.enqueue()
|
||||
|
||||
return unhandled
|
||||
}
|
||||
@@ -3548,8 +3532,6 @@ open class MessageTable(context: Context?, databaseHelper: SignalDatabase) : Dat
|
||||
mentions.deleteAllMentions()
|
||||
writableDatabase.deleteAll(TABLE_NAME)
|
||||
calls.updateCallEventDeletionTimestamps()
|
||||
|
||||
OptimizeMessageSearchIndexJob.enqueue()
|
||||
}
|
||||
|
||||
fun getNearestExpiringViewOnceMessage(): ViewOnceExpirationInfo? {
|
||||
|
||||
@@ -7,9 +7,7 @@ import android.text.TextUtils
|
||||
import net.zetetic.database.sqlcipher.SQLiteDatabase
|
||||
import org.intellij.lang.annotations.Language
|
||||
import org.signal.core.util.SqlUtil
|
||||
import org.signal.core.util.ThreadUtil
|
||||
import org.signal.core.util.logging.Log
|
||||
import org.signal.core.util.withinTransaction
|
||||
import org.thoughtcrime.securesms.jobs.RebuildMessageSearchIndexJob
|
||||
|
||||
/**
|
||||
@@ -36,7 +34,10 @@ class SearchTable(context: Context, databaseHelper: SignalDatabase) : DatabaseTa
|
||||
// We've taken the default of tokenize value of "unicode61 categories 'L* N* Co'" and added the Sc (currency) and So (emoji) categories to allow searching for those characters.
|
||||
// https://www.sqlite.org/fts5.html#tokenizers
|
||||
// https://www.compart.com/en/unicode/category
|
||||
"""CREATE VIRTUAL TABLE $FTS_TABLE_NAME USING fts5($BODY, $THREAD_ID UNINDEXED, content=${MessageTable.TABLE_NAME}, content_rowid=${MessageTable.ID}, tokenize = "unicode61 categories 'L* N* Co Sc So'")"""
|
||||
"""CREATE VIRTUAL TABLE $FTS_TABLE_NAME USING fts5($BODY, $THREAD_ID UNINDEXED, content=${MessageTable.TABLE_NAME}, content_rowid=${MessageTable.ID}, tokenize = "unicode61 categories 'L* N* Co Sc So'")""",
|
||||
|
||||
// Not technically a `CREATE` statement, but it's part of table creation. FTS5 just has weird configuration syntax. See https://www.sqlite.org/fts5.html#the_secure_delete_configuration_option
|
||||
"""INSERT INTO $FTS_TABLE_NAME ($FTS_TABLE_NAME, rank) VALUES('secure-delete', 1);"""
|
||||
)
|
||||
|
||||
private const val TRIGGER_AFTER_INSERT = "message_ai"
|
||||
@@ -165,73 +166,6 @@ class SearchTable(context: Context, databaseHelper: SignalDatabase) : DatabaseTa
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* This performs the same thing as the `optimize` command in SQLite, but broken into iterative stages to avoid locking up the database for too long.
|
||||
* If what's going on in this method seems weird, that's because it is, but please read the sqlite docs -- we're following their algorithm:
|
||||
* https://www.sqlite.org/fts5.html#the_optimize_command
|
||||
*
|
||||
* Note that in order for the [SqlUtil.getTotalChanges] call to work, we have to be within a transaction, or else the connection pool screws everything up
|
||||
* (the stats are on a per-connection basis).
|
||||
*
|
||||
* There's this double-batching mechanism happening here to strike a balance between making individual transactions short while also not hammering the
|
||||
* database with a ton of independent transactions.
|
||||
*
|
||||
* To give you some ballpark numbers, on a large database (~400k messages), it takes ~75 iterations to fully optimize everything.
|
||||
*/
|
||||
fun optimizeIndex(timeout: Long): Boolean {
|
||||
val pageSize = 64 // chosen through experimentation
|
||||
val batchSize = 10 // chosen through experimentation
|
||||
val noChangeThreshold = 2 // if less changes occurred than this, operation is considered no-op (see sqlite docs ref'd in kdoc)
|
||||
|
||||
val startTime = System.currentTimeMillis()
|
||||
var totalIterations = 0
|
||||
var totalBatches = 0
|
||||
var actualWorkTime = 0L
|
||||
var finished = false
|
||||
|
||||
while (!finished) {
|
||||
var batchIterations = 0
|
||||
val batchStartTime = System.currentTimeMillis()
|
||||
|
||||
writableDatabase.withinTransaction { db ->
|
||||
// Note the negative page size -- see sqlite docs ref'd in kdoc
|
||||
db.execSQL("INSERT INTO $FTS_TABLE_NAME ($FTS_TABLE_NAME, rank) values ('merge', -$pageSize)")
|
||||
var previousCount = SqlUtil.getTotalChanges(db)
|
||||
|
||||
val iterativeStatement = db.compileStatement("INSERT INTO $FTS_TABLE_NAME ($FTS_TABLE_NAME, rank) values ('merge', $pageSize)")
|
||||
iterativeStatement.execute()
|
||||
var count = SqlUtil.getTotalChanges(db)
|
||||
|
||||
while (batchIterations < batchSize && count - previousCount >= noChangeThreshold) {
|
||||
previousCount = count
|
||||
iterativeStatement.execute()
|
||||
|
||||
count = SqlUtil.getTotalChanges(db)
|
||||
batchIterations++
|
||||
}
|
||||
|
||||
if (count - previousCount < noChangeThreshold) {
|
||||
finished = true
|
||||
}
|
||||
}
|
||||
|
||||
totalIterations += batchIterations
|
||||
totalBatches++
|
||||
actualWorkTime += System.currentTimeMillis() - batchStartTime
|
||||
|
||||
if (actualWorkTime >= timeout) {
|
||||
Log.w(TAG, "Timed out during optimization! We did $totalIterations iterations across $totalBatches batches, taking ${System.currentTimeMillis() - startTime} ms. Bailed out to avoid database lockup.")
|
||||
return false
|
||||
}
|
||||
|
||||
// We want to sleep in between batches to give other db operations a chance to run
|
||||
ThreadUtil.sleep(50)
|
||||
}
|
||||
|
||||
Log.d(TAG, "Took ${System.currentTimeMillis() - startTime} ms and $totalIterations iterations across $totalBatches batches to optimize. Of that time, $actualWorkTime ms were spent actually working (~${actualWorkTime / totalBatches} ms/batch). The rest was spent sleeping.")
|
||||
return true
|
||||
}
|
||||
|
||||
/**
|
||||
* Drops all tables and recreates them.
|
||||
*/
|
||||
|
||||
@@ -51,7 +51,6 @@ import org.thoughtcrime.securesms.dependencies.AppDependencies
|
||||
import org.thoughtcrime.securesms.groups.BadGroupIdException
|
||||
import org.thoughtcrime.securesms.groups.GroupId
|
||||
import org.thoughtcrime.securesms.jobs.MultiDeviceDeleteSyncJob
|
||||
import org.thoughtcrime.securesms.jobs.OptimizeMessageSearchIndexJob
|
||||
import org.thoughtcrime.securesms.keyvalue.SignalStore
|
||||
import org.thoughtcrime.securesms.mms.SlideDeck
|
||||
import org.thoughtcrime.securesms.mms.StickerSlide
|
||||
@@ -363,7 +362,6 @@ class ThreadTable(context: Context, databaseHelper: SignalDatabase) : DatabaseTa
|
||||
|
||||
notifyAttachmentListeners()
|
||||
notifyStickerPackListeners()
|
||||
OptimizeMessageSearchIndexJob.enqueue()
|
||||
}
|
||||
|
||||
fun trimThread(
|
||||
@@ -397,7 +395,6 @@ class ThreadTable(context: Context, databaseHelper: SignalDatabase) : DatabaseTa
|
||||
|
||||
notifyAttachmentListeners()
|
||||
notifyStickerPackListeners()
|
||||
OptimizeMessageSearchIndexJob.enqueue()
|
||||
}
|
||||
|
||||
private fun trimThreadInternal(
|
||||
@@ -1188,8 +1185,6 @@ class ThreadTable(context: Context, databaseHelper: SignalDatabase) : DatabaseTa
|
||||
AppDependencies.databaseObserver.notifyConversationDeleteListeners(selectedConversations)
|
||||
|
||||
ConversationUtil.clearShortcuts(context, recipientIds)
|
||||
|
||||
OptimizeMessageSearchIndexJob.enqueue()
|
||||
}
|
||||
|
||||
@SuppressLint("DiscouragedApi")
|
||||
|
||||
@@ -97,6 +97,7 @@ import org.thoughtcrime.securesms.database.helpers.migration.V236_FixInAppSubscr
|
||||
import org.thoughtcrime.securesms.database.helpers.migration.V237_ResetGroupForceUpdateTimestamps
|
||||
import org.thoughtcrime.securesms.database.helpers.migration.V238_AddGroupSendEndorsementsColumns
|
||||
import org.thoughtcrime.securesms.database.helpers.migration.V239_MessageFullTextSearchEmojiSupport
|
||||
import org.thoughtcrime.securesms.database.helpers.migration.V240_MessageFullTextSearchSecureDelete
|
||||
|
||||
/**
|
||||
* Contains all of the database migrations for [SignalDatabase]. Broken into a separate file for cleanliness.
|
||||
@@ -196,10 +197,11 @@ object SignalDatabaseMigrations {
|
||||
236 to V236_FixInAppSubscriberCurrencyIfAble,
|
||||
237 to V237_ResetGroupForceUpdateTimestamps,
|
||||
238 to V238_AddGroupSendEndorsementsColumns,
|
||||
239 to V239_MessageFullTextSearchEmojiSupport
|
||||
239 to V239_MessageFullTextSearchEmojiSupport,
|
||||
240 to V240_MessageFullTextSearchSecureDelete
|
||||
)
|
||||
|
||||
const val DATABASE_VERSION = 239
|
||||
const val DATABASE_VERSION = 240
|
||||
|
||||
@JvmStatic
|
||||
fun migrate(context: Application, db: SQLiteDatabase, oldVersion: Int, newVersion: Int) {
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
/*
|
||||
* Copyright 2024 Signal Messenger, LLC
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
package org.thoughtcrime.securesms.database.helpers.migration
|
||||
|
||||
import android.app.Application
|
||||
import net.zetetic.database.sqlcipher.SQLiteDatabase
|
||||
|
||||
/**
|
||||
* Sets the 'secure-delete' flag on the message_fts table.
|
||||
* https://www.sqlite.org/fts5.html#the_secure_delete_configuration_option
|
||||
*/
|
||||
@Suppress("ClassName")
|
||||
object V240_MessageFullTextSearchSecureDelete : SignalDatabaseMigration {
|
||||
|
||||
const val FTS_TABLE_NAME = "message_fts"
|
||||
|
||||
override fun migrate(context: Application, db: SQLiteDatabase, oldVersion: Int, newVersion: Int) {
|
||||
db.execSQL("""INSERT INTO $FTS_TABLE_NAME ($FTS_TABLE_NAME, rank) VALUES('secure-delete', 1);""")
|
||||
}
|
||||
}
|
||||
@@ -61,7 +61,6 @@ import org.thoughtcrime.securesms.migrations.EmojiSearchIndexCheckMigrationJob;
|
||||
import org.thoughtcrime.securesms.migrations.IdentityTableCleanupMigrationJob;
|
||||
import org.thoughtcrime.securesms.migrations.LegacyMigrationJob;
|
||||
import org.thoughtcrime.securesms.migrations.MigrationCompleteJob;
|
||||
import org.thoughtcrime.securesms.migrations.OptimizeMessageSearchIndexMigrationJob;
|
||||
import org.thoughtcrime.securesms.migrations.PassingMigrationJob;
|
||||
import org.thoughtcrime.securesms.migrations.PinOptOutMigration;
|
||||
import org.thoughtcrime.securesms.migrations.PinReminderMigrationJob;
|
||||
@@ -181,7 +180,6 @@ public final class JobManagerFactories {
|
||||
put(MultiDeviceViewOnceOpenJob.KEY, new MultiDeviceViewOnceOpenJob.Factory());
|
||||
put(MultiDeviceViewedUpdateJob.KEY, new MultiDeviceViewedUpdateJob.Factory());
|
||||
put(NullMessageSendJob.KEY, new NullMessageSendJob.Factory());
|
||||
put(OptimizeMessageSearchIndexJob.KEY, new OptimizeMessageSearchIndexJob.Factory());
|
||||
put(PaymentLedgerUpdateJob.KEY, new PaymentLedgerUpdateJob.Factory());
|
||||
put(PaymentNotificationSendJob.KEY, new PaymentNotificationSendJob.Factory());
|
||||
put(PaymentNotificationSendJobV2.KEY, new PaymentNotificationSendJobV2.Factory());
|
||||
@@ -271,7 +269,6 @@ public final class JobManagerFactories {
|
||||
put(IdentityTableCleanupMigrationJob.KEY, new IdentityTableCleanupMigrationJob.Factory());
|
||||
put(LegacyMigrationJob.KEY, new LegacyMigrationJob.Factory());
|
||||
put(MigrationCompleteJob.KEY, new MigrationCompleteJob.Factory());
|
||||
put(OptimizeMessageSearchIndexMigrationJob.KEY,new OptimizeMessageSearchIndexMigrationJob.Factory());
|
||||
put(PinOptOutMigration.KEY, new PinOptOutMigration.Factory());
|
||||
put(PinReminderMigrationJob.KEY, new PinReminderMigrationJob.Factory());
|
||||
put(PniAccountInitializationMigrationJob.KEY, new PniAccountInitializationMigrationJob.Factory());
|
||||
@@ -342,6 +339,8 @@ public final class JobManagerFactories {
|
||||
put("SmsSentJob", new FailingJob.Factory());
|
||||
put("MmsSendJobV2", new FailingJob.Factory());
|
||||
put("AttachmentUploadJobV2", new FailingJob.Factory());
|
||||
put("OptimizeMessageSearchIndexJob", new FailingJob.Factory());
|
||||
put("OptimizeMessageSearchIndexMigrationJob", new PassingMigrationJob.Factory());
|
||||
}};
|
||||
}
|
||||
|
||||
|
||||
@@ -1,58 +0,0 @@
|
||||
package org.thoughtcrime.securesms.jobs
|
||||
|
||||
import org.signal.core.util.logging.Log
|
||||
import org.thoughtcrime.securesms.database.SignalDatabase
|
||||
import org.thoughtcrime.securesms.dependencies.AppDependencies
|
||||
import org.thoughtcrime.securesms.jobmanager.Job
|
||||
import org.thoughtcrime.securesms.keyvalue.SignalStore
|
||||
import org.thoughtcrime.securesms.transport.RetryLaterException
|
||||
import java.lang.Exception
|
||||
import kotlin.time.Duration.Companion.seconds
|
||||
|
||||
/**
|
||||
* Optimizes the message search index incrementally.
|
||||
*/
|
||||
class OptimizeMessageSearchIndexJob private constructor(parameters: Parameters) : BaseJob(parameters) {
|
||||
|
||||
companion object {
|
||||
const val KEY = "OptimizeMessageSearchIndexJob"
|
||||
|
||||
private val TAG = Log.tag(OptimizeMessageSearchIndexJob::class.java)
|
||||
|
||||
@JvmStatic
|
||||
fun enqueue() {
|
||||
AppDependencies.jobManager.add(OptimizeMessageSearchIndexJob())
|
||||
}
|
||||
}
|
||||
|
||||
constructor() : this(
|
||||
Parameters.Builder()
|
||||
.setQueue("OptimizeMessageSearchIndexJob")
|
||||
.setMaxAttempts(5)
|
||||
.setMaxInstancesForQueue(2)
|
||||
.build()
|
||||
)
|
||||
|
||||
override fun serialize(): ByteArray? = null
|
||||
override fun getFactoryKey() = KEY
|
||||
override fun onFailure() = Unit
|
||||
override fun onShouldRetry(e: Exception) = e is RetryLaterException
|
||||
override fun getNextRunAttemptBackoff(pastAttemptCount: Int, exception: Exception): Long = 30.seconds.inWholeMilliseconds
|
||||
|
||||
override fun onRun() {
|
||||
if (!SignalStore.registration.isRegistrationComplete || SignalStore.account.aci == null) {
|
||||
Log.w(TAG, "Registration not finished yet! Skipping.")
|
||||
return
|
||||
}
|
||||
|
||||
val success = SignalDatabase.messageSearch.optimizeIndex(5.seconds.inWholeMilliseconds)
|
||||
|
||||
if (!success) {
|
||||
throw RetryLaterException()
|
||||
}
|
||||
}
|
||||
|
||||
class Factory : Job.Factory<OptimizeMessageSearchIndexJob> {
|
||||
override fun create(parameters: Parameters, serializedData: ByteArray?) = OptimizeMessageSearchIndexJob(parameters)
|
||||
}
|
||||
}
|
||||
@@ -116,7 +116,7 @@ public class ApplicationMigrations {
|
||||
static final int SMS_MMS_MERGE = 71;
|
||||
static final int REBUILD_MESSAGE_FTS_INDEX = 72;
|
||||
static final int UPDATE_SMS_JOBS = 73;
|
||||
static final int OPTIMIZE_MESSAGE_FTS_INDEX = 74;
|
||||
// static final int OPTIMIZE_MESSAGE_FTS_INDEX = 74;
|
||||
static final int REACTION_DATABASE_MIGRATION = 75;
|
||||
static final int REBUILD_MESSAGE_FTS_INDEX_2 = 76;
|
||||
static final int GLIDE_CACHE_CLEAR = 77;
|
||||
@@ -547,9 +547,9 @@ public class ApplicationMigrations {
|
||||
jobs.put(Version.UPDATE_SMS_JOBS, new UpdateSmsJobsMigrationJob());
|
||||
}
|
||||
|
||||
if (lastSeenVersion < Version.OPTIMIZE_MESSAGE_FTS_INDEX) {
|
||||
jobs.put(Version.OPTIMIZE_MESSAGE_FTS_INDEX, new OptimizeMessageSearchIndexMigrationJob());
|
||||
}
|
||||
// if (lastSeenVersion < Version.OPTIMIZE_MESSAGE_FTS_INDEX) {
|
||||
// jobs.put(Version.OPTIMIZE_MESSAGE_FTS_INDEX, new OptimizeMessageSearchIndexMigrationJob());
|
||||
// }
|
||||
|
||||
if (lastSeenVersion < Version.REACTION_DATABASE_MIGRATION) {
|
||||
jobs.put(Version.REACTION_DATABASE_MIGRATION, new DatabaseMigrationJob());
|
||||
|
||||
@@ -1,34 +0,0 @@
|
||||
package org.thoughtcrime.securesms.migrations
|
||||
|
||||
import org.signal.core.util.logging.Log
|
||||
import org.thoughtcrime.securesms.jobmanager.Job
|
||||
import org.thoughtcrime.securesms.jobs.OptimizeMessageSearchIndexJob
|
||||
|
||||
/**
|
||||
* Kicks off a job to optimize the message search index.
|
||||
*/
|
||||
internal class OptimizeMessageSearchIndexMigrationJob(
|
||||
parameters: Parameters = Parameters.Builder().build()
|
||||
) : MigrationJob(parameters) {
|
||||
|
||||
companion object {
|
||||
val TAG = Log.tag(OptimizeMessageSearchIndexMigrationJob::class.java)
|
||||
const val KEY = "OptimizeMessageSearchIndexMigrationJob"
|
||||
}
|
||||
|
||||
override fun getFactoryKey(): String = KEY
|
||||
|
||||
override fun isUiBlocking(): Boolean = false
|
||||
|
||||
override fun performMigration() {
|
||||
OptimizeMessageSearchIndexJob.enqueue()
|
||||
}
|
||||
|
||||
override fun shouldRetry(e: Exception): Boolean = false
|
||||
|
||||
class Factory : Job.Factory<OptimizeMessageSearchIndexMigrationJob> {
|
||||
override fun create(parameters: Parameters, serializedData: ByteArray?): OptimizeMessageSearchIndexMigrationJob {
|
||||
return OptimizeMessageSearchIndexMigrationJob(parameters)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user