diff --git a/app/build.gradle.kts b/app/build.gradle.kts index 571271b65d..110dcffe04 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -51,6 +51,7 @@ val localProperties: Properties? = if (localPropertiesFile.exists()) { null } val quickstartCredentialsDir: String? = localProperties?.getProperty("quickstart.credentials.dir") +val benchmarkBackupFile: String? = localProperties?.getProperty("benchmark.backup.file") val selectableVariants = listOf( "nightlyProdSpinner", @@ -531,6 +532,15 @@ android { } variant.sources.assets?.addGeneratedSourceDirectory(taskProvider) { it.outputDir } } + + onVariants(selector().withBuildType("benchmark")) { variant -> + val taskProvider = tasks.register("copyBenchmarkBackup${variant.name.capitalize()}") { + if (benchmarkBackupFile != null) { + inputFile.set(File(benchmarkBackupFile)) + } + } + variant.sources.assets?.addGeneratedSourceDirectory(taskProvider) { it.outputDir } + } } val releaseDir = "$projectDir/src/release/java" @@ -902,3 +912,27 @@ abstract class CopyQuickstartCredentialsTask : DefaultTask() { chosen.copyTo(dest.resolve(chosen.name), overwrite = true) } } + +abstract class CopyBenchmarkBackupTask : DefaultTask() { + @get:InputFile + @get:Optional + abstract val inputFile: RegularFileProperty + + @get:OutputDirectory + abstract val outputDir: DirectoryProperty + + @TaskAction + fun copy() { + val dest = outputDir.get().asFile.resolve("backups") + dest.mkdirs() + + if (!inputFile.isPresent) { + logger.lifecycle("benchmark.backup.file is not set in local.properties. Benchmark tests using backup data will crash at runtime.") + return + } + + val backupFile = inputFile.get().asFile + logger.lifecycle("Using benchmark backup: ${backupFile.absolutePath} (${backupFile.length() / 1024}KB)") + backupFile.copyTo(dest.resolve("backup.binproto"), overwrite = true) + } +} diff --git a/app/src/benchmark/java/org/thoughtcrime/securesms/BenchmarkApplicationContext.kt b/app/src/benchmark/java/org/thoughtcrime/securesms/BenchmarkApplicationContext.kt index fe7eb85ce9..b47c6aa32d 100644 --- a/app/src/benchmark/java/org/thoughtcrime/securesms/BenchmarkApplicationContext.kt +++ b/app/src/benchmark/java/org/thoughtcrime/securesms/BenchmarkApplicationContext.kt @@ -18,6 +18,7 @@ import org.thoughtcrime.securesms.jobs.AccountConsistencyWorkerJob import org.thoughtcrime.securesms.jobs.ArchiveBackupIdReservationJob import org.thoughtcrime.securesms.jobs.AttachmentCompressionJob import org.thoughtcrime.securesms.jobs.AttachmentUploadJob +import org.thoughtcrime.securesms.jobs.AvatarGroupsV2DownloadJob import org.thoughtcrime.securesms.jobs.CreateReleaseChannelJob import org.thoughtcrime.securesms.jobs.DirectoryRefreshJob import org.thoughtcrime.securesms.jobs.DownloadLatestEmojiDataJob @@ -39,6 +40,12 @@ import org.thoughtcrime.securesms.jobs.PushGroupSendJob import org.thoughtcrime.securesms.jobs.PushProcessMessageJob import org.thoughtcrime.securesms.jobs.ReactionSendJob import org.thoughtcrime.securesms.jobs.RefreshAttributesJob +import org.thoughtcrime.securesms.jobs.RefreshSvrCredentialsJob +import org.thoughtcrime.securesms.jobs.RequestGroupV2InfoJob +import org.thoughtcrime.securesms.jobs.ResetSvrGuessCountJob +import org.thoughtcrime.securesms.jobs.RestoreOptimizedMediaJob +import org.thoughtcrime.securesms.jobs.RetrieveProfileAvatarJob +import org.thoughtcrime.securesms.jobs.RetrieveProfileJob import org.thoughtcrime.securesms.jobs.RetrieveRemoteAnnouncementsJob import org.thoughtcrime.securesms.jobs.RotateCertificateJob import org.thoughtcrime.securesms.jobs.SendDeliveryReceiptJob @@ -117,6 +124,7 @@ class BenchmarkApplicationContext : ApplicationContext() { val blockedJobs = setOf( AccountConsistencyWorkerJob.KEY, ArchiveBackupIdReservationJob.KEY, + AvatarGroupsV2DownloadJob.KEY, CreateReleaseChannelJob.KEY, DirectoryRefreshJob.KEY, DownloadLatestEmojiDataJob.KEY, @@ -130,6 +138,12 @@ class BenchmarkApplicationContext : ApplicationContext() { PreKeysSyncJob.KEY, ProfileUploadJob.KEY, RefreshAttributesJob.KEY, + RefreshSvrCredentialsJob.KEY, + RequestGroupV2InfoJob.KEY, + ResetSvrGuessCountJob.KEY, + RestoreOptimizedMediaJob.KEY, + RetrieveProfileAvatarJob.KEY, + RetrieveProfileJob.KEY, RetrieveRemoteAnnouncementsJob.KEY, RotateCertificateJob.KEY, StickerPackDownloadJob.KEY, diff --git a/app/src/benchmarkShared/java/org/signal/benchmark/BenchmarkCommandReceiver.kt b/app/src/benchmarkShared/java/org/signal/benchmark/BenchmarkCommandReceiver.kt index 18735bd7e4..5e27109074 100644 --- a/app/src/benchmarkShared/java/org/signal/benchmark/BenchmarkCommandReceiver.kt +++ b/app/src/benchmarkShared/java/org/signal/benchmark/BenchmarkCommandReceiver.kt @@ -15,6 +15,7 @@ import org.signal.benchmark.setup.Generator import org.signal.benchmark.setup.Harness import org.signal.benchmark.setup.OtherClient import org.signal.core.util.ThreadUtil +import org.thoughtcrime.securesms.dependencies.AppDependencies import org.signal.core.util.logging.Log import org.thoughtcrime.securesms.database.SignalDatabase import org.thoughtcrime.securesms.database.TestDbUtils @@ -152,17 +153,22 @@ class BenchmarkCommandReceiver : BroadcastReceiver() { } private fun handleDeleteThread() { - val threadId = SignalDatabase.threads.getRecentConversationList(1, false, false).use { cursor -> + val threadId = SignalDatabase.rawDatabase.rawQuery("SELECT thread_id, COUNT(*) AS msg_count FROM message GROUP BY thread_id ORDER BY msg_count DESC LIMIT 1").use { cursor -> if (cursor.moveToFirst()) { - cursor.getLong(cursor.getColumnIndexOrThrow("_id")) + val id = cursor.getLong(0) + val count = cursor.getLong(1) + Log.i(TAG, "Found largest thread $id with $count messages") + id } else { - Log.w(TAG, "No active threads found for deletion benchmark") + Log.w(TAG, "No threads found for deletion benchmark") return } } - Log.i(TAG, "Deleting thread $threadId") + val recipientName = SignalDatabase.threads.getRecipientForThreadId(threadId) + ?.getDisplayName(AppDependencies.application) ?: "unknown" + Log.i(TAG, "Deleting thread $threadId (recipient: $recipientName)") SignalDatabase.threads.deleteConversation(threadId, syncThreadDelete = false) - Log.i(TAG, "Thread $threadId deleted") + Log.i(TAG, "Thread $threadId deleted (recipient: $recipientName)") } private fun getOutgoingGroupMessageTimestamps(): List { diff --git a/app/src/benchmarkShared/java/org/signal/benchmark/BenchmarkSetupActivity.kt b/app/src/benchmarkShared/java/org/signal/benchmark/BenchmarkSetupActivity.kt index d762b8195c..73afa423eb 100644 --- a/app/src/benchmarkShared/java/org/signal/benchmark/BenchmarkSetupActivity.kt +++ b/app/src/benchmarkShared/java/org/signal/benchmark/BenchmarkSetupActivity.kt @@ -10,16 +10,31 @@ import androidx.compose.runtime.setValue import androidx.lifecycle.lifecycleScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.launch +import org.signal.benchmark.setup.Harness import org.signal.benchmark.setup.TestMessages import org.signal.benchmark.setup.TestUsers +import org.signal.core.models.ServiceId.PNI +import org.signal.core.util.logging.Log import org.thoughtcrime.securesms.BaseActivity +import org.thoughtcrime.securesms.backup.v2.BackupRepository +import org.thoughtcrime.securesms.crypto.ProfileKeyUtil import org.thoughtcrime.securesms.database.SignalDatabase import org.thoughtcrime.securesms.database.TestDbUtils +import org.thoughtcrime.securesms.database.model.databaseprotos.RestoreDecisionState +import org.thoughtcrime.securesms.keyvalue.SignalStore +import org.thoughtcrime.securesms.keyvalue.Skipped import org.thoughtcrime.securesms.mms.OutgoingMessage +import org.thoughtcrime.securesms.profiles.ProfileName import org.thoughtcrime.securesms.recipients.Recipient +import org.thoughtcrime.securesms.registration.util.RegistrationUtil import org.thoughtcrime.securesms.util.TextSecurePreferences class BenchmarkSetupActivity : BaseActivity() { + + companion object { + private val TAG = Log.tag(BenchmarkSetupActivity::class) + } + override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) @@ -43,6 +58,7 @@ class BenchmarkSetupActivity : BaseActivity() { "group-read-receipt" -> setupGroupReceipt(enableReadReceipts = true) "thread-delete" -> setupThreadDelete() "thread-delete-group" -> setupThreadDeleteGroup() + "backup-restore" -> setupBackupRestore() } setupComplete = true } @@ -151,6 +167,35 @@ class BenchmarkSetupActivity : BaseActivity() { SignalDatabase.threads.update(threadId, true) } + private fun setupBackupRestore() { + TestUsers.setupSelf() + + val profileKey = ProfileKeyUtil.getSelfProfileKey() + val selfData = BackupRepository.SelfData( + aci = Harness.SELF_ACI, + pni = SignalStore.account.requirePni(), + e164 = Harness.SELF_E164, + profileKey = profileKey + ) + + val backupBytes = assets.open("backups/backup.binproto").use { it.readBytes() } + Log.i(TAG, "Read ${backupBytes.size} bytes from backup asset") + + val result = BackupRepository.importPlaintextTest( + length = backupBytes.size.toLong(), + inputStreamFactory = { backupBytes.inputStream() }, + selfData = selfData + ) + + Log.i(TAG, "Backup import result: $result") + + SignalStore.svr.optOut() + SignalStore.registration.restoreDecisionState = RestoreDecisionState.Skipped + SignalDatabase.recipients.setProfileKey(Recipient.self().id, profileKey) + SignalDatabase.recipients.setProfileName(Recipient.self().id, ProfileName.fromParts("Tester", "McTesterson")) + RegistrationUtil.maybeMarkRegistrationComplete() + } + private fun setupGroupReceipt(includeMsl: Boolean = false, enableReadReceipts: Boolean = false) { TestUsers.setupSelf() val groupId = TestUsers.setupGroup() diff --git a/app/src/main/java/org/thoughtcrime/securesms/database/MessageTable.kt b/app/src/main/java/org/thoughtcrime/securesms/database/MessageTable.kt index 3a148e91f8..d232937d0a 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/database/MessageTable.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/database/MessageTable.kt @@ -43,6 +43,7 @@ import org.signal.core.util.delete import org.signal.core.util.deleteAll import org.signal.core.util.exists import org.signal.core.util.forEach +import org.signal.core.util.forceForeignKeyConstraintsEnabled import org.signal.core.util.insertInto import org.signal.core.util.logging.Log import org.signal.core.util.readToList @@ -4086,38 +4087,54 @@ open class MessageTable(context: Context?, databaseHelper: SignalDatabase) : Dat SignalTrace.beginSection("MessageTable#deleteMessagesInThread") for (threadId in threadsWithPossibleDeletes) { - val subSelect = "SELECT ${TABLE_NAME}.$ID FROM $TABLE_NAME WHERE ${TABLE_NAME}.$THREAD_ID = $threadId $extraWhere LIMIT $DELETE_BATCH_SIZE" + val batchTable = "tmp_delete_batch" + val batchSelect = "SELECT $ID FROM $batchTable" var deletedCount: Int do { deletedCount = writableDatabase.withinTransaction { db -> + db.execSQL("CREATE TEMP TABLE IF NOT EXISTS $batchTable ($ID INTEGER PRIMARY KEY)") + db.execSQL("DELETE FROM $batchTable") + db.execSQL("INSERT INTO $batchTable SELECT ${TABLE_NAME}.$ID FROM $TABLE_NAME WHERE ${TABLE_NAME}.$THREAD_ID = $threadId $extraWhere LIMIT $DELETE_BATCH_SIZE") + // Expand to include revision chain members so they're always deleted together + db.execSQL("INSERT OR IGNORE INTO $batchTable SELECT $ID FROM $TABLE_NAME WHERE $LATEST_REVISION_ID IN (SELECT $ID FROM $batchTable) OR $ORIGINAL_MESSAGE_ID IN (SELECT $ID FROM $batchTable)") + db.delete(StorySendTable.TABLE_NAME) - .where("${StorySendTable.TABLE_NAME}.${StorySendTable.MESSAGE_ID} IN ($subSelect)") + .where("${StorySendTable.TABLE_NAME}.${StorySendTable.MESSAGE_ID} IN ($batchSelect)") .run() db.delete(ReactionTable.TABLE_NAME) - .where("${ReactionTable.TABLE_NAME}.${ReactionTable.MESSAGE_ID} IN ($subSelect)") + .where("${ReactionTable.TABLE_NAME}.${ReactionTable.MESSAGE_ID} IN ($batchSelect)") .run() db.delete(CallTable.TABLE_NAME) - .where("${CallTable.TABLE_NAME}.${CallTable.MESSAGE_ID} IN ($subSelect)") + .where("${CallTable.TABLE_NAME}.${CallTable.MESSAGE_ID} IN ($batchSelect)") .run() db.delete(AttachmentTable.TABLE_NAME) - .where("${AttachmentTable.TABLE_NAME}.${AttachmentTable.MESSAGE_ID} IN ($subSelect)") + .where("${AttachmentTable.TABLE_NAME}.${AttachmentTable.MESSAGE_ID} IN ($batchSelect)") .run() db.delete(GroupReceiptTable.TABLE_NAME) - .where("${GroupReceiptTable.TABLE_NAME}.${GroupReceiptTable.MMS_ID} IN ($subSelect)") + .where("${GroupReceiptTable.TABLE_NAME}.${GroupReceiptTable.MMS_ID} IN ($batchSelect)") .run() db.delete(MentionTable.TABLE_NAME) - .where("${MentionTable.TABLE_NAME}.${MentionTable.MESSAGE_ID} IN ($subSelect)") + .where("${MentionTable.TABLE_NAME}.${MentionTable.MESSAGE_ID} IN ($batchSelect)") .run() - // Delete the messages themselves - db.delete(TABLE_NAME) - .where("$ID IN ($subSelect)") + // Null self-referential FK links so DELETE doesn't trigger CASCADE checks + db.update(TABLE_NAME) + .values(LATEST_REVISION_ID to null, ORIGINAL_MESSAGE_ID to null) + .where("$LATEST_REVISION_ID IN ($batchSelect) OR $ORIGINAL_MESSAGE_ID IN ($batchSelect)") .run() + + SignalTrace.beginSection("deleteMessages") + val count = db.delete(TABLE_NAME) + .where("$ID IN ($batchSelect)") + .run() + SignalTrace.endSection() + + count } totalDeletedCount += deletedCount diff --git a/benchmark/src/main/java/org/thoughtcrime/benchmark/BenchmarkMetrics.kt b/benchmark/src/main/java/org/thoughtcrime/benchmark/BenchmarkMetrics.kt index 66e0b84eaf..4574996e08 100644 --- a/benchmark/src/main/java/org/thoughtcrime/benchmark/BenchmarkMetrics.kt +++ b/benchmark/src/main/java/org/thoughtcrime/benchmark/BenchmarkMetrics.kt @@ -51,6 +51,7 @@ object BenchmarkMetrics { val threadDeletion: List get() = listOf( TraceSectionMetric("ThreadTable#deleteConversations", Mode.Sum), - TraceSectionMetric("MessageTable#deleteMessagesInThread", Mode.Sum) + TraceSectionMetric("MessageTable#deleteMessagesInThread", Mode.Sum), + TraceSectionMetric("deleteMessages", Mode.Sum) ) } diff --git a/benchmark/src/main/java/org/thoughtcrime/benchmark/BenchmarkSetup.kt b/benchmark/src/main/java/org/thoughtcrime/benchmark/BenchmarkSetup.kt index 37b9ae60b9..c77fd93063 100644 --- a/benchmark/src/main/java/org/thoughtcrime/benchmark/BenchmarkSetup.kt +++ b/benchmark/src/main/java/org/thoughtcrime/benchmark/BenchmarkSetup.kt @@ -8,8 +8,11 @@ object BenchmarkSetup { private const val TARGET_PACKAGE = "org.thoughtcrime.securesms.benchmark" private const val RECEIVER = "org.signal.benchmark.BenchmarkCommandReceiver" - fun setup(type: String, device: UiDevice, timeout: Long = 25_000L) { - device.executeShellCommand("pm clear $TARGET_PACKAGE") + fun setup(type: String, device: UiDevice, timeout: Long = 25_000L, clearData: Boolean = true) { + if (clearData) { + device.executeShellCommand("pm clear $TARGET_PACKAGE") + } + device.executeShellCommand("am start -W -n $TARGET_PACKAGE/org.signal.benchmark.BenchmarkSetupActivity --es setup-type $type") device.wait(Until.hasObject(By.textContains("done")), timeout) } diff --git a/benchmark/src/main/java/org/thoughtcrime/benchmark/ThreadDeletionBenchmarks.kt b/benchmark/src/main/java/org/thoughtcrime/benchmark/ThreadDeletionBenchmarks.kt index 53d29977f1..57532e1df1 100644 --- a/benchmark/src/main/java/org/thoughtcrime/benchmark/ThreadDeletionBenchmarks.kt +++ b/benchmark/src/main/java/org/thoughtcrime/benchmark/ThreadDeletionBenchmarks.kt @@ -12,6 +12,7 @@ import androidx.benchmark.macro.junit4.MacrobenchmarkRule import androidx.test.ext.junit.runners.AndroidJUnit4 import androidx.test.uiautomator.By import androidx.test.uiautomator.Until +import org.junit.Ignore import org.junit.Rule import org.junit.Test import org.junit.runner.RunWith @@ -73,4 +74,26 @@ class ThreadDeletionBenchmarks { device.wait(Until.gone(By.textContains("Title")), 300_000L) } } + + @Ignore("Needs locally provided backup file not available in CI yet") + @Test + fun deleteGroupThread20kMessagesWithBackupRestore() { + benchmarkRule.measureRepeated( + packageName = "org.thoughtcrime.securesms.benchmark", + metrics = BenchmarkMetrics.threadDeletion, + iterations = 1, + compilationMode = CompilationMode.Partial(), + setupBlock = { + BenchmarkSetup.setup("backup-restore", device, timeout = 60_000L) + killProcess() + + startActivityAndWait() + device.waitForIdle() + device.wait(Until.findObject(By.textContains("CuQ75j")), 10_000) + } + ) { + BenchmarkSetup.deleteThread(device) + device.wait(Until.gone(By.textContains("CuQ75j")), 300_000L) + } + } }