From 8d931391db919496445bc9caac33f36c155b07c3 Mon Sep 17 00:00:00 2001 From: Greyson Parrelli Date: Mon, 27 Oct 2025 12:24:59 -0400 Subject: [PATCH] Improve backup import resilience for duplicate messages. --- .../v2/importer/ChatItemArchiveImporter.kt | 31 ++++++++++++++----- .../main/java/org/signal/core/util/SqlUtil.kt | 4 +-- .../org/signal/core/util/SqlUtilTest.java | 8 ++--- 3 files changed, 30 insertions(+), 13 deletions(-) diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/importer/ChatItemArchiveImporter.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/importer/ChatItemArchiveImporter.kt index 455d67b9c2..6abfb6deca 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/importer/ChatItemArchiveImporter.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/importer/ChatItemArchiveImporter.kt @@ -84,6 +84,7 @@ import org.whispersystems.signalservice.api.push.ServiceId import org.whispersystems.signalservice.api.util.UuidUtil import org.whispersystems.signalservice.internal.push.DataMessage import java.math.BigInteger +import java.sql.SQLException import java.util.Optional import java.util.UUID import org.thoughtcrime.securesms.backup.v2.proto.GiftBadge as BackupGiftBadge @@ -226,12 +227,17 @@ class ChatItemArchiveImporter( } var messageInsertIndex = 0 - SqlUtil.buildBulkInsert(MessageTable.TABLE_NAME, MESSAGE_COLUMNS, buffer.messages.map { it.contentValues }).forEach { query -> - db.rawQuery("${query.where} RETURNING ${MessageTable.ID}", query.whereArgs).forEach { cursor -> - val finalMessageId = cursor.requireLong(MessageTable.ID) - val relatedInsert = buffer.messages[messageInsertIndex++] - relatedInsert.followUp?.invoke(finalMessageId) + try { + SqlUtil.buildBulkInsert(MessageTable.TABLE_NAME, MESSAGE_COLUMNS, buffer.messages.map { it.contentValues }, onConflict = "IGNORE").forEach { query -> + db.rawQuery("${query.where} RETURNING ${MessageTable.ID}", query.whereArgs).forEach { cursor -> + val finalMessageId = cursor.requireLong(MessageTable.ID) + val relatedInsert = buffer.messages[messageInsertIndex++] + relatedInsert.followUp?.invoke(finalMessageId) + } } + } catch (e: SQLException) { + Log.w(TAG, "Failed to bulk-insert message! Trying one at at time.", e) + performIndividualMessageInserts(buffer.messages) } SqlUtil.buildBulkInsert(ReactionTable.TABLE_NAME, REACTION_COLUMNS, buffer.reactions).forEach { @@ -249,6 +255,18 @@ class ChatItemArchiveImporter( return true } + private fun performIndividualMessageInserts(messageInserts: List) { + for (message in messageInserts) { + val values = message.contentValues + try { + db.insert(MessageTable.TABLE_NAME, SQLiteDatabase.CONFLICT_IGNORE, values) + message.followUp?.invoke(messageId - 1) + } catch (e: SQLException) { + Log.w(TAG, "Failed to insert message with timestamp ${message.contentValues.get(MessageTable.DATE_SENT)}. Must skip.", e) + } + } + } + private fun ChatItem.toMessageInsert(fromRecipientId: RecipientId, chatRecipientId: RecipientId, threadId: Long): MessageInsert { val contentValues = this.toMessageContentValues(fromRecipientId, chatRecipientId, threadId) @@ -1260,8 +1278,7 @@ class ChatItemArchiveImporter( private class MessageInsert( val contentValues: ContentValues, - val followUp: ((Long) -> Unit)?, - val edits: List? = null + val followUp: ((Long) -> Unit)? ) private class Buffer( diff --git a/core-util/src/main/java/org/signal/core/util/SqlUtil.kt b/core-util/src/main/java/org/signal/core/util/SqlUtil.kt index 781b0bb9b9..d0c5676b64 100644 --- a/core-util/src/main/java/org/signal/core/util/SqlUtil.kt +++ b/core-util/src/main/java/org/signal/core/util/SqlUtil.kt @@ -403,13 +403,13 @@ object SqlUtil { } @JvmStatic - fun buildBulkInsert(tableName: String, columns: Array, contentValues: List): List { + fun buildBulkInsert(tableName: String, columns: Array, contentValues: List, onConflict: String? = null): List { return buildBulkInsert(tableName, columns, contentValues, MAX_QUERY_ARGS) } @JvmStatic @VisibleForTesting - fun buildBulkInsert(tableName: String, columns: Array, contentValues: List, maxQueryArgs: Int): List { + fun buildBulkInsert(tableName: String, columns: Array, contentValues: List, maxQueryArgs: Int, onConflict: String? = null): List { val batchSize = maxQueryArgs / columns.size return contentValues diff --git a/core-util/src/test/java/org/signal/core/util/SqlUtilTest.java b/core-util/src/test/java/org/signal/core/util/SqlUtilTest.java index b358ff8350..a3bd84b72b 100644 --- a/core-util/src/test/java/org/signal/core/util/SqlUtilTest.java +++ b/core-util/src/test/java/org/signal/core/util/SqlUtilTest.java @@ -238,7 +238,7 @@ public final class SqlUtilTest { contentValues.add(cv1); - List output = SqlUtil.buildBulkInsert("mytable", new String[] { "a", "b"}, contentValues); + List output = SqlUtil.buildBulkInsert("mytable", new String[] { "a", "b"}, contentValues, null); assertEquals(1, output.size()); assertEquals("INSERT INTO mytable (a, b) VALUES (?, ?)", output.get(0).getWhere()); @@ -256,7 +256,7 @@ public final class SqlUtilTest { contentValues.add(cv1); - List output = SqlUtil.buildBulkInsert("mytable", new String[] { "a", "b", "c"}, contentValues); + List output = SqlUtil.buildBulkInsert("mytable", new String[] { "a", "b", "c"}, contentValues, null); assertEquals(1, output.size()); assertEquals("INSERT INTO mytable (a, b, c) VALUES (?, ?, null)", output.get(0).getWhere()); @@ -278,7 +278,7 @@ public final class SqlUtilTest { contentValues.add(cv1); contentValues.add(cv2); - List output = SqlUtil.buildBulkInsert("mytable", new String[] { "a", "b"}, contentValues); + List output = SqlUtil.buildBulkInsert("mytable", new String[] { "a", "b"}, contentValues, null); assertEquals(1, output.size()); assertEquals("INSERT INTO mytable (a, b) VALUES (?, ?), (?, ?)", output.get(0).getWhere()); @@ -305,7 +305,7 @@ public final class SqlUtilTest { contentValues.add(cv2); contentValues.add(cv3); - List output = SqlUtil.buildBulkInsert("mytable", new String[] { "a", "b"}, contentValues, 4); + List output = SqlUtil.buildBulkInsert("mytable", new String[] { "a", "b"}, contentValues, 4, null); assertEquals(2, output.size());