Add import/export tests for backup of recipients and threads.

2026-09-28 07:51:42 +01:00 · 2024-03-06 20:42:01 -04:00
parent 5740b768d0
commit 32fe927bfc
6 changed files with 421 additions and 30 deletions
@@ -15,21 +15,32 @@ import org.thoughtcrime.securesms.backup.v2.proto.BackupInfo
 import org.thoughtcrime.securesms.backup.v2.proto.Call
 import org.thoughtcrime.securesms.backup.v2.proto.Chat
 import org.thoughtcrime.securesms.backup.v2.proto.ChatItem
+import org.thoughtcrime.securesms.backup.v2.proto.Contact
+import org.thoughtcrime.securesms.backup.v2.proto.DistributionList
 import org.thoughtcrime.securesms.backup.v2.proto.Frame
+import org.thoughtcrime.securesms.backup.v2.proto.Group
 import org.thoughtcrime.securesms.backup.v2.proto.Recipient
+import org.thoughtcrime.securesms.backup.v2.proto.ReleaseNotes
 import org.thoughtcrime.securesms.backup.v2.proto.Self
 import org.thoughtcrime.securesms.backup.v2.proto.StickerPack
 import org.thoughtcrime.securesms.backup.v2.stream.EncryptedBackupReader
 import org.thoughtcrime.securesms.backup.v2.stream.EncryptedBackupWriter
 import org.thoughtcrime.securesms.keyvalue.SignalStore
+import org.whispersystems.signalservice.api.push.DistributionId
 import org.whispersystems.signalservice.api.push.ServiceId
 import org.whispersystems.signalservice.api.subscriptions.SubscriberId
+import org.whispersystems.signalservice.api.util.toByteArray
 import java.io.ByteArrayInputStream
 import java.io.ByteArrayOutputStream
 import java.util.ArrayList
 import java.util.UUID
 import kotlin.random.Random
+import kotlin.time.Duration.Companion.days

+/**
+ * Test the import and export of message backup frames to make sure what
+ * goes in, comes out.
+ */
 class ImportExportTest {
  companion object {
    val SELF_ACI = ServiceId.ACI.from(UUID.fromString("77770000-b477-4f35-a824-d92987a63641"))
@@ -39,6 +50,7 @@ class ImportExportTest {

    val defaultBackupInfo = BackupInfo(version = 1L, backupTimeMs = 123456L)
    val selfRecipient = Recipient(id = 1, self = Self())
+    val releaseNotes = Recipient(id = 2, releaseNotes = ReleaseNotes())
    val standardAccountData = AccountData(
      profileKey = SELF_PROFILE_KEY.serialize().toByteString(),
      username = "testusername",
@@ -69,6 +81,11 @@ class ImportExportTest {
        preferredReactionEmoji = listOf("a", "b", "c")
      )
    )
+
+    /**
+     * When using standardFrames you must start recipient ids at 3.
+     */
+    private val standardFrames = arrayOf(defaultBackupInfo, standardAccountData, selfRecipient, releaseNotes)
  }

  @Before
@@ -82,13 +99,290 @@ class ImportExportTest {

  @Test
  fun accountAndSelf() {
+    importExport(*standardFrames)
+  }
+
+  @Test
+  fun individualRecipients() {
    importExport(
-      defaultBackupInfo,
-      standardAccountData,
-      selfRecipient
+      *standardFrames,
+      Recipient(
+        id = 3,
+        contact = Contact(
+          aci = TestRecipientUtils.nextAci().toByteString(),
+          pni = TestRecipientUtils.nextPni().toByteString(),
+          username = "coolusername",
+          e164 = 141255501234,
+          blocked = true,
+          hidden = true,
+          registered = Contact.Registered.REGISTERED,
+          unregisteredTimestamp = 0L,
+          profileKey = TestRecipientUtils.generateProfileKey().toByteString(),
+          profileSharing = true,
+          profileGivenName = "Alexa",
+          profileFamilyName = "Kim",
+          hideStory = true
+        )
+      ),
+      Recipient(
+        id = 4,
+        contact = Contact(
+          aci = null,
+          pni = null,
+          username = null,
+          e164 = 141255501235,
+          blocked = true,
+          hidden = true,
+          registered = Contact.Registered.NOT_REGISTERED,
+          unregisteredTimestamp = 1234568927398L,
+          profileKey = TestRecipientUtils.generateProfileKey().toByteString(),
+          profileSharing = false,
+          profileGivenName = "Peter",
+          profileFamilyName = "Kim",
+          hideStory = true
+        )
+      )
    )
  }

+  @Test
+  fun groupRecipients() {
+    importExport(
+      *standardFrames,
+      Recipient(
+        id = 3,
+        group = Group(
+          masterKey = TestRecipientUtils.generateGroupMasterKey().toByteString(),
+          whitelisted = true,
+          hideStory = true,
+          storySendMode = Group.StorySendMode.ENABLED,
+          name = "Cool test group"
+        )
+      ),
+      Recipient(
+        id = 4,
+        group = Group(
+          masterKey = TestRecipientUtils.generateGroupMasterKey().toByteString(),
+          whitelisted = false,
+          hideStory = false,
+          storySendMode = Group.StorySendMode.DEFAULT,
+          name = "Cool test group"
+        )
+      )
+    )
+  }
+
+  @Test
+  fun distributionListRecipients() {
+    importExport(
+      *standardFrames,
+      Recipient(
+        id = 3,
+        contact = Contact(
+          aci = TestRecipientUtils.nextAci().toByteString(),
+          pni = TestRecipientUtils.nextPni().toByteString(),
+          username = "coolusername",
+          e164 = 141255501234,
+          blocked = true,
+          hidden = true,
+          registered = Contact.Registered.REGISTERED,
+          unregisteredTimestamp = 0L,
+          profileKey = TestRecipientUtils.generateProfileKey().toByteString(),
+          profileSharing = true,
+          profileGivenName = "Alexa",
+          profileFamilyName = "Kim",
+          hideStory = true
+        )
+      ),
+      Recipient(
+        id = 4,
+        contact = Contact(
+          aci = null,
+          pni = null,
+          username = null,
+          e164 = 141255501235,
+          blocked = true,
+          hidden = true,
+          registered = Contact.Registered.REGISTERED,
+          unregisteredTimestamp = 0L,
+          profileKey = TestRecipientUtils.generateProfileKey().toByteString(),
+          profileSharing = true,
+          profileGivenName = "Peter",
+          profileFamilyName = "Kim",
+          hideStory = true
+        )
+      ),
+      Recipient(
+        id = 5,
+        contact = Contact(
+          aci = null,
+          pni = null,
+          username = null,
+          e164 = 141255501236,
+          blocked = true,
+          hidden = true,
+          registered = Contact.Registered.REGISTERED,
+          unregisteredTimestamp = 0L,
+          profileKey = TestRecipientUtils.generateProfileKey().toByteString(),
+          profileSharing = true,
+          profileGivenName = "Father",
+          profileFamilyName = "Kim",
+          hideStory = true
+        )
+      ),
+      Recipient(
+        id = 6,
+        distributionList = DistributionList(
+          name = "Kim Family",
+          distributionId = DistributionId.create().asUuid().toByteArray().toByteString(),
+          allowReplies = true,
+          deletionTimestamp = 0L,
+          privacyMode = DistributionList.PrivacyMode.ONLY_WITH,
+          memberRecipientIds = listOf(3, 4, 5)
+        )
+      )
+    )
+  }
+
+  @Test
+  fun deletedDistributionList() {
+    val alexa = Recipient(
+      id = 3,
+      contact = Contact(
+        aci = TestRecipientUtils.nextAci().toByteString(),
+        pni = TestRecipientUtils.nextPni().toByteString(),
+        username = "coolusername",
+        e164 = 141255501234,
+        blocked = true,
+        hidden = true,
+        registered = Contact.Registered.REGISTERED,
+        unregisteredTimestamp = 0L,
+        profileKey = TestRecipientUtils.generateProfileKey().toByteString(),
+        profileSharing = true,
+        profileGivenName = "Alexa",
+        profileFamilyName = "Kim",
+        hideStory = true
+      )
+    )
+    import(
+      *standardFrames,
+      alexa,
+      Recipient(
+        id = 6,
+        distributionList = DistributionList(
+          name = "Deleted list",
+          distributionId = DistributionId.create().asUuid().toByteArray().toByteString(),
+          allowReplies = true,
+          deletionTimestamp = 12345L,
+          privacyMode = DistributionList.PrivacyMode.ONLY_WITH,
+          memberRecipientIds = listOf(3)
+        )
+      )
+    )
+    val exported = export()
+    val expected = exportFrames(
+      *standardFrames,
+      alexa
+    )
+    compare(expected, exported)
+  }
+
+  @Test
+  fun chatThreads() {
+    importExport(
+      *standardFrames,
+      Recipient(
+        id = 3,
+        contact = Contact(
+          aci = TestRecipientUtils.nextAci().toByteString(),
+          pni = TestRecipientUtils.nextPni().toByteString(),
+          username = "coolusername",
+          e164 = 141255501234,
+          blocked = false,
+          hidden = false,
+          registered = Contact.Registered.REGISTERED,
+          unregisteredTimestamp = 0L,
+          profileKey = TestRecipientUtils.generateProfileKey().toByteString(),
+          profileSharing = true,
+          profileGivenName = "Alexa",
+          profileFamilyName = "Kim",
+          hideStory = true
+        )
+      ),
+      Recipient(
+        id = 4,
+        group = Group(
+          masterKey = TestRecipientUtils.generateGroupMasterKey().toByteString(),
+          whitelisted = true,
+          hideStory = true,
+          storySendMode = Group.StorySendMode.DEFAULT,
+          name = "Cool test group"
+        )
+      ),
+      Chat(
+        id = 1,
+        recipientId = 3,
+        archived = true,
+        pinnedOrder = 1,
+        expirationTimerMs = 1.days.inWholeMilliseconds,
+        muteUntilMs = System.currentTimeMillis(),
+        markedUnread = true,
+        dontNotifyForMentionsIfMuted = true,
+        wallpaper = null
+      )
+    )
+  }
+
+  /**
+   * Export passed in frames as a backup. Does not automatically include
+   * any standard frames (e.g. backup header).
+   */
+  private fun exportFrames(vararg objects: Any): ByteArray {
+    val outputStream = ByteArrayOutputStream()
+    val writer = EncryptedBackupWriter(
+      key = SignalStore.svr().getOrCreateMasterKey().deriveBackupKey(),
+      aci = SignalStore.account().aci!!,
+      outputStream = outputStream,
+      append = { mac -> outputStream.write(mac) }
+    )
+
+    writer.use {
+      for (obj in objects) {
+        when (obj) {
+          is BackupInfo -> writer.write(obj)
+          is AccountData -> writer.write(Frame(account = obj))
+          is Recipient -> writer.write(Frame(recipient = obj))
+          is Chat -> writer.write(Frame(chat = obj))
+          is ChatItem -> writer.write(Frame(chatItem = obj))
+          is Call -> writer.write(Frame(call = obj))
+          is StickerPack -> writer.write(Frame(stickerPack = obj))
+          else -> Assert.fail("invalid object $obj")
+        }
+      }
+    }
+    return outputStream.toByteArray()
+  }
+
+  /**
+   * Exports the passed in frames as a backup and then attempts to
+   * import them.
+   */
+  private fun import(vararg objects: Any) {
+    val importData = exportFrames(*objects)
+    BackupRepository.import(length = importData.size.toLong(), inputStreamFactory = { ByteArrayInputStream(importData) }, selfData = BackupRepository.SelfData(SELF_ACI, SELF_PNI, SELF_E164, SELF_PROFILE_KEY))
+  }
+
+  /**
+   * Export our current database as a backup.
+   */
+  private fun export() = BackupRepository.export()
+
+  /**
+   * Imports the passed in frames and then exports them.
+   *
+   * It will do a comparison to assert that the import and export
+   * are equal.
+   */
  private fun importExport(vararg objects: Any) {
    val outputStream = ByteArrayOutputStream()
    val writer = EncryptedBackupWriter(
@@ -115,7 +409,7 @@ class ImportExportTest {
    val importData = outputStream.toByteArray()
    BackupRepository.import(length = importData.size.toLong(), inputStreamFactory = { ByteArrayInputStream(importData) }, selfData = BackupRepository.SelfData(SELF_ACI, SELF_PNI, SELF_E164, SELF_PROFILE_KEY))

-    val export = BackupRepository.export()
+    val export = export()
    compare(importData, export)
  }

@@ -180,6 +474,18 @@ class ImportExportTest {
  }

  private fun <T, R : Comparable<R>> prettyAssertEquals(import: List<T>, export: List<T>, selector: (T) -> R?) {
+    if (import.size != export.size) {
+      var msg = StringBuilder()
+      for (i in import) {
+        msg.append(i)
+        msg.append("\n")
+      }
+      for (i in export) {
+        msg.append(i)
+        msg.append("\n")
+      }
+      Assert.fail(msg.toString())
+    }
    Assert.assertEquals(import.size, export.size)
    val sortedImport = import.sortedBy(selector)
    val sortedExport = export.sortedBy(selector)
@@ -0,0 +1,48 @@
+/*
+ * Copyright 2024 Signal Messenger, LLC
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+package org.thoughtcrime.securesms.backup.v2
+
+import org.thoughtcrime.securesms.crypto.ProfileKeyUtil
+import org.whispersystems.signalservice.api.util.toByteArray
+import java.util.UUID
+import kotlin.random.Random
+
+object TestRecipientUtils {
+
+  private var upperGenAci = 13131313L
+  private var lowerGenAci = 0L
+
+  private var upperGenPni = 12121212L
+  private var lowerGenPni = 0L
+
+  private var groupMasterKeyRandom = Random(12345)
+
+  fun generateProfileKey(): ByteArray {
+    return ProfileKeyUtil.createNew().serialize()
+  }
+
+  fun nextPni(): ByteArray {
+    synchronized(this) {
+      lowerGenPni++
+      var uuid = UUID(upperGenPni, lowerGenPni)
+      return uuid.toByteArray()
+    }
+  }
+
+  fun nextAci(): ByteArray {
+    synchronized(this) {
+      lowerGenAci++
+      var uuid = UUID(upperGenAci, lowerGenAci)
+      return uuid.toByteArray()
+    }
+  }
+
+  fun generateGroupMasterKey(): ByteArray {
+    val masterKey = ByteArray(32)
+    groupMasterKeyRandom.nextBytes(masterKey)
+    return masterKey
+  }
+}
@@ -34,6 +34,7 @@ fun DistributionListTables.getAllForBackup(): List<BackupRecipient> {
  val records = readableDatabase
    .select()
    .from(DistributionListTables.ListTable.TABLE_NAME)
+    .where(DistributionListTables.ListTable.IS_NOT_DELETED)
    .run()
    .readToList { cursor ->
      val id: DistributionListId = DistributionListId.from(cursor.requireLong(DistributionListTables.ListTable.ID))
@@ -38,6 +38,7 @@ import org.thoughtcrime.securesms.database.model.databaseprotos.RecipientExtras
 import org.thoughtcrime.securesms.dependencies.ApplicationDependencies
 import org.thoughtcrime.securesms.groups.GroupId
 import org.thoughtcrime.securesms.groups.v2.processing.GroupsV2StateProcessor
+import org.thoughtcrime.securesms.keyvalue.SignalStore
 import org.thoughtcrime.securesms.phonenumbers.PhoneNumberFormatter
 import org.thoughtcrime.securesms.profiles.ProfileName
 import org.thoughtcrime.securesms.recipients.Recipient
@@ -127,6 +128,7 @@ fun RecipientTable.restoreRecipientFromBackup(recipient: BackupRecipient, backup
    recipient.group != null -> restoreGroupFromBackup(recipient.group)
    recipient.distributionList != null -> SignalDatabase.distributionLists.restoreFromBackup(recipient.distributionList, backupState)
    recipient.self != null -> Recipient.self().id
+    recipient.releaseNotes != null -> restoreReleaseNotes()
    else -> {
      Log.w(TAG, "Unrecognized recipient type!")
      null
@@ -187,6 +189,7 @@ private fun RecipientTable.restoreContactFromBackup(contact: Contact): Recipient
    .values(
      RecipientTable.BLOCKED to contact.blocked,
      RecipientTable.HIDDEN to contact.hidden,
+      RecipientTable.TYPE to RecipientTable.RecipientType.INDIVIDUAL.id,
      RecipientTable.PROFILE_FAMILY_NAME to contact.profileFamilyName.nullIfBlank(),
      RecipientTable.PROFILE_GIVEN_NAME to contact.profileGivenName.nullIfBlank(),
      RecipientTable.PROFILE_JOINED_NAME to ProfileName.fromParts(contact.profileGivenName.nullIfBlank(), contact.profileFamilyName.nullIfBlank()).toString().nullIfBlank(),
@@ -203,6 +206,15 @@ private fun RecipientTable.restoreContactFromBackup(contact: Contact): Recipient
  return id
 }

+private fun RecipientTable.restoreReleaseNotes(): RecipientId {
+  val releaseChannelId: RecipientId = insertReleaseChannelRecipient()
+  SignalStore.releaseChannelValues().setReleaseChannelRecipientId(releaseChannelId)
+
+  setProfileName(releaseChannelId, ProfileName.asGiven("Signal"))
+  setMuted(releaseChannelId, Long.MAX_VALUE)
+  return releaseChannelId
+}
+
 private fun RecipientTable.restoreGroupFromBackup(group: Group): RecipientId {
  val masterKey = GroupMasterKey(group.masterKey.toByteArray())
  val groupId = GroupId.v2(masterKey)
@@ -6,15 +6,16 @@
 package org.thoughtcrime.securesms.backup.v2.database

 import android.database.Cursor
+import androidx.core.content.contentValuesOf
 import org.signal.core.util.SqlUtil
 import org.signal.core.util.insertInto
 import org.signal.core.util.logging.Log
 import org.signal.core.util.requireBoolean
 import org.signal.core.util.requireInt
 import org.signal.core.util.requireLong
-import org.signal.core.util.select
 import org.signal.core.util.toInt
 import org.thoughtcrime.securesms.backup.v2.proto.Chat
+import org.thoughtcrime.securesms.database.RecipientTable
 import org.thoughtcrime.securesms.database.ThreadTable
 import org.thoughtcrime.securesms.recipients.RecipientId
 import java.io.Closeable
@@ -22,16 +23,21 @@ import java.io.Closeable
 private val TAG = Log.tag(ThreadTable::class.java)

 fun ThreadTable.getThreadsForBackup(): ChatIterator {
-  val cursor = readableDatabase
-    .select(
-      ThreadTable.ID,
-      ThreadTable.RECIPIENT_ID,
-      ThreadTable.ARCHIVED,
-      ThreadTable.PINNED,
-      ThreadTable.EXPIRES_IN
-    )
-    .from(ThreadTable.TABLE_NAME)
-    .run()
+  //language=sql
+  val query = """
+      SELECT 
+        ${ThreadTable.TABLE_NAME}.${ThreadTable.ID}, 
+        ${ThreadTable.RECIPIENT_ID}, 
+        ${ThreadTable.PINNED}, 
+        ${ThreadTable.READ}, 
+        ${ThreadTable.ARCHIVED},
+        ${RecipientTable.TABLE_NAME}.${RecipientTable.MESSAGE_EXPIRATION_TIME},
+        ${RecipientTable.TABLE_NAME}.${RecipientTable.MUTE_UNTIL},
+        ${RecipientTable.TABLE_NAME}.${RecipientTable.MENTION_SETTING} 
+      FROM ${ThreadTable.TABLE_NAME} 
+        LEFT OUTER JOIN ${RecipientTable.TABLE_NAME} ON ${ThreadTable.TABLE_NAME}.${ThreadTable.RECIPIENT_ID} = ${RecipientTable.TABLE_NAME}.${RecipientTable.ID}
+    """
+  val cursor = readableDatabase.query(query)

  return ChatIterator(cursor)
 }
@@ -43,14 +49,29 @@ fun ThreadTable.clearAllDataForBackupRestore() {
 }

 fun ThreadTable.restoreFromBackup(chat: Chat, recipientId: RecipientId): Long? {
-  return writableDatabase
+  val threadId = writableDatabase
    .insertInto(ThreadTable.TABLE_NAME)
    .values(
      ThreadTable.RECIPIENT_ID to recipientId.serialize(),
      ThreadTable.PINNED to chat.pinnedOrder,
-      ThreadTable.ARCHIVED to chat.archived.toInt()
+      ThreadTable.ARCHIVED to chat.archived.toInt(),
+      ThreadTable.READ to if (chat.markedUnread) ThreadTable.ReadStatus.FORCED_UNREAD.serialize() else ThreadTable.ReadStatus.READ.serialize(),
+      ThreadTable.ACTIVE to 1
    )
    .run()
+  writableDatabase
+    .update(
+      RecipientTable.TABLE_NAME,
+      contentValuesOf(
+        RecipientTable.MENTION_SETTING to (if (chat.dontNotifyForMentionsIfMuted) RecipientTable.MentionSetting.DO_NOT_NOTIFY.id else RecipientTable.MentionSetting.ALWAYS_NOTIFY.id),
+        RecipientTable.MUTE_UNTIL to chat.muteUntilMs,
+        RecipientTable.MESSAGE_EXPIRATION_TIME to chat.expirationTimerMs
+      ),
+      "${RecipientTable.ID} = ?",
+      SqlUtil.buildArgs(recipientId.toLong())
+    )
+
+  return threadId
 }

 class ChatIterator(private val cursor: Cursor) : Iterator<Chat>, Closeable {
@@ -68,7 +89,10 @@ class ChatIterator(private val cursor: Cursor) : Iterator<Chat>, Closeable {
      recipientId = cursor.requireLong(ThreadTable.RECIPIENT_ID),
      archived = cursor.requireBoolean(ThreadTable.ARCHIVED),
      pinnedOrder = cursor.requireInt(ThreadTable.PINNED),
-      expirationTimerMs = cursor.requireLong(ThreadTable.EXPIRES_IN)
+      expirationTimerMs = cursor.requireLong(RecipientTable.MESSAGE_EXPIRATION_TIME),
+      muteUntilMs = cursor.requireLong(RecipientTable.MUTE_UNTIL),
+      markedUnread = ThreadTable.ReadStatus.deserialize(cursor.requireInt(ThreadTable.READ)) == ThreadTable.ReadStatus.FORCED_UNREAD,
+      dontNotifyForMentionsIfMuted = RecipientTable.MentionSetting.DO_NOT_NOTIFY.id == cursor.requireInt(RecipientTable.MENTION_SETTING)
    )
  }

@@ -28,10 +28,22 @@ object RecipientBackupProcessor {

  fun export(state: ExportState, emitter: BackupFrameEmitter) {
    val selfId = Recipient.self().id.toLong()
+    val releaseChannelId = SignalStore.releaseChannelValues().releaseChannelRecipientId
+    if (releaseChannelId != null) {
+      emitter.emit(
+        Frame(
+          recipient = BackupRecipient(
+            id = releaseChannelId.toLong(),
+            releaseNotes = ReleaseNotes()
+          )
+        )
+      )
+    }

    SignalDatabase.recipients.getContactsForBackup(selfId).use { reader ->
      for (backupRecipient in reader) {
        if (backupRecipient != null) {
+          state.recipientIds.add(backupRecipient.id)
          emitter.emit(Frame(recipient = backupRecipient))
        }
      }
@@ -48,18 +60,6 @@ object RecipientBackupProcessor {
      state.recipientIds.add(it.id)
      emitter.emit(Frame(recipient = it))
    }
-
-    val releaseChannelId = SignalStore.releaseChannelValues().releaseChannelRecipientId
-    if (releaseChannelId != null) {
-      emitter.emit(
-        Frame(
-          recipient = BackupRecipient(
-            id = releaseChannelId.toLong(),
-            releaseNotes = ReleaseNotes()
-          )
-        )
-      )
-    }
  }

  fun import(recipient: BackupRecipient, backupState: BackupState) {