Export backupV2 using actual desired file format.

app/src/androidTest/java/org/thoughtcrime/securesms/backup/v2/BackupTest.kt

@@ -19,6 +19,7 @@ import org.signal.core.util.requireBlob
 import org.signal.core.util.requireLong
 import org.signal.core.util.requireString
 import org.signal.core.util.select
+import org.signal.core.util.toInt
 import org.signal.libsignal.zkgroup.profiles.ProfileKey
 import org.thoughtcrime.securesms.backup.v2.database.clearAllDataForBackupRestore
 import org.thoughtcrime.securesms.database.EmojiSearchTable
@@ -40,6 +41,7 @@ import org.thoughtcrime.securesms.util.TextSecurePreferences
 import org.whispersystems.signalservice.api.push.ServiceId.ACI
 import org.whispersystems.signalservice.api.push.ServiceId.PNI
 import org.whispersystems.signalservice.api.subscriptions.SubscriberId
+import java.io.ByteArrayInputStream
 import java.util.UUID
 import kotlin.random.Random
 
@@ -176,7 +178,6 @@ class BackupTest {
       SignalStore.settings().setKeepMutedChatsArchived(true)
 
       SignalStore.storyValues().viewedReceiptsEnabled = false
-      SignalStore.storyValues().userHasReadOnboardingStory = true
       SignalStore.storyValues().userHasViewedOnboardingStory = true
       SignalStore.storyValues().isFeatureDisabled = false
       SignalStore.storyValues().userHasBeenNotifiedAboutStories = true
@@ -227,7 +228,8 @@ class BackupTest {
     val startingMainData: DatabaseData = SignalDatabase.rawDatabase.readAllContents()
     val startingKeyValueData: DatabaseData = if (validateKeyValue) SignalDatabase.rawDatabase.readAllContents() else emptyMap()
 
-    BackupRepository.import(BackupRepository.export())
+    val exported: ByteArray = BackupRepository.export()
+    BackupRepository.import(length = exported.size.toLong(), inputStreamFactory = { ByteArrayInputStream(exported) }, selfData = BackupRepository.SelfData(SELF_ACI, SELF_PNI, SELF_E164, SELF_PROFILE_KEY))
 
     val endingData: DatabaseData = SignalDatabase.rawDatabase.readAllContents()
     val endingKeyValueData: DatabaseData = if (validateKeyValue) SignalDatabase.rawDatabase.readAllContents() else emptyMap()
@@ -299,7 +301,7 @@ class BackupTest {
     fun standardMessage(
       outgoing: Boolean,
       sentTimestamp: Long = System.currentTimeMillis(),
-      receivedTimestamp: Long = sentTimestamp + 1,
+      receivedTimestamp: Long = if (outgoing) sentTimestamp else sentTimestamp + 1,
       serverTimestamp: Long = sentTimestamp,
       body: String? = null,
       read: Boolean = true,
@@ -328,7 +330,7 @@ class BackupTest {
     fun remoteDeletedMessage(
       outgoing: Boolean,
       sentTimestamp: Long = System.currentTimeMillis(),
-      receivedTimestamp: Long = sentTimestamp + 1,
+      receivedTimestamp: Long = if (outgoing) sentTimestamp else sentTimestamp + 1,
       serverTimestamp: Long = sentTimestamp
     ): Long {
       return db.insertMessage(
@@ -350,7 +352,7 @@ class BackupTest {
     outgoing: Boolean,
     threadId: Long,
     sentTimestamp: Long = System.currentTimeMillis(),
-    receivedTimestamp: Long = sentTimestamp + 1,
+    receivedTimestamp: Long = if (outgoing) sentTimestamp else sentTimestamp + 1,
     serverTimestamp: Long = sentTimestamp,
     body: String? = null,
     read: Boolean = true,
@@ -390,12 +392,12 @@ class BackupTest {
 
     if (quotes != null) {
       val quoteDetails = this.getQuoteDetailsFor(quotes)
-      contentValues.put(MessageTable.QUOTE_ID, quoteDetails.quotedSentTimestamp)
+      contentValues.put(MessageTable.QUOTE_ID, if (quoteTargetMissing) MessageTable.QUOTE_TARGET_MISSING_ID else quoteDetails.quotedSentTimestamp)
       contentValues.put(MessageTable.QUOTE_AUTHOR, quoteDetails.authorId.serialize())
       contentValues.put(MessageTable.QUOTE_BODY, quoteDetails.body)
       contentValues.put(MessageTable.QUOTE_BODY_RANGES, quoteDetails.bodyRanges)
       contentValues.put(MessageTable.QUOTE_TYPE, quoteDetails.type)
-      contentValues.put(MessageTable.QUOTE_MISSING, quoteTargetMissing)
+      contentValues.put(MessageTable.QUOTE_MISSING, quoteTargetMissing.toInt())
     }
 
     if (body != null && (randomMention || randomStyling)) {
@@ -493,7 +495,7 @@ class BackupTest {
 
         if (!contentEquals(expectedValue, actualValue)) {
           if (!describedRow) {
-            builder.append("-- ROW $i\n")
+            builder.append("-- ROW ${i + 1}\n")
             describedRow = true
           }
           builder.append("  [$key] Expected: ${expectedValue.prettyPrint()} || Actual: ${actualValue.prettyPrint()} \n")
@@ -502,6 +504,8 @@ class BackupTest {
 
       if (describedRow) {
         builder.append("\n")
+        builder.append("Expected: $expectedRow\n")
+        builder.append("Actual: $actualRow\n")
       }
     }
 

app/src/main/java/org/thoughtcrime/securesms/backup/v2/BackupRepository.kt

@@ -8,31 +8,46 @@ package org.thoughtcrime.securesms.backup.v2
 import org.signal.core.util.EventTimer
 import org.signal.core.util.logging.Log
 import org.signal.core.util.withinTransaction
+import org.signal.libsignal.zkgroup.profiles.ProfileKey
 import org.thoughtcrime.securesms.backup.v2.database.ChatItemImportInserter
 import org.thoughtcrime.securesms.backup.v2.database.clearAllDataForBackupRestore
 import org.thoughtcrime.securesms.backup.v2.processor.AccountDataProcessor
 import org.thoughtcrime.securesms.backup.v2.processor.ChatBackupProcessor
 import org.thoughtcrime.securesms.backup.v2.processor.ChatItemBackupProcessor
 import org.thoughtcrime.securesms.backup.v2.processor.RecipientBackupProcessor
-import org.thoughtcrime.securesms.backup.v2.stream.BackupExportStream
+import org.thoughtcrime.securesms.backup.v2.stream.BackupExportWriter
-import org.thoughtcrime.securesms.backup.v2.stream.PlainTextBackupExportStream
+import org.thoughtcrime.securesms.backup.v2.stream.EncryptedBackupReader
-import org.thoughtcrime.securesms.backup.v2.stream.PlainTextBackupImportStream
+import org.thoughtcrime.securesms.backup.v2.stream.EncryptedBackupWriter
+import org.thoughtcrime.securesms.backup.v2.stream.PlainTextBackupReader
+import org.thoughtcrime.securesms.backup.v2.stream.PlainTextBackupWriter
 import org.thoughtcrime.securesms.database.SignalDatabase
 import org.thoughtcrime.securesms.keyvalue.SignalStore
 import org.thoughtcrime.securesms.recipients.RecipientId
-import java.io.ByteArrayInputStream
+import org.whispersystems.signalservice.api.push.ServiceId.ACI
+import org.whispersystems.signalservice.api.push.ServiceId.PNI
 import java.io.ByteArrayOutputStream
+import java.io.InputStream
 
 object BackupRepository {
 
   private val TAG = Log.tag(BackupRepository::class.java)
 
-  fun export(): ByteArray {
+  fun export(plaintext: Boolean = false): ByteArray {
     val eventTimer = EventTimer()
 
     val outputStream = ByteArrayOutputStream()
-    val writer: BackupExportStream = PlainTextBackupExportStream(outputStream)
+    val writer: BackupExportWriter = if (plaintext) {
+      PlainTextBackupWriter(outputStream)
+    } else {
+      EncryptedBackupWriter(
+        key = SignalStore.svr().getOrCreateMasterKey().deriveBackupKey(),
+        aci = SignalStore.account().aci!!,
+        outputStream = outputStream,
+        append = { mac -> outputStream.write(mac) }
+      )
+    }
 
+    writer.use {
       // Note: Without a transaction, we may export inconsistent state. But because we have a transaction,
       // writes from other threads are blocked. This is something to think more about.
       SignalDatabase.rawDatabase.withinTransaction {
@@ -56,17 +71,26 @@ object BackupRepository {
           eventTimer.emit("message")
         }
       }
+    }
 
     Log.d(TAG, "export() ${eventTimer.stop().summary}")
 
     return outputStream.toByteArray()
   }
 
-  fun import(data: ByteArray) {
+  fun import(length: Long, inputStreamFactory: () -> InputStream, selfData: SelfData, plaintext: Boolean = false) {
     val eventTimer = EventTimer()
 
-    val stream = ByteArrayInputStream(data)
+    val frameReader = if (plaintext) {
-    val frameReader = PlainTextBackupImportStream(stream)
+      PlainTextBackupReader(inputStreamFactory())
+    } else {
+      EncryptedBackupReader(
+        key = SignalStore.svr().getOrCreateMasterKey().deriveBackupKey(),
+        aci = selfData.aci,
+        streamLength = length,
+        dataStream = inputStreamFactory
+      )
+    }
 
     // Note: Without a transaction, bad imports could lead to lost data. But because we have a transaction,
     // writes from other threads are blocked. This is something to think more about.
@@ -76,6 +100,12 @@ object BackupRepository {
       SignalDatabase.distributionLists.clearAllDataForBackupRestore()
       SignalDatabase.threads.clearAllDataForBackupRestore()
       SignalDatabase.messages.clearAllDataForBackupRestore()
+      SignalDatabase.attachments.clearAllDataForBackupRestore()
+
+      // Add back self after clearing data
+      val selfId: RecipientId = SignalDatabase.recipients.getAndPossiblyMerge(selfData.aci, selfData.pni, selfData.e164, pniVerified = true, changeSelf = true)
+      SignalDatabase.recipients.setProfileKey(selfId, selfData.profileKey)
+      SignalDatabase.recipients.setProfileSharing(selfId, true)
 
       val backupState = BackupState()
       val chatItemInserter: ChatItemImportInserter = ChatItemBackupProcessor.beginImport(backupState)
@@ -83,7 +113,7 @@ object BackupRepository {
       for (frame in frameReader) {
         when {
           frame.account != null -> {
-            AccountDataProcessor.import(frame.account)
+            AccountDataProcessor.import(frame.account, selfId)
             eventTimer.emit("account")
           }
 
@@ -118,6 +148,13 @@ object BackupRepository {
 
     Log.d(TAG, "import() ${eventTimer.stop().summary}")
   }
+
+  data class SelfData(
+    val aci: ACI,
+    val pni: PNI,
+    val e164: String,
+    val profileKey: ProfileKey
+  )
 }
 
 class BackupState {

app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/AttachmentTableBackupExtensions.kt

@@ -0,0 +1,13 @@
+/*
+ * Copyright 2023 Signal Messenger, LLC
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+package org.thoughtcrime.securesms.backup.v2.database
+
+import org.signal.core.util.delete
+import org.thoughtcrime.securesms.database.AttachmentTable
+
+fun AttachmentTable.clearAllDataForBackupRestore() {
+  writableDatabase.delete(AttachmentTable.TABLE_NAME).run()
+}

app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/ChatItemExportIterator.kt

@@ -6,6 +6,7 @@
 package org.thoughtcrime.securesms.backup.v2.database
 
 import android.database.Cursor
+import okio.ByteString.Companion.toByteString
 import org.signal.core.util.Base64
 import org.signal.core.util.logging.Log
 import org.signal.core.util.requireBlob
@@ -14,16 +15,16 @@ import org.signal.core.util.requireInt
 import org.signal.core.util.requireLong
 import org.signal.core.util.requireString
 import org.thoughtcrime.securesms.backup.v2.proto.ChatItem
-import org.thoughtcrime.securesms.backup.v2.proto.ExpirationTimerChange
+import org.thoughtcrime.securesms.backup.v2.proto.ChatUpdateMessage
-import org.thoughtcrime.securesms.backup.v2.proto.ProfileChange
+import org.thoughtcrime.securesms.backup.v2.proto.ExpirationTimerChatUpdate
+import org.thoughtcrime.securesms.backup.v2.proto.ProfileChangeChatUpdate
 import org.thoughtcrime.securesms.backup.v2.proto.Quote
 import org.thoughtcrime.securesms.backup.v2.proto.Reaction
 import org.thoughtcrime.securesms.backup.v2.proto.RemoteDeletedMessage
 import org.thoughtcrime.securesms.backup.v2.proto.SendStatus
-import org.thoughtcrime.securesms.backup.v2.proto.SimpleUpdate
+import org.thoughtcrime.securesms.backup.v2.proto.SimpleChatUpdate
 import org.thoughtcrime.securesms.backup.v2.proto.StandardMessage
 import org.thoughtcrime.securesms.backup.v2.proto.Text
-import org.thoughtcrime.securesms.backup.v2.proto.UpdateMessage
 import org.thoughtcrime.securesms.database.GroupReceiptTable
 import org.thoughtcrime.securesms.database.MessageTable
 import org.thoughtcrime.securesms.database.MessageTypes
@@ -35,6 +36,8 @@ import org.thoughtcrime.securesms.database.model.databaseprotos.BodyRangeList
 import org.thoughtcrime.securesms.database.model.databaseprotos.ProfileChangeDetails
 import org.thoughtcrime.securesms.mms.QuoteModel
 import org.thoughtcrime.securesms.util.JsonUtils
+import org.whispersystems.signalservice.api.util.UuidUtil
+import org.whispersystems.signalservice.api.util.toByteArray
 import java.io.Closeable
 import java.io.IOException
 import java.util.LinkedList
@@ -90,31 +93,31 @@ class ChatItemExportIterator(private val cursor: Cursor, private val batchSize:
 
       when {
         record.remoteDeleted -> builder.remoteDeletedMessage = RemoteDeletedMessage()
-        MessageTypes.isJoinedType(record.type) -> builder.updateMessage = UpdateMessage(simpleUpdate = SimpleUpdate(type = SimpleUpdate.Type.JOINED_SIGNAL))
+        MessageTypes.isJoinedType(record.type) -> builder.updateMessage = ChatUpdateMessage(simpleUpdate = SimpleChatUpdate(type = SimpleChatUpdate.Type.JOINED_SIGNAL))
-        MessageTypes.isIdentityUpdate(record.type) -> builder.updateMessage = UpdateMessage(simpleUpdate = SimpleUpdate(type = SimpleUpdate.Type.IDENTITY_UPDATE))
+        MessageTypes.isIdentityUpdate(record.type) -> builder.updateMessage = ChatUpdateMessage(simpleUpdate = SimpleChatUpdate(type = SimpleChatUpdate.Type.IDENTITY_UPDATE))
-        MessageTypes.isIdentityVerified(record.type) -> builder.updateMessage = UpdateMessage(simpleUpdate = SimpleUpdate(type = SimpleUpdate.Type.IDENTITY_VERIFIED))
+        MessageTypes.isIdentityVerified(record.type) -> builder.updateMessage = ChatUpdateMessage(simpleUpdate = SimpleChatUpdate(type = SimpleChatUpdate.Type.IDENTITY_VERIFIED))
-        MessageTypes.isIdentityDefault(record.type) -> builder.updateMessage = UpdateMessage(simpleUpdate = SimpleUpdate(type = SimpleUpdate.Type.IDENTITY_DEFAULT))
+        MessageTypes.isIdentityDefault(record.type) -> builder.updateMessage = ChatUpdateMessage(simpleUpdate = SimpleChatUpdate(type = SimpleChatUpdate.Type.IDENTITY_DEFAULT))
-        MessageTypes.isChangeNumber(record.type) -> builder.updateMessage = UpdateMessage(simpleUpdate = SimpleUpdate(type = SimpleUpdate.Type.CHANGE_NUMBER))
+        MessageTypes.isChangeNumber(record.type) -> builder.updateMessage = ChatUpdateMessage(simpleUpdate = SimpleChatUpdate(type = SimpleChatUpdate.Type.CHANGE_NUMBER))
-        MessageTypes.isBoostRequest(record.type) -> builder.updateMessage = UpdateMessage(simpleUpdate = SimpleUpdate(type = SimpleUpdate.Type.BOOST_REQUEST))
+        MessageTypes.isBoostRequest(record.type) -> builder.updateMessage = ChatUpdateMessage(simpleUpdate = SimpleChatUpdate(type = SimpleChatUpdate.Type.BOOST_REQUEST))
-        MessageTypes.isEndSessionType(record.type) -> builder.updateMessage = UpdateMessage(simpleUpdate = SimpleUpdate(type = SimpleUpdate.Type.END_SESSION))
+        MessageTypes.isEndSessionType(record.type) -> builder.updateMessage = ChatUpdateMessage(simpleUpdate = SimpleChatUpdate(type = SimpleChatUpdate.Type.END_SESSION))
-        MessageTypes.isChatSessionRefresh(record.type) -> builder.updateMessage = UpdateMessage(simpleUpdate = SimpleUpdate(type = SimpleUpdate.Type.CHAT_SESSION_REFRESH))
+        MessageTypes.isChatSessionRefresh(record.type) -> builder.updateMessage = ChatUpdateMessage(simpleUpdate = SimpleChatUpdate(type = SimpleChatUpdate.Type.CHAT_SESSION_REFRESH))
-        MessageTypes.isBadDecryptType(record.type) -> builder.updateMessage = UpdateMessage(simpleUpdate = SimpleUpdate(type = SimpleUpdate.Type.BAD_DECRYPT))
+        MessageTypes.isBadDecryptType(record.type) -> builder.updateMessage = ChatUpdateMessage(simpleUpdate = SimpleChatUpdate(type = SimpleChatUpdate.Type.BAD_DECRYPT))
-        MessageTypes.isPaymentsActivated(record.type) -> builder.updateMessage = UpdateMessage(simpleUpdate = SimpleUpdate(type = SimpleUpdate.Type.PAYMENTS_ACTIVATED))
+        MessageTypes.isPaymentsActivated(record.type) -> builder.updateMessage = ChatUpdateMessage(simpleUpdate = SimpleChatUpdate(type = SimpleChatUpdate.Type.PAYMENTS_ACTIVATED))
-        MessageTypes.isPaymentsRequestToActivate(record.type) -> builder.updateMessage = UpdateMessage(simpleUpdate = SimpleUpdate(type = SimpleUpdate.Type.PAYMENT_ACTIVATION_REQUEST))
+        MessageTypes.isPaymentsRequestToActivate(record.type) -> builder.updateMessage = ChatUpdateMessage(simpleUpdate = SimpleChatUpdate(type = SimpleChatUpdate.Type.PAYMENT_ACTIVATION_REQUEST))
-        MessageTypes.isExpirationTimerUpdate(record.type) -> builder.updateMessage = UpdateMessage(expirationTimerChange = ExpirationTimerChange((record.expiresIn / 1000).toInt()))
+        MessageTypes.isExpirationTimerUpdate(record.type) -> builder.updateMessage = ChatUpdateMessage(expirationTimerChange = ExpirationTimerChatUpdate((record.expiresIn / 1000).toInt()))
         MessageTypes.isProfileChange(record.type) -> {
-          builder.updateMessage = UpdateMessage(
+          builder.updateMessage = ChatUpdateMessage(
             profileChange = try {
               val decoded: ByteArray = Base64.decode(record.body!!)
               val profileChangeDetails = ProfileChangeDetails.ADAPTER.decode(decoded)
               if (profileChangeDetails.profileNameChange != null) {
-                ProfileChange(previousName = profileChangeDetails.profileNameChange.previous, newName = profileChangeDetails.profileNameChange.newValue)
+                ProfileChangeChatUpdate(previousName = profileChangeDetails.profileNameChange.previous, newName = profileChangeDetails.profileNameChange.newValue)
               } else {
-                ProfileChange()
+                ProfileChangeChatUpdate()
               }
             } catch (e: IOException) {
               Log.w(TAG, "Profile name change details could not be read", e)
-              ProfileChange()
+              ProfileChangeChatUpdate()
             }
           )
         }
@@ -142,9 +145,9 @@ class ChatItemExportIterator(private val cursor: Cursor, private val batchSize:
       chatId = record.threadId
       authorId = record.fromRecipientId
       dateSent = record.dateSent
-      dateReceived = record.dateReceived
+      sealedSender = record.sealedSender
-      expireStart = if (record.expireStarted > 0) record.expireStarted else null
+      expireStartDate = if (record.expireStarted > 0) record.expireStarted else null
-      expiresIn = if (record.expiresIn > 0) record.expiresIn else null
+      expiresInMs = if (record.expiresIn > 0) record.expiresIn else null
       revisions = emptyList()
       sms = !MessageTypes.isSecureType(record.type)
 
@@ -155,7 +158,7 @@ class ChatItemExportIterator(private val cursor: Cursor, private val batchSize:
       } else {
         incoming = ChatItem.IncomingMessageDetails(
           dateServerSent = record.dateServer,
-          sealedSender = record.sealedSender,
+          dateReceived = record.dateReceived,
           read = record.read
         )
       }
@@ -169,21 +172,21 @@ class ChatItemExportIterator(private val cursor: Cursor, private val batchSize:
         body = this.body!!,
         bodyRanges = this.bodyRanges?.toBackupBodyRanges() ?: emptyList()
       ),
-      linkPreview = null,
+      // TODO Link previews!
+      linkPreview = emptyList(),
       longText = null,
       reactions = reactionRecords.toBackupReactions()
     )
   }
 
   private fun BackupMessageRecord.toQuote(): Quote? {
-    return if (this.quoteTargetSentTimestamp > 0) {
+    return if (this.quoteTargetSentTimestamp != MessageTable.QUOTE_NOT_PRESENT_ID && this.quoteAuthor > 0) {
       // TODO Attachments!
       val type = QuoteModel.Type.fromCode(this.quoteType)
       Quote(
-        targetSentTimestamp = this.quoteTargetSentTimestamp,
+        targetSentTimestamp = this.quoteTargetSentTimestamp.takeIf { !this.quoteMissing && it != MessageTable.QUOTE_TARGET_MISSING_ID },
         authorId = this.quoteAuthor,
         text = this.quoteBody,
-        originalMessageMissing = this.quoteMissing,
         bodyRanges = this.quoteBodyRanges?.toBackupBodyRanges() ?: emptyList(),
         type = when (type) {
           QuoteModel.Type.NORMAL -> Quote.Type.NORMAL
@@ -207,7 +210,7 @@ class ChatItemExportIterator(private val cursor: Cursor, private val batchSize:
       BackupBodyRange(
         start = it.start,
         length = it.length,
-        mentionAci = it.mentionUuid,
+        mentionAci = it.mentionUuid?.let { UuidUtil.parseOrThrow(it) }?.toByteArray()?.toByteString(),
         style = it.style?.toBackupBodyRangeStyle()
       )
     }
@@ -245,9 +248,9 @@ class ChatItemExportIterator(private val cursor: Cursor, private val batchSize:
     }
 
     val status: SendStatus.Status = when {
-      this.viewedReceiptCount > 0 -> SendStatus.Status.VIEWED
+      this.viewed -> SendStatus.Status.VIEWED
-      this.readReceiptCount > 0 -> SendStatus.Status.READ
+      this.hasReadReceipt -> SendStatus.Status.READ
-      this.deliveryReceiptCount > 0 -> SendStatus.Status.DELIVERED
+      this.hasDeliveryReceipt -> SendStatus.Status.DELIVERED
       this.baseType == MessageTypes.BASE_SENT_TYPE -> SendStatus.Status.SENT
       MessageTypes.isFailedMessageType(this.type) -> SendStatus.Status.FAILED
       else -> SendStatus.Status.PENDING
@@ -257,7 +260,7 @@ class ChatItemExportIterator(private val cursor: Cursor, private val batchSize:
       SendStatus(
         recipientId = this.toRecipientId,
         deliveryStatus = status,
-        timestamp = this.receiptTimestamp,
+        lastStatusUpdateTimestamp = this.receiptTimestamp,
         sealedSender = this.sealedSender,
         networkFailure = this.networkFailureRecipientIds.contains(this.toRecipientId),
         identityKeyMismatch = this.identityMismatchRecipientIds.contains(this.toRecipientId)
@@ -271,7 +274,7 @@ class ChatItemExportIterator(private val cursor: Cursor, private val batchSize:
         recipientId = it.recipientId.toLong(),
         deliveryStatus = it.status.toBackupDeliveryStatus(),
         sealedSender = it.isUnidentified,
-        timestamp = it.timestamp,
+        lastStatusUpdateTimestamp = it.timestamp,
         networkFailure = networkFailureRecipientIds.contains(it.recipientId.toLong()),
         identityKeyMismatch = identityMismatchRecipientIds.contains(it.recipientId.toLong())
       )
@@ -337,9 +340,9 @@ class ChatItemExportIterator(private val cursor: Cursor, private val batchSize:
       quoteType = this.requireInt(MessageTable.QUOTE_TYPE),
       originalMessageId = this.requireLong(MessageTable.ORIGINAL_MESSAGE_ID),
       latestRevisionId = this.requireLong(MessageTable.LATEST_REVISION_ID),
-      deliveryReceiptCount = this.requireInt(MessageTable.DELIVERY_RECEIPT_COUNT),
+      hasDeliveryReceipt = this.requireBoolean(MessageTable.HAS_DELIVERY_RECEIPT),
-      viewedReceiptCount = this.requireInt(MessageTable.VIEWED_RECEIPT_COUNT),
+      viewed = this.requireBoolean(MessageTable.VIEWED_COLUMN),
-      readReceiptCount = this.requireInt(MessageTable.READ_RECEIPT_COUNT),
+      hasReadReceipt = this.requireBoolean(MessageTable.HAS_READ_RECEIPT),
       read = this.requireBoolean(MessageTable.READ),
       receiptTimestamp = this.requireLong(MessageTable.RECEIPT_TIMESTAMP),
       networkFailureRecipientIds = this.requireString(MessageTable.NETWORK_FAILURES).parseNetworkFailures(),
@@ -371,9 +374,9 @@ class ChatItemExportIterator(private val cursor: Cursor, private val batchSize:
     val quoteType: Int,
     val originalMessageId: Long,
     val latestRevisionId: Long,
-    val deliveryReceiptCount: Int,
+    val hasDeliveryReceipt: Boolean,
-    val readReceiptCount: Int,
+    val hasReadReceipt: Boolean,
-    val viewedReceiptCount: Int,
+    val viewed: Boolean,
     val receiptTimestamp: Long,
     val read: Boolean,
     val networkFailureRecipientIds: Set<Long>,

app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/ChatItemImportInserter.kt

@@ -14,12 +14,12 @@ import org.signal.core.util.toInt
 import org.thoughtcrime.securesms.backup.v2.BackupState
 import org.thoughtcrime.securesms.backup.v2.proto.BodyRange
 import org.thoughtcrime.securesms.backup.v2.proto.ChatItem
+import org.thoughtcrime.securesms.backup.v2.proto.ChatUpdateMessage
 import org.thoughtcrime.securesms.backup.v2.proto.Quote
 import org.thoughtcrime.securesms.backup.v2.proto.Reaction
 import org.thoughtcrime.securesms.backup.v2.proto.SendStatus
-import org.thoughtcrime.securesms.backup.v2.proto.SimpleUpdate
+import org.thoughtcrime.securesms.backup.v2.proto.SimpleChatUpdate
 import org.thoughtcrime.securesms.backup.v2.proto.StandardMessage
-import org.thoughtcrime.securesms.backup.v2.proto.UpdateMessage
 import org.thoughtcrime.securesms.database.GroupReceiptTable
 import org.thoughtcrime.securesms.database.MessageTable
 import org.thoughtcrime.securesms.database.MessageTypes
@@ -35,6 +35,7 @@ import org.thoughtcrime.securesms.mms.QuoteModel
 import org.thoughtcrime.securesms.recipients.Recipient
 import org.thoughtcrime.securesms.recipients.RecipientId
 import org.thoughtcrime.securesms.util.JsonUtils
+import org.whispersystems.signalservice.api.util.UuidUtil
 
 /**
  * An object that will ingest all fo the [ChatItem]s you want to write, buffer them until hitting a specified batch size, and then batch insert them
@@ -58,9 +59,9 @@ class ChatItemImportInserter(
       MessageTable.BODY,
       MessageTable.FROM_RECIPIENT_ID,
       MessageTable.TO_RECIPIENT_ID,
-      MessageTable.DELIVERY_RECEIPT_COUNT,
+      MessageTable.HAS_DELIVERY_RECEIPT,
-      MessageTable.READ_RECEIPT_COUNT,
+      MessageTable.HAS_READ_RECEIPT,
-      MessageTable.VIEWED_RECEIPT_COUNT,
+      MessageTable.VIEWED_COLUMN,
       MessageTable.MISMATCHED_IDENTITIES,
       MessageTable.EXPIRES_IN,
       MessageTable.EXPIRE_STARTED,
@@ -173,32 +174,32 @@ class ChatItemImportInserter(
     contentValues.put(MessageTable.FROM_RECIPIENT_ID, fromRecipientId.serialize())
     contentValues.put(MessageTable.TO_RECIPIENT_ID, (if (this.outgoing != null) chatRecipientId else selfId).serialize())
     contentValues.put(MessageTable.THREAD_ID, threadId)
-    contentValues.put(MessageTable.DATE_RECEIVED, this.dateReceived)
+    contentValues.put(MessageTable.DATE_RECEIVED, this.incoming?.dateReceived ?: this.dateSent)
-    contentValues.put(MessageTable.RECEIPT_TIMESTAMP, this.outgoing?.sendStatus?.maxOf { it.timestamp } ?: 0)
+    contentValues.put(MessageTable.RECEIPT_TIMESTAMP, this.outgoing?.sendStatus?.maxOf { it.lastStatusUpdateTimestamp } ?: 0)
     contentValues.putNull(MessageTable.LATEST_REVISION_ID)
     contentValues.putNull(MessageTable.ORIGINAL_MESSAGE_ID)
     contentValues.put(MessageTable.REVISION_NUMBER, 0)
-    contentValues.put(MessageTable.EXPIRES_IN, this.expiresIn ?: 0)
+    contentValues.put(MessageTable.EXPIRES_IN, this.expiresInMs ?: 0)
-    contentValues.put(MessageTable.EXPIRE_STARTED, this.expireStart ?: 0)
+    contentValues.put(MessageTable.EXPIRE_STARTED, this.expireStartDate ?: 0)
 
     if (this.outgoing != null) {
-      val viewReceiptCount = this.outgoing.sendStatus.count { it.deliveryStatus == SendStatus.Status.VIEWED }
+      val viewed = this.outgoing.sendStatus.any { it.deliveryStatus == SendStatus.Status.VIEWED }
-      val readReceiptCount = Integer.max(viewReceiptCount, this.outgoing.sendStatus.count { it.deliveryStatus == SendStatus.Status.READ })
+      val hasReadReceipt = viewed || this.outgoing.sendStatus.any { it.deliveryStatus == SendStatus.Status.READ }
-      val deliveryReceiptCount = Integer.max(readReceiptCount, this.outgoing.sendStatus.count { it.deliveryStatus == SendStatus.Status.DELIVERED })
+      val hasDeliveryReceipt = viewed || hasReadReceipt || this.outgoing.sendStatus.any { it.deliveryStatus == SendStatus.Status.DELIVERED }
 
-      contentValues.put(MessageTable.VIEWED_RECEIPT_COUNT, viewReceiptCount)
+      contentValues.put(MessageTable.VIEWED_COLUMN, viewed.toInt())
-      contentValues.put(MessageTable.READ_RECEIPT_COUNT, readReceiptCount)
+      contentValues.put(MessageTable.HAS_READ_RECEIPT, hasReadReceipt.toInt())
-      contentValues.put(MessageTable.DELIVERY_RECEIPT_COUNT, deliveryReceiptCount)
+      contentValues.put(MessageTable.HAS_DELIVERY_RECEIPT, hasDeliveryReceipt.toInt())
       contentValues.put(MessageTable.UNIDENTIFIED, this.outgoing.sendStatus.count { it.sealedSender })
       contentValues.put(MessageTable.READ, 1)
 
       contentValues.addNetworkFailures(this, backupState)
       contentValues.addIdentityKeyMismatches(this, backupState)
     } else {
-      contentValues.put(MessageTable.VIEWED_RECEIPT_COUNT, 0)
+      contentValues.put(MessageTable.VIEWED_COLUMN, 0)
-      contentValues.put(MessageTable.READ_RECEIPT_COUNT, 0)
+      contentValues.put(MessageTable.HAS_READ_RECEIPT, 0)
-      contentValues.put(MessageTable.DELIVERY_RECEIPT_COUNT, 0)
+      contentValues.put(MessageTable.HAS_DELIVERY_RECEIPT, 0)
-      contentValues.put(MessageTable.UNIDENTIFIED, this.incoming?.sealedSender?.toInt() ?: 0)
+      contentValues.put(MessageTable.UNIDENTIFIED, this.sealedSender?.toInt())
       contentValues.put(MessageTable.READ, this.incoming?.read?.toInt() ?: 0)
     }
 
@@ -264,7 +265,7 @@ class ChatItemImportInserter(
           GroupReceiptTable.MMS_ID to messageId,
           GroupReceiptTable.RECIPIENT_ID to recipientId.serialize(),
           GroupReceiptTable.STATUS to sendStatus.deliveryStatus.toLocalSendStatus(),
-          GroupReceiptTable.TIMESTAMP to sendStatus.timestamp,
+          GroupReceiptTable.TIMESTAMP to sendStatus.lastStatusUpdateTimestamp,
           GroupReceiptTable.UNIDENTIFIED to sendStatus.sealedSender
         )
       } else {
@@ -308,27 +309,28 @@ class ChatItemImportInserter(
     }
   }
 
-  private fun ContentValues.addUpdateMessage(updateMessage: UpdateMessage) {
+  private fun ContentValues.addUpdateMessage(updateMessage: ChatUpdateMessage) {
     var typeFlags: Long = 0
     when {
       updateMessage.simpleUpdate != null -> {
         typeFlags = when (updateMessage.simpleUpdate.type) {
-          SimpleUpdate.Type.JOINED_SIGNAL -> MessageTypes.JOINED_TYPE
+          SimpleChatUpdate.Type.UNKNOWN -> 0
-          SimpleUpdate.Type.IDENTITY_UPDATE -> MessageTypes.KEY_EXCHANGE_IDENTITY_UPDATE_BIT
+          SimpleChatUpdate.Type.JOINED_SIGNAL -> MessageTypes.JOINED_TYPE
-          SimpleUpdate.Type.IDENTITY_VERIFIED -> MessageTypes.KEY_EXCHANGE_IDENTITY_VERIFIED_BIT
+          SimpleChatUpdate.Type.IDENTITY_UPDATE -> MessageTypes.KEY_EXCHANGE_IDENTITY_UPDATE_BIT
-          SimpleUpdate.Type.IDENTITY_DEFAULT -> MessageTypes.KEY_EXCHANGE_IDENTITY_DEFAULT_BIT
+          SimpleChatUpdate.Type.IDENTITY_VERIFIED -> MessageTypes.KEY_EXCHANGE_IDENTITY_VERIFIED_BIT
-          SimpleUpdate.Type.CHANGE_NUMBER -> MessageTypes.CHANGE_NUMBER_TYPE
+          SimpleChatUpdate.Type.IDENTITY_DEFAULT -> MessageTypes.KEY_EXCHANGE_IDENTITY_DEFAULT_BIT
-          SimpleUpdate.Type.BOOST_REQUEST -> MessageTypes.BOOST_REQUEST_TYPE
+          SimpleChatUpdate.Type.CHANGE_NUMBER -> MessageTypes.CHANGE_NUMBER_TYPE
-          SimpleUpdate.Type.END_SESSION -> MessageTypes.END_SESSION_BIT
+          SimpleChatUpdate.Type.BOOST_REQUEST -> MessageTypes.BOOST_REQUEST_TYPE
-          SimpleUpdate.Type.CHAT_SESSION_REFRESH -> MessageTypes.ENCRYPTION_REMOTE_FAILED_BIT
+          SimpleChatUpdate.Type.END_SESSION -> MessageTypes.END_SESSION_BIT
-          SimpleUpdate.Type.BAD_DECRYPT -> MessageTypes.BAD_DECRYPT_TYPE
+          SimpleChatUpdate.Type.CHAT_SESSION_REFRESH -> MessageTypes.ENCRYPTION_REMOTE_FAILED_BIT
-          SimpleUpdate.Type.PAYMENTS_ACTIVATED -> MessageTypes.SPECIAL_TYPE_PAYMENTS_ACTIVATED
+          SimpleChatUpdate.Type.BAD_DECRYPT -> MessageTypes.BAD_DECRYPT_TYPE
-          SimpleUpdate.Type.PAYMENT_ACTIVATION_REQUEST -> MessageTypes.SPECIAL_TYPE_PAYMENTS_ACTIVATE_REQUEST
+          SimpleChatUpdate.Type.PAYMENTS_ACTIVATED -> MessageTypes.SPECIAL_TYPE_PAYMENTS_ACTIVATED
+          SimpleChatUpdate.Type.PAYMENT_ACTIVATION_REQUEST -> MessageTypes.SPECIAL_TYPE_PAYMENTS_ACTIVATE_REQUEST
         }
       }
       updateMessage.expirationTimerChange != null -> {
         typeFlags = MessageTypes.EXPIRATION_TIMER_UPDATE_BIT
-        put(MessageTable.EXPIRES_IN, updateMessage.expirationTimerChange.expiresIn.toLong() * 1000)
+        put(MessageTable.EXPIRES_IN, updateMessage.expirationTimerChange.expiresInMs.toLong())
       }
       updateMessage.profileChange != null -> {
         typeFlags = MessageTypes.PROFILE_CHANGE_TYPE
@@ -341,13 +343,13 @@ class ChatItemImportInserter(
   }
 
   private fun ContentValues.addQuote(quote: Quote) {
-    this.put(MessageTable.QUOTE_ID, quote.targetSentTimestamp)
+    this.put(MessageTable.QUOTE_ID, quote.targetSentTimestamp ?: MessageTable.QUOTE_TARGET_MISSING_ID)
     this.put(MessageTable.QUOTE_AUTHOR, backupState.backupToLocalRecipientId[quote.authorId]!!.serialize())
     this.put(MessageTable.QUOTE_BODY, quote.text)
     this.put(MessageTable.QUOTE_TYPE, quote.type.toLocalQuoteType())
     this.put(MessageTable.QUOTE_BODY_RANGES, quote.bodyRanges.toLocalBodyRanges()?.encode())
     // TODO quote attachments
-    this.put(MessageTable.QUOTE_MISSING, quote.originalMessageMissing.toInt())
+    this.put(MessageTable.QUOTE_MISSING, (quote.targetSentTimestamp == null).toInt())
   }
 
   private fun Quote.Type.toLocalQuoteType(): Int {
@@ -398,7 +400,7 @@ class ChatItemImportInserter(
     return BodyRangeList(
       ranges = this.map { bodyRange ->
         BodyRangeList.BodyRange(
-          mentionUuid = bodyRange.mentionAci,
+          mentionUuid = bodyRange.mentionAci?.let { UuidUtil.fromByteString(it) }?.toString(),
           style = bodyRange.style?.let {
             when (bodyRange.style) {
               BodyRange.Style.BOLD -> BodyRangeList.BodyRange.Style.BOLD
@@ -418,6 +420,7 @@ class ChatItemImportInserter(
 
   private fun SendStatus.Status.toLocalSendStatus(): Int {
     return when (this) {
+      SendStatus.Status.UNKNOWN -> GroupReceiptTable.STATUS_UNKNOWN
       SendStatus.Status.FAILED -> GroupReceiptTable.STATUS_UNKNOWN
       SendStatus.Status.PENDING -> GroupReceiptTable.STATUS_UNDELIVERED
       SendStatus.Status.SENT -> GroupReceiptTable.STATUS_UNDELIVERED

app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/DistributionListTablesBackupExtensions.kt

@@ -58,7 +58,6 @@ fun DistributionListTables.getAllForBackup(): List<BackupRecipient> {
           distributionId = record.distributionId.asUuid().toByteArray().toByteString(),
           allowReplies = record.allowsReplies,
           deletionTimestamp = record.deletedAtTimestamp,
-          isUnknown = record.isUnknown,
           privacyMode = record.privacyMode.toBackupPrivacyMode(),
           memberRecipientIds = record.members.map { it.toLong() }
         )
@@ -81,7 +80,6 @@ fun DistributionListTables.restoreFromBackup(dlist: BackupDistributionList, back
     allowsReplies = dlist.allowReplies,
     deletionTimestamp = dlist.deletionTimestamp,
     storageId = null,
-    isUnknown = dlist.isUnknown,
     privacyMode = dlist.privacyMode.toLocalPrivacyMode()
   )!!
 
@@ -108,6 +106,7 @@ private fun DistributionListPrivacyMode.toBackupPrivacyMode(): BackupDistributio
 
 private fun BackupDistributionList.PrivacyMode.toLocalPrivacyMode(): DistributionListPrivacyMode {
   return when (this) {
+    BackupDistributionList.PrivacyMode.UNKNOWN -> DistributionListPrivacyMode.ALL
     BackupDistributionList.PrivacyMode.ONLY_WITH -> DistributionListPrivacyMode.ONLY_WITH
     BackupDistributionList.PrivacyMode.ALL -> DistributionListPrivacyMode.ALL
     BackupDistributionList.PrivacyMode.ALL_EXCEPT -> DistributionListPrivacyMode.ALL_EXCEPT

app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/MessageTableBackupExtensions.kt

@@ -40,9 +40,9 @@ fun MessageTable.getMessagesForBackup(): ChatItemExportIterator {
       MessageTable.QUOTE_TYPE,
       MessageTable.ORIGINAL_MESSAGE_ID,
       MessageTable.LATEST_REVISION_ID,
-      MessageTable.DELIVERY_RECEIPT_COUNT,
+      MessageTable.HAS_DELIVERY_RECEIPT,
-      MessageTable.READ_RECEIPT_COUNT,
+      MessageTable.HAS_READ_RECEIPT,
-      MessageTable.VIEWED_RECEIPT_COUNT,
+      MessageTable.VIEWED_COLUMN,
       MessageTable.RECEIPT_TIMESTAMP,
       MessageTable.READ,
       MessageTable.NETWORK_FAILURES,

app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/RecipientTableBackupExtensions.kt

@@ -127,9 +127,7 @@ fun RecipientTable.restoreRecipientFromBackup(recipient: BackupRecipient, backup
 /**
  * Given [AccountData], this will insert the necessary data for the local user into the [RecipientTable].
  */
-fun RecipientTable.restoreSelfFromBackup(accountData: AccountData) {
+fun RecipientTable.restoreSelfFromBackup(accountData: AccountData, selfId: RecipientId) {
-  val self = Recipient.trustedPush(ACI.parseOrThrow(accountData.aci.toByteArray()), PNI.parseOrNull(accountData.pni.toByteArray()), accountData.e164.toString())
-
   val values = ContentValues().apply {
     put(RecipientTable.PROFILE_GIVEN_NAME, accountData.givenName.nullIfBlank())
     put(RecipientTable.PROFILE_FAMILY_NAME, accountData.familyName.nullIfBlank())
@@ -152,7 +150,7 @@ fun RecipientTable.restoreSelfFromBackup(accountData: AccountData) {
   writableDatabase
     .update(RecipientTable.TABLE_NAME)
     .values(values)
-    .where("${RecipientTable.ID} = ?", self.id)
+    .where("${RecipientTable.ID} = ?", selfId)
     .run()
 }
 
@@ -181,7 +179,7 @@ private fun RecipientTable.restoreContactFromBackup(contact: Contact): Recipient
       RecipientTable.HIDDEN to contact.hidden,
       RecipientTable.PROFILE_FAMILY_NAME to contact.profileFamilyName.nullIfBlank(),
       RecipientTable.PROFILE_GIVEN_NAME to contact.profileGivenName.nullIfBlank(),
-      RecipientTable.PROFILE_JOINED_NAME to contact.profileJoinedName.nullIfBlank(),
+      RecipientTable.PROFILE_JOINED_NAME to ProfileName.fromParts(contact.profileGivenName.nullIfBlank(), contact.profileFamilyName.nullIfBlank()).toString().nullIfBlank(),
       RecipientTable.PROFILE_KEY to if (profileKey == null) null else Base64.encodeWithPadding(profileKey),
       RecipientTable.PROFILE_SHARING to contact.profileSharing.toInt(),
       RecipientTable.REGISTERED to contact.registered.toLocalRegisteredState().id,
@@ -205,12 +203,12 @@ private fun Contact.toLocalExtras(): RecipientExtras {
  * Provides a nice iterable interface over a [RecipientTable] cursor, converting rows to [BackupRecipient]s.
  * Important: Because this is backed by a cursor, you must close it. It's recommended to use `.use()` or try-with-resources.
  */
-class BackupContactIterator(private val cursor: Cursor, private val selfId: Long) : Iterator<BackupRecipient>, Closeable {
+class BackupContactIterator(private val cursor: Cursor, private val selfId: Long) : Iterator<BackupRecipient?>, Closeable {
   override fun hasNext(): Boolean {
     return cursor.count > 0 && !cursor.isLast
   }
 
-  override fun next(): BackupRecipient {
+  override fun next(): BackupRecipient? {
     if (!cursor.moveToNext()) {
       throw NoSuchElementException()
     }
@@ -225,10 +223,15 @@ class BackupContactIterator(private val cursor: Cursor, private val selfId: Long
 
     val aci = ACI.parseOrNull(cursor.requireString(RecipientTable.ACI_COLUMN))
     val pni = PNI.parseOrNull(cursor.requireString(RecipientTable.PNI_COLUMN))
+    val e164 = cursor.requireString(RecipientTable.E164)?.e164ToLong()
     val registeredState = RecipientTable.RegisteredState.fromId(cursor.requireInt(RecipientTable.REGISTERED))
     val profileKey = cursor.requireString(RecipientTable.PROFILE_KEY)
     val extras = RecipientTableCursorUtil.getExtras(cursor)
 
+    if (aci == null && pni == null && e164 == null) {
+      return null
+    }
+
     return BackupRecipient(
       id = id,
       contact = Contact(
@@ -244,7 +247,6 @@ class BackupContactIterator(private val cursor: Cursor, private val selfId: Long
         profileSharing = cursor.requireBoolean(RecipientTable.PROFILE_SHARING),
         profileGivenName = cursor.requireString(RecipientTable.PROFILE_GIVEN_NAME).nullIfBlank(),
         profileFamilyName = cursor.requireString(RecipientTable.PROFILE_FAMILY_NAME).nullIfBlank(),
-        profileJoinedName = cursor.requireString(RecipientTable.PROFILE_JOINED_NAME).nullIfBlank(),
         hideStory = extras?.hideStory() ?: false
       )
     )

app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/ThreadTableBackupExtensions.kt

@@ -7,13 +7,20 @@ package org.thoughtcrime.securesms.backup.v2.database
 
 import android.database.Cursor
 import org.signal.core.util.SqlUtil
+import org.signal.core.util.insertInto
+import org.signal.core.util.logging.Log
 import org.signal.core.util.requireBoolean
+import org.signal.core.util.requireInt
 import org.signal.core.util.requireLong
 import org.signal.core.util.select
+import org.signal.core.util.toInt
 import org.thoughtcrime.securesms.backup.v2.proto.Chat
 import org.thoughtcrime.securesms.database.ThreadTable
+import org.thoughtcrime.securesms.recipients.RecipientId
 import java.io.Closeable
 
+private val TAG = Log.tag(ThreadTable::class.java)
+
 fun ThreadTable.getThreadsForBackup(): ChatIterator {
   val cursor = readableDatabase
     .select(
@@ -35,6 +42,17 @@ fun ThreadTable.clearAllDataForBackupRestore() {
   clearCache()
 }
 
+fun ThreadTable.restoreFromBackup(chat: Chat, recipientId: RecipientId): Long? {
+  return writableDatabase
+    .insertInto(ThreadTable.TABLE_NAME)
+    .values(
+      ThreadTable.RECIPIENT_ID to recipientId.serialize(),
+      ThreadTable.PINNED to chat.pinnedOrder,
+      ThreadTable.ARCHIVED to chat.archived.toInt()
+    )
+    .run()
+}
+
 class ChatIterator(private val cursor: Cursor) : Iterator<Chat>, Closeable {
   override fun hasNext(): Boolean {
     return cursor.count > 0 && !cursor.isLast
@@ -49,8 +67,8 @@ class ChatIterator(private val cursor: Cursor) : Iterator<Chat>, Closeable {
       id = cursor.requireLong(ThreadTable.ID),
       recipientId = cursor.requireLong(ThreadTable.RECIPIENT_ID),
       archived = cursor.requireBoolean(ThreadTable.ARCHIVED),
-      pinned = cursor.requireBoolean(ThreadTable.PINNED),
+      pinnedOrder = cursor.requireInt(ThreadTable.PINNED),
-      expirationTimer = cursor.requireLong(ThreadTable.EXPIRES_IN)
+      expirationTimerMs = cursor.requireLong(ThreadTable.EXPIRES_IN)
     )
   }
 

app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/AccountDataProcessor.kt

@@ -19,6 +19,7 @@ import org.thoughtcrime.securesms.jobs.RetrieveProfileAvatarJob
 import org.thoughtcrime.securesms.keyvalue.PhoneNumberPrivacyValues
 import org.thoughtcrime.securesms.keyvalue.SignalStore
 import org.thoughtcrime.securesms.recipients.Recipient
+import org.thoughtcrime.securesms.recipients.RecipientId
 import org.thoughtcrime.securesms.subscription.Subscriber
 import org.thoughtcrime.securesms.util.ProfileUtil
 import org.thoughtcrime.securesms.util.TextSecurePreferences
@@ -35,21 +36,11 @@ object AccountDataProcessor {
     val self = Recipient.self().fresh()
     val record = recipients.getRecordForSync(self.id)
 
-    val pniIdentityKey = SignalStore.account().pniIdentityKey
-    val aciIdentityKey = SignalStore.account().aciIdentityKey
-
     val subscriber: Subscriber? = SignalStore.donationsValues().getSubscriber()
 
     emitter.emit(
       Frame(
         account = AccountData(
-          aci = SignalStore.account().aci!!.toByteString(),
-          pni = SignalStore.account().pni!!.toByteString(),
-          e164 = SignalStore.account().e164!!.toLong(),
-          pniIdentityPrivateKey = pniIdentityKey.privateKey.serialize().toByteString(),
-          pniIdentityPublicKey = pniIdentityKey.publicKey.serialize().toByteString(),
-          aciIdentityPrivateKey = aciIdentityKey.privateKey.serialize().toByteString(),
-          aciIdentityPublicKey = aciIdentityKey.publicKey.serialize().toByteString(),
           profileKey = self.profileKey?.toByteString() ?: EMPTY,
           givenName = self.profileName.givenName,
           familyName = self.profileName.familyName,
@@ -60,14 +51,12 @@ object AccountDataProcessor {
           subscriberCurrencyCode = subscriber?.currencyCode ?: defaultAccountRecord.subscriberCurrencyCode,
           accountSettings = AccountData.AccountSettings(
             storyViewReceiptsEnabled = SignalStore.storyValues().viewedReceiptsEnabled,
-            hasReadOnboardingStory = SignalStore.storyValues().userHasViewedOnboardingStory || SignalStore.storyValues().userHasReadOnboardingStory,
-            noteToSelfArchived = record != null && record.syncExtras.isArchived,
             noteToSelfMarkedUnread = record != null && record.syncExtras.isForcedUnread,
             typingIndicators = TextSecurePreferences.isTypingIndicatorsEnabled(context),
             readReceipts = TextSecurePreferences.isReadReceiptsEnabled(context),
             sealedSenderIndicators = TextSecurePreferences.isShowUnidentifiedDeliveryIndicatorsEnabled(context),
             linkPreviews = SignalStore.settings().isLinkPreviewsEnabled,
-            unlistedPhoneNumber = SignalStore.phoneNumberPrivacy().phoneNumberListingMode.isUnlisted,
+            notDiscoverableByPhoneNumber = SignalStore.phoneNumberPrivacy().phoneNumberListingMode.isUnlisted,
             phoneNumberSharingMode = SignalStore.phoneNumberPrivacy().phoneNumberSharingMode.toBackupPhoneNumberSharingMode(),
             preferContactAvatars = SignalStore.settings().isPreferSystemContactPhotos,
             universalExpireTimer = SignalStore.settings().universalExpireTimer,
@@ -84,16 +73,12 @@ object AccountDataProcessor {
     )
   }
 
-  fun import(accountData: AccountData) {
+  fun import(accountData: AccountData, selfId: RecipientId) {
-    SignalStore.account().restoreAciIdentityKeyFromBackup(accountData.aciIdentityPublicKey.toByteArray(), accountData.aciIdentityPrivateKey.toByteArray())
+    recipients.restoreSelfFromBackup(accountData, selfId)
-    SignalStore.account().restorePniIdentityKeyFromBackup(accountData.pniIdentityPublicKey.toByteArray(), accountData.pniIdentityPrivateKey.toByteArray())
-
-    recipients.restoreSelfFromBackup(accountData)
 
     SignalStore.account().setRegistered(true)
 
     val context = ApplicationDependencies.getApplication()
-
     val settings = accountData.accountSettings
 
     if (settings != null) {
@@ -101,7 +86,7 @@ object AccountDataProcessor {
       TextSecurePreferences.setTypingIndicatorsEnabled(context, settings.typingIndicators)
       TextSecurePreferences.setShowUnidentifiedDeliveryIndicatorsEnabled(context, settings.sealedSenderIndicators)
       SignalStore.settings().isLinkPreviewsEnabled = settings.linkPreviews
-      SignalStore.phoneNumberPrivacy().phoneNumberListingMode = if (settings.unlistedPhoneNumber) PhoneNumberPrivacyValues.PhoneNumberListingMode.UNLISTED else PhoneNumberPrivacyValues.PhoneNumberListingMode.LISTED
+      SignalStore.phoneNumberPrivacy().phoneNumberListingMode = if (settings.notDiscoverableByPhoneNumber) PhoneNumberPrivacyValues.PhoneNumberListingMode.UNLISTED else PhoneNumberPrivacyValues.PhoneNumberListingMode.LISTED
       SignalStore.phoneNumberPrivacy().phoneNumberSharingMode = settings.phoneNumberSharingMode.toLocalPhoneNumberMode()
       SignalStore.settings().isPreferSystemContactPhotos = settings.preferContactAvatars
       SignalStore.settings().universalExpireTimer = settings.universalExpireTimer
@@ -111,7 +96,6 @@ object AccountDataProcessor {
       SignalStore.storyValues().userHasBeenNotifiedAboutStories = settings.hasSetMyStoriesPrivacy
       SignalStore.storyValues().userHasViewedOnboardingStory = settings.hasViewedOnboardingStory
       SignalStore.storyValues().isFeatureDisabled = settings.storiesDisabled
-      SignalStore.storyValues().userHasReadOnboardingStory = settings.hasReadOnboardingStory
       SignalStore.storyValues().userHasSeenGroupStoryEducationSheet = settings.hasSeenGroupStoryEducationSheet
       SignalStore.storyValues().viewedReceiptsEnabled = settings.storyViewReceiptsEnabled ?: settings.readReceipts
 

app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/ChatBackupProcessor.kt

@@ -8,12 +8,12 @@ package org.thoughtcrime.securesms.backup.v2.processor
 import org.signal.core.util.logging.Log
 import org.thoughtcrime.securesms.backup.v2.BackupState
 import org.thoughtcrime.securesms.backup.v2.database.getThreadsForBackup
+import org.thoughtcrime.securesms.backup.v2.database.restoreFromBackup
 import org.thoughtcrime.securesms.backup.v2.proto.Chat
 import org.thoughtcrime.securesms.backup.v2.proto.Frame
 import org.thoughtcrime.securesms.backup.v2.stream.BackupFrameEmitter
 import org.thoughtcrime.securesms.database.SignalDatabase
 import org.thoughtcrime.securesms.recipients.RecipientId
-import java.util.Collections
 
 object ChatBackupProcessor {
   val TAG = Log.tag(ChatBackupProcessor::class.java)
@@ -27,26 +27,18 @@ object ChatBackupProcessor {
   }
 
   fun import(chat: Chat, backupState: BackupState) {
-    // TODO Perf can be improved here by doing a single insert instead of insert + multiple updates
-
     val recipientId: RecipientId? = backupState.backupToLocalRecipientId[chat.recipientId]
-
+    if (recipientId == null) {
-    if (recipientId != null) {
+      Log.w(TAG, "Missing recipient for chat ${chat.id}")
-      val threadId = SignalDatabase.threads.getOrCreateThreadIdFor(org.thoughtcrime.securesms.recipients.Recipient.resolved(recipientId))
+      return
-
-      if (chat.archived) {
-        SignalDatabase.threads.archiveConversation(threadId)
-      }
-
-      if (chat.pinned) {
-        SignalDatabase.threads.pinConversations(Collections.singleton(threadId))
     }
 
+    SignalDatabase.threads.restoreFromBackup(chat, recipientId)?.let { threadId ->
       backupState.chatIdToLocalRecipientId[chat.id] = recipientId
       backupState.chatIdToLocalThreadId[chat.id] = threadId
       backupState.chatIdToBackupRecipientId[chat.id] = chat.recipientId
-    } else {
+    }
-      Log.w(TAG, "Recipient doesnt exist with id $recipientId")
+
-    }
+    // TODO there's several fields in the chat that actually need to be restored on the recipient table
   }
 }

app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/RecipientBackupProcessor.kt

@@ -27,9 +27,11 @@ object RecipientBackupProcessor {
 
     SignalDatabase.recipients.getContactsForBackup(selfId).use { reader ->
       for (backupRecipient in reader) {
+        if (backupRecipient != null) {
           emitter.emit(Frame(recipient = backupRecipient))
         }
       }
+    }
 
     SignalDatabase.recipients.getGroupsForBackup().use { reader ->
       for (backupRecipient in reader) {

app/src/main/java/org/thoughtcrime/securesms/backup/v2/stream/BackupEncryptedOutputStream.kt

@@ -0,0 +1,75 @@
+/*
+ * Copyright 2023 Signal Messenger, LLC
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+package org.thoughtcrime.securesms.backup.v2.stream
+
+import org.signal.libsignal.protocol.kdf.HKDF
+import java.io.FilterOutputStream
+import java.io.OutputStream
+import javax.crypto.Cipher
+import javax.crypto.Mac
+import javax.crypto.spec.IvParameterSpec
+import javax.crypto.spec.SecretKeySpec
+
+class BackupEncryptedOutputStream(key: ByteArray, backupId: ByteArray, wrapped: OutputStream) : FilterOutputStream(wrapped) {
+
+  val cipher: Cipher = Cipher.getInstance("AES/CBC/PKCS5Padding")
+  val mac: Mac = Mac.getInstance("HmacSHA256")
+
+  var finalMac: ByteArray? = null
+
+  init {
+    if (key.size != 32) {
+      throw IllegalArgumentException("Key must be 32 bytes!")
+    }
+
+    if (backupId.size != 16) {
+      throw IllegalArgumentException("BackupId must be 32 bytes!")
+    }
+
+    val extendedKey = HKDF.deriveSecrets(key, backupId, "20231003_Signal_Backups_EncryptMessageBackup".toByteArray(), 80)
+    val macKey = extendedKey.copyOfRange(0, 32)
+    val cipherKey = extendedKey.copyOfRange(32, 64)
+    val iv = extendedKey.copyOfRange(64, 80)
+
+    cipher.init(Cipher.ENCRYPT_MODE, SecretKeySpec(cipherKey, "AES"), IvParameterSpec(iv))
+    mac.init(SecretKeySpec(macKey, "HmacSHA256"))
+  }
+
+  override fun write(b: Int) {
+    throw UnsupportedOperationException()
+  }
+
+  override fun write(data: ByteArray) {
+    write(data, 0, data.size)
+  }
+
+  override fun write(data: ByteArray, off: Int, len: Int) {
+    cipher.update(data, off, len)?.let { ciphertext ->
+      mac.update(ciphertext)
+      super.write(ciphertext)
+    }
+  }
+
+  override fun flush() {
+    cipher.doFinal()?.let { ciphertext ->
+      mac.update(ciphertext)
+      super.write(ciphertext)
+    }
+
+    finalMac = mac.doFinal()
+
+    super.flush()
+  }
+
+  override fun close() {
+    flush()
+    super.close()
+  }
+
+  fun getMac(): ByteArray {
+    return finalMac ?: throw IllegalStateException("Mac not yet available! You must call flush() before asking for the mac.")
+  }
+}

app/src/main/java/org/thoughtcrime/securesms/backup/v2/stream/BackupExportWriter.kt

@@ -7,6 +7,6 @@ package org.thoughtcrime.securesms.backup.v2.stream
 
 import org.thoughtcrime.securesms.backup.v2.proto.Frame
 
-interface BackupExportStream {
+interface BackupExportWriter : AutoCloseable {
   fun write(frame: Frame)
 }

app/src/main/java/org/thoughtcrime/securesms/backup/v2/stream/EncryptedBackupReader.kt

@@ -0,0 +1,112 @@
+/*
+ * Copyright 2023 Signal Messenger, LLC
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+package org.thoughtcrime.securesms.backup.v2.stream
+
+import org.signal.core.util.readFully
+import org.signal.core.util.readNBytesOrThrow
+import org.signal.core.util.readVarInt32
+import org.signal.core.util.stream.MacInputStream
+import org.signal.core.util.stream.TruncatingInputStream
+import org.thoughtcrime.securesms.backup.v2.proto.Frame
+import org.whispersystems.signalservice.api.backup.BackupKey
+import org.whispersystems.signalservice.api.push.ServiceId.ACI
+import java.io.EOFException
+import java.io.IOException
+import java.io.InputStream
+import java.util.zip.GZIPInputStream
+import javax.crypto.Cipher
+import javax.crypto.CipherInputStream
+import javax.crypto.Mac
+import javax.crypto.spec.IvParameterSpec
+import javax.crypto.spec.SecretKeySpec
+
+/**
+ * Provides the ability to read backup frames in a streaming fashion from a target [InputStream].
+ * As it's being read, it will be both decrypted and uncompressed. Specifically, the data is decrypted,
+ * that decrypted data is gunzipped, then that data is read as frames.
+ */
+class EncryptedBackupReader(
+  key: BackupKey,
+  aci: ACI,
+  streamLength: Long,
+  dataStream: () -> InputStream
+) : Iterator<Frame>, AutoCloseable {
+
+  var next: Frame? = null
+  val stream: InputStream
+
+  init {
+    val keyMaterial = key.deriveSecrets(aci)
+
+    val cipher = Cipher.getInstance("AES/CBC/PKCS5Padding").apply {
+      init(Cipher.DECRYPT_MODE, SecretKeySpec(keyMaterial.cipherKey, "AES"), IvParameterSpec(keyMaterial.iv))
+    }
+
+    validateMac(keyMaterial.macKey, streamLength, dataStream())
+
+    stream = GZIPInputStream(
+      CipherInputStream(
+        TruncatingInputStream(
+          wrapped = dataStream(),
+          maxBytes = streamLength - MAC_SIZE
+        ),
+        cipher
+      )
+    )
+
+    next = read()
+  }
+
+  override fun hasNext(): Boolean {
+    return next != null
+  }
+
+  override fun next(): Frame {
+    next?.let { out ->
+      next = read()
+      return out
+    } ?: throw NoSuchElementException()
+  }
+
+  private fun read(): Frame? {
+    try {
+      val length = stream.readVarInt32().also { if (it < 0) return null }
+      val frameBytes: ByteArray = stream.readNBytesOrThrow(length)
+
+      return Frame.ADAPTER.decode(frameBytes)
+    } catch (e: EOFException) {
+      return null
+    }
+  }
+
+  override fun close() {
+    stream.close()
+  }
+
+  companion object {
+    const val MAC_SIZE = 32
+
+    fun validateMac(macKey: ByteArray, streamLength: Long, dataStream: InputStream) {
+      val mac = Mac.getInstance("HmacSHA256").apply {
+        init(SecretKeySpec(macKey, "HmacSHA256"))
+      }
+
+      val macStream = MacInputStream(
+        wrapped = TruncatingInputStream(dataStream, maxBytes = streamLength - MAC_SIZE),
+        mac = mac
+      )
+
+      macStream.readFully(false)
+
+      val calculatedMac = macStream.mac.doFinal()
+      val expectedMac = dataStream.readNBytesOrThrow(MAC_SIZE)
+
+      if (!calculatedMac.contentEquals(expectedMac)) {
+        throw IOException("Invalid MAC!")
+      }
+    }
+  }
+}

app/src/main/java/org/thoughtcrime/securesms/backup/v2/stream/EncryptedBackupWriter.kt

@@ -0,0 +1,76 @@
+/*
+ * Copyright 2023 Signal Messenger, LLC
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+package org.thoughtcrime.securesms.backup.v2.stream
+
+import org.signal.core.util.stream.MacOutputStream
+import org.signal.core.util.writeVarInt32
+import org.thoughtcrime.securesms.backup.v2.proto.Frame
+import org.whispersystems.signalservice.api.backup.BackupKey
+import org.whispersystems.signalservice.api.push.ServiceId.ACI
+import java.io.IOException
+import java.io.OutputStream
+import java.util.zip.GZIPOutputStream
+import javax.crypto.Cipher
+import javax.crypto.CipherOutputStream
+import javax.crypto.Mac
+import javax.crypto.spec.IvParameterSpec
+import javax.crypto.spec.SecretKeySpec
+
+/**
+ * Provides the ability to write backup frames in a streaming fashion to a target [OutputStream].
+ * As it's being written, it will be both encrypted and compressed. Specifically, the backup frames
+ * are gzipped, that gzipped data is encrypted, and then an HMAC of the encrypted data is appended
+ * to the end of the [outputStream].
+ */
+class EncryptedBackupWriter(
+  key: BackupKey,
+  aci: ACI,
+  private val outputStream: OutputStream,
+  private val append: (ByteArray) -> Unit
+) : BackupExportWriter {
+
+  private val mainStream: GZIPOutputStream
+  private val macStream: MacOutputStream
+
+  init {
+    val keyMaterial = key.deriveSecrets(aci)
+
+    val cipher = Cipher.getInstance("AES/CBC/PKCS5Padding").apply {
+      init(Cipher.ENCRYPT_MODE, SecretKeySpec(keyMaterial.cipherKey, "AES"), IvParameterSpec(keyMaterial.iv))
+    }
+
+    val mac = Mac.getInstance("HmacSHA256").apply {
+      init(SecretKeySpec(keyMaterial.macKey, "HmacSHA256"))
+    }
+
+    macStream = MacOutputStream(outputStream, mac)
+
+    mainStream = GZIPOutputStream(
+      CipherOutputStream(
+        macStream,
+        cipher
+      )
+    )
+  }
+
+  @Throws(IOException::class)
+  override fun write(frame: Frame) {
+    val frameBytes: ByteArray = frame.encode()
+
+    mainStream.writeVarInt32(frameBytes.size)
+    mainStream.write(frameBytes)
+  }
+
+  @Throws(IOException::class)
+  override fun close() {
+    // We need to close the main stream in order for the gzip and all the cipher operations to fully finish before
+    // we can calculate the MAC. Unfortunately flush()/finish() is not sufficient. So we have to defer to the
+    // caller to append the bytes to the end of the data however they see fit (like appending to a file).
+    mainStream.close()
+    val mac = macStream.mac.doFinal()
+    append(mac)
+  }
+}

app/src/main/java/org/thoughtcrime/securesms/backup/v2/stream/PlainTextBackupReader.kt

@@ -5,8 +5,8 @@
 
 package org.thoughtcrime.securesms.backup.v2.stream
 
-import org.signal.core.util.Conversions
 import org.signal.core.util.readNBytesOrThrow
+import org.signal.core.util.readVarInt32
 import org.thoughtcrime.securesms.backup.v2.proto.Frame
 import java.io.EOFException
 import java.io.InputStream
@@ -14,7 +14,7 @@ import java.io.InputStream
 /**
  * Reads a plaintext backup import stream one frame at a time.
  */
-class PlainTextBackupImportStream(val inputStream: InputStream) : BackupImportStream, Iterator<Frame> {
+class PlainTextBackupReader(val inputStream: InputStream) : Iterator<Frame> {
 
   var next: Frame? = null
 
@@ -33,15 +33,12 @@ class PlainTextBackupImportStream(val inputStream: InputStream) : BackupImportSt
     } ?: throw NoSuchElementException()
   }
 
-  override fun read(): Frame? {
+  private fun read(): Frame? {
     try {
-      val lengthBytes: ByteArray = inputStream.readNBytesOrThrow(4)
+      val length = inputStream.readVarInt32().also { if (it < 0) return null }
-      val length = Conversions.byteArrayToInt(lengthBytes)
-
       val frameBytes: ByteArray = inputStream.readNBytesOrThrow(length)
-      val frame: Frame = Frame.ADAPTER.decode(frameBytes)
 
-      return frame
+      return Frame.ADAPTER.decode(frameBytes)
     } catch (e: EOFException) {
       return null
     }

app/src/main/java/org/thoughtcrime/securesms/backup/v2/stream/PlainTextBackupWriter.kt

@@ -5,7 +5,7 @@
 
 package org.thoughtcrime.securesms.backup.v2.stream
 
-import org.signal.core.util.Conversions
+import org.signal.core.util.writeVarInt32
 import org.thoughtcrime.securesms.backup.v2.proto.Frame
 import java.io.IOException
 import java.io.OutputStream
@@ -13,14 +13,17 @@ import java.io.OutputStream
 /**
  * Writes backup frames to the wrapped stream in plain text. Only for testing!
  */
-class PlainTextBackupExportStream(private val outputStream: OutputStream) : BackupExportStream {
+class PlainTextBackupWriter(private val outputStream: OutputStream) : BackupExportWriter {
 
   @Throws(IOException::class)
   override fun write(frame: Frame) {
     val frameBytes: ByteArray = frame.encode()
-    val lengthBytes: ByteArray = Conversions.intToByteArray(frameBytes.size)
 
-    outputStream.write(lengthBytes)
+    outputStream.writeVarInt32(frameBytes.size)
     outputStream.write(frameBytes)
   }
+
+  override fun close() {
+    outputStream.close()
+  }
 }

app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/backup/InternalBackupPlaygroundFragment.kt

@@ -5,23 +5,70 @@
 
 package org.thoughtcrime.securesms.components.settings.app.internal.backup
 
+import android.app.Activity.RESULT_OK
+import android.content.Intent
+import android.content.res.Configuration
+import android.os.Bundle
+import android.widget.Toast
+import androidx.activity.result.ActivityResultLauncher
+import androidx.activity.result.contract.ActivityResultContracts
 import androidx.compose.foundation.layout.Arrangement
 import androidx.compose.foundation.layout.Column
+import androidx.compose.foundation.layout.Row
+import androidx.compose.foundation.layout.Spacer
+import androidx.compose.foundation.layout.fillMaxSize
+import androidx.compose.foundation.layout.height
+import androidx.compose.foundation.layout.padding
+import androidx.compose.foundation.layout.width
+import androidx.compose.material3.MaterialTheme
 import androidx.compose.material3.Surface
+import androidx.compose.material3.Switch
 import androidx.compose.material3.Text
 import androidx.compose.runtime.Composable
 import androidx.compose.runtime.getValue
 import androidx.compose.ui.Alignment
+import androidx.compose.ui.Modifier
+import androidx.compose.ui.text.style.TextAlign
 import androidx.compose.ui.tooling.preview.Preview
+import androidx.compose.ui.unit.dp
 import androidx.fragment.app.viewModels
 import org.signal.core.ui.Buttons
+import org.signal.core.ui.theme.SignalTheme
+import org.signal.core.util.getLength
 import org.thoughtcrime.securesms.components.settings.app.internal.backup.InternalBackupPlaygroundViewModel.BackupState
 import org.thoughtcrime.securesms.components.settings.app.internal.backup.InternalBackupPlaygroundViewModel.ScreenState
 import org.thoughtcrime.securesms.compose.ComposeFragment
 
 class InternalBackupPlaygroundFragment : ComposeFragment() {
 
-  val viewModel: InternalBackupPlaygroundViewModel by viewModels()
+  private val viewModel: InternalBackupPlaygroundViewModel by viewModels()
+  private lateinit var exportFileLauncher: ActivityResultLauncher<Intent>
+  private lateinit var importFileLauncher: ActivityResultLauncher<Intent>
+
+  override fun onCreate(savedInstanceState: Bundle?) {
+    super.onCreate(savedInstanceState)
+
+    exportFileLauncher = registerForActivityResult(ActivityResultContracts.StartActivityForResult()) { result ->
+      if (result.resultCode == RESULT_OK) {
+        result.data?.data?.let { uri ->
+          requireContext().contentResolver.openOutputStream(uri)?.use { outputStream ->
+            outputStream.write(viewModel.backupData!!)
+            Toast.makeText(requireContext(), "Saved successfully", Toast.LENGTH_SHORT).show()
+          } ?: Toast.makeText(requireContext(), "Failed to open output stream", Toast.LENGTH_SHORT).show()
+        } ?: Toast.makeText(requireContext(), "No URI selected", Toast.LENGTH_SHORT).show()
+      }
+    }
+
+    importFileLauncher = registerForActivityResult(ActivityResultContracts.StartActivityForResult()) { result ->
+      if (result.resultCode == RESULT_OK) {
+        result.data?.data?.let { uri ->
+          requireContext().contentResolver.getLength(uri)?.let { length ->
+            viewModel.import(length) { requireContext().contentResolver.openInputStream(uri)!! }
+          }
+        } ?: Toast.makeText(requireContext(), "No URI selected", Toast.LENGTH_SHORT).show()
+      }
+    }
+  }
 
   @Composable
   override fun FragmentContent() {
@@ -30,22 +77,65 @@ class InternalBackupPlaygroundFragment : ComposeFragment() {
     Screen(
       state = state,
       onExportClicked = { viewModel.export() },
-      onImportClicked = { viewModel.import() }
+      onImportMemoryClicked = { viewModel.import() },
+      onImportFileClicked = {
+        val intent = Intent().apply {
+          action = Intent.ACTION_GET_CONTENT
+          type = "application/octet-stream"
+          addCategory(Intent.CATEGORY_OPENABLE)
+        }
+
+        importFileLauncher.launch(intent)
+      },
+      onPlaintextClicked = { viewModel.onPlaintextToggled() },
+      onSaveToDiskClicked = {
+        val intent = Intent().apply {
+          action = Intent.ACTION_CREATE_DOCUMENT
+          type = "application/octet-stream"
+          addCategory(Intent.CATEGORY_OPENABLE)
+          putExtra(Intent.EXTRA_TITLE, "backup-${if (state.plaintext) "plaintext" else "encrypted"}-${System.currentTimeMillis()}.bin")
+        }
+
+        exportFileLauncher.launch(intent)
+      }
     )
   }
+
+  override fun onActivityResult(requestCode: Int, resultCode: Int, data: Intent?) {
+    super.onActivityResult(requestCode, resultCode, data)
+  }
 }
 
 @Composable
 fun Screen(
   state: ScreenState,
   onExportClicked: () -> Unit = {},
-  onImportClicked: () -> Unit = {}
+  onImportMemoryClicked: () -> Unit = {},
+  onImportFileClicked: () -> Unit = {},
+  onPlaintextClicked: () -> Unit = {},
+  onSaveToDiskClicked: () -> Unit = {}
 ) {
   Surface {
     Column(
       horizontalAlignment = Alignment.CenterHorizontally,
-      verticalArrangement = Arrangement.Center
+      verticalArrangement = Arrangement.Center,
+      modifier = Modifier
+        .fillMaxSize()
+        .padding(16.dp)
     ) {
+      Row(
+        verticalAlignment = Alignment.CenterVertically
+      ) {
+        StateLabel(text = "Plaintext?")
+        Spacer(modifier = Modifier.width(8.dp))
+        Switch(
+          checked = state.plaintext,
+          onCheckedChange = { onPlaintextClicked() }
+        )
+      }
+
+      Spacer(modifier = Modifier.height(8.dp))
+
       Buttons.LargePrimary(
         onClick = onExportClicked,
         enabled = !state.backupState.inProgress
@@ -53,17 +143,92 @@ fun Screen(
         Text("Export")
       }
       Buttons.LargeTonal(
-        onClick = onImportClicked,
+        onClick = onImportMemoryClicked,
         enabled = state.backupState == BackupState.EXPORT_DONE
       ) {
-        Text("Import")
+        Text("Import from memory")
+      }
+      Buttons.LargeTonal(
+        onClick = onImportFileClicked
+      ) {
+        Text("Import from file")
+      }
+
+      Spacer(modifier = Modifier.height(16.dp))
+
+      when (state.backupState) {
+        BackupState.NONE -> {
+          StateLabel("")
+        }
+        BackupState.EXPORT_IN_PROGRESS -> {
+          StateLabel("Export in progress...")
+        }
+        BackupState.EXPORT_DONE -> {
+          StateLabel("Export complete. Sitting in memory. You can click 'Import' to import that data, or you can save it to a file.")
+
+          Spacer(modifier = Modifier.height(8.dp))
+
+          Buttons.MediumTonal(onClick = onSaveToDiskClicked) {
+            Text("Save to file")
+          }
+        }
+        BackupState.IMPORT_IN_PROGRESS -> {
+          StateLabel("Import in progress...")
+        }
       }
     }
   }
 }
 
-@Preview
+@Composable
+private fun StateLabel(text: String) {
+  Text(
+    text = text,
+    style = MaterialTheme.typography.labelSmall,
+    textAlign = TextAlign.Center
+  )
+}
+
+@Preview(name = "Light Theme", group = "screen", uiMode = Configuration.UI_MODE_NIGHT_NO)
+@Preview(name = "Dark Theme", group = "screen", uiMode = Configuration.UI_MODE_NIGHT_YES)
 @Composable
 fun PreviewScreen() {
-  Screen(state = ScreenState(backupState = BackupState.NONE))
+  SignalTheme {
+    Surface {
+      Screen(state = ScreenState(backupState = BackupState.NONE, plaintext = false))
+    }
+  }
+}
+
+@Preview(name = "Light Theme", group = "screen", uiMode = Configuration.UI_MODE_NIGHT_NO)
+@Preview(name = "Dark Theme", group = "screen", uiMode = Configuration.UI_MODE_NIGHT_YES)
+@Composable
+fun PreviewScreenExportInProgress() {
+  SignalTheme {
+    Surface {
+      Screen(state = ScreenState(backupState = BackupState.EXPORT_IN_PROGRESS, plaintext = false))
+    }
+  }
+}
+
+@Preview(name = "Light Theme", group = "screen", uiMode = Configuration.UI_MODE_NIGHT_NO)
+@Preview(name = "Dark Theme", group = "screen", uiMode = Configuration.UI_MODE_NIGHT_YES)
+@Composable
+fun PreviewScreenExportDone() {
+  SignalTheme {
+    Surface {
+      Screen(state = ScreenState(backupState = BackupState.EXPORT_DONE, plaintext = false))
+    }
+  }
+}
+
+@Preview(name = "Light Theme", group = "screen", uiMode = Configuration.UI_MODE_NIGHT_NO)
+@Preview(name = "Dark Theme", group = "screen", uiMode = Configuration.UI_MODE_NIGHT_YES)
+@Composable
+fun PreviewScreenImportInProgress() {
+  SignalTheme {
+    Surface {
+      Screen(state = ScreenState(backupState = BackupState.IMPORT_IN_PROGRESS, plaintext = false))
+    }
+  }
 }

app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/backup/InternalBackupPlaygroundViewModel.kt

@@ -14,7 +14,11 @@ import io.reactivex.rxjava3.core.Single
 import io.reactivex.rxjava3.disposables.CompositeDisposable
 import io.reactivex.rxjava3.kotlin.plusAssign
 import io.reactivex.rxjava3.schedulers.Schedulers
+import org.signal.libsignal.zkgroup.profiles.ProfileKey
 import org.thoughtcrime.securesms.backup.v2.BackupRepository
+import org.thoughtcrime.securesms.recipients.Recipient
+import java.io.ByteArrayInputStream
+import java.io.InputStream
 
 class InternalBackupPlaygroundViewModel : ViewModel() {
 
@@ -22,13 +26,14 @@ class InternalBackupPlaygroundViewModel : ViewModel() {
 
   val disposables = CompositeDisposable()
 
-  private val _state: MutableState<ScreenState> = mutableStateOf(ScreenState(backupState = BackupState.NONE))
+  private val _state: MutableState<ScreenState> = mutableStateOf(ScreenState(backupState = BackupState.NONE, plaintext = false))
   val state: State<ScreenState> = _state
 
   fun export() {
     _state.value = _state.value.copy(backupState = BackupState.EXPORT_IN_PROGRESS)
+    val plaintext = _state.value.plaintext
 
-    disposables += Single.fromCallable { BackupRepository.export() }
+    disposables += Single.fromCallable { BackupRepository.export(plaintext = plaintext) }
       .subscribeOn(Schedulers.io())
       .observeOn(AndroidSchedulers.mainThread())
       .subscribe { data ->
@@ -40,8 +45,12 @@ class InternalBackupPlaygroundViewModel : ViewModel() {
   fun import() {
     backupData?.let {
       _state.value = _state.value.copy(backupState = BackupState.IMPORT_IN_PROGRESS)
+      val plaintext = _state.value.plaintext
 
-      disposables += Single.fromCallable { BackupRepository.import(it) }
+      val self = Recipient.self()
+      val selfData = BackupRepository.SelfData(self.aci.get(), self.pni.get(), self.e164.get(), ProfileKey(self.profileKey))
+
+      disposables += Single.fromCallable { BackupRepository.import(it.size.toLong(), { ByteArrayInputStream(it) }, selfData, plaintext = plaintext) }
         .subscribeOn(Schedulers.io())
         .observeOn(AndroidSchedulers.mainThread())
         .subscribe { nothing ->
@@ -51,12 +60,33 @@ class InternalBackupPlaygroundViewModel : ViewModel() {
     }
   }
 
+  fun import(length: Long, inputStreamFactory: () -> InputStream) {
+    _state.value = _state.value.copy(backupState = BackupState.IMPORT_IN_PROGRESS)
+    val plaintext = _state.value.plaintext
+
+    val self = Recipient.self()
+    val selfData = BackupRepository.SelfData(self.aci.get(), self.pni.get(), self.e164.get(), ProfileKey(self.profileKey))
+
+    disposables += Single.fromCallable { BackupRepository.import(length, inputStreamFactory, selfData, plaintext = plaintext) }
+      .subscribeOn(Schedulers.io())
+      .observeOn(AndroidSchedulers.mainThread())
+      .subscribe { nothing ->
+        backupData = null
+        _state.value = _state.value.copy(backupState = BackupState.NONE)
+      }
+  }
+
+  fun onPlaintextToggled() {
+    _state.value = _state.value.copy(plaintext = !_state.value.plaintext)
+  }
+
   override fun onCleared() {
     disposables.clear()
   }
 
   data class ScreenState(
-    val backupState: BackupState
+    val backupState: BackupState,
+    val plaintext: Boolean
   )
 
   enum class BackupState(val inProgress: Boolean = false) {

app/src/main/java/org/thoughtcrime/securesms/database/MessageTable.kt

@@ -207,6 +207,9 @@ open class MessageTable(context: Context?, databaseHelper: SignalDatabase) : Dat
     const val ORIGINAL_MESSAGE_ID = "original_message_id"
     const val REVISION_NUMBER = "revision_number"
 
+    const val QUOTE_NOT_PRESENT_ID = 0L
+    const val QUOTE_TARGET_MISSING_ID = -1L
+
     const val CREATE_TABLE = """
       CREATE TABLE $TABLE_NAME (
         $ID INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -2316,7 +2319,7 @@ open class MessageTable(context: Context?, databaseHelper: SignalDatabase) : Dat
       val quoteAttachments: List<Attachment> = associatedAttachments.filter { it.isQuote }.toList()
       val quoteMentions: List<Mention> = parseQuoteMentions(cursor)
       val quoteBodyRanges: BodyRangeList? = parseQuoteBodyRanges(cursor)
-      val quote: QuoteModel? = if (quoteId > 0 && quoteAuthor > 0 && (!TextUtils.isEmpty(quoteText) || quoteAttachments.isNotEmpty())) {
+      val quote: QuoteModel? = if (quoteId != QUOTE_NOT_PRESENT_ID && quoteAuthor > 0 && (!TextUtils.isEmpty(quoteText) || quoteAttachments.isNotEmpty())) {
         QuoteModel(quoteId, RecipientId.from(quoteAuthor), quoteText ?: "", quoteMissing, quoteAttachments, quoteMentions, QuoteModel.Type.fromCode(quoteType), quoteBodyRanges)
       } else {
         null
@@ -5119,7 +5122,7 @@ open class MessageTable(context: Context?, databaseHelper: SignalDatabase) : Dat
       val quoteAttachments: List<Attachment> = attachments.filter { it.isQuote }
       val quoteDeck = SlideDeck(quoteAttachments)
 
-      return if (quoteId > 0 && quoteAuthor > 0) {
+      return if (quoteId != QUOTE_NOT_PRESENT_ID && quoteAuthor > 0) {
         if (quoteText != null && (quoteMentions.isNotEmpty() || bodyRanges != null)) {
           val updated: UpdatedBodyAndMentions = MentionUtil.updateBodyAndMentionsWithDisplayNames(context, quoteText, quoteMentions)
           val styledText = SpannableString(updated.body)

app/src/main/java/org/thoughtcrime/securesms/database/RecipientTable.kt

@@ -688,7 +688,7 @@ open class RecipientTable(context: Context, databaseHelper: SignalDatabase) : Da
 
     val foundRecords = queries.flatMap { query ->
       readableDatabase.query(TABLE_NAME, null, query.where, query.whereArgs, null, null, null).readToList { cursor ->
-        getRecord(context, cursor)
+        RecipientTableCursorUtil.getRecord(context, cursor)
       }
     }
 

app/src/main/java/org/thoughtcrime/securesms/util/FeatureFlags.java

@@ -200,7 +200,6 @@ public final class FeatureFlags {
   @SuppressWarnings("MismatchedQueryAndUpdateOfCollection")
   @VisibleForTesting
   static final Map<String, Object> FORCED_VALUES = new HashMap<String, Object>() {{
-    put(INTERNAL_USER, true);
   }};
 
   /**

app/src/main/protowire/Backup.proto

@@ -6,7 +6,8 @@ option java_package = "org.thoughtcrime.securesms.backup.v2.proto";
 
 message BackupInfo {
   uint64 version = 1;
-  uint64 backupTime = 2;
+  uint64 backupTimeMs = 2;
+  bytes  iv = 3;
 }
 
 message Frame {
@@ -45,46 +46,36 @@ message AccountData {
   }
 
   message AccountSettings {
-    bool noteToSelfArchived = 1;
+    bool readReceipts = 1;
-    bool readReceipts = 2;
+    bool sealedSenderIndicators = 2;
-    bool sealedSenderIndicators = 3;
+    bool typingIndicators = 3;
-    bool typingIndicators = 4;
+    bool noteToSelfMarkedUnread = 4;
-    bool proxiedLinkPreviews = 5;
+    bool linkPreviews = 5;
-    bool noteToSelfMarkedUnread = 6;
+    bool notDiscoverableByPhoneNumber = 6;
-    bool linkPreviews = 7;
+    bool preferContactAvatars = 7;
-    bool unlistedPhoneNumber = 8;
+    uint32 universalExpireTimer = 8;
-    bool preferContactAvatars = 9;
+    repeated string preferredReactionEmoji = 9;
-    uint32 universalExpireTimer = 10;
+    bool displayBadgesOnProfile = 10;
-    repeated string preferredReactionEmoji = 11;
+    bool keepMutedChatsArchived = 11;
-    bool displayBadgesOnProfile = 12;
+    bool hasSetMyStoriesPrivacy = 12;
-    bool keepMutedChatsArchived = 13;
+    bool hasViewedOnboardingStory = 13;
-    bool hasSetMyStoriesPrivacy = 14;
+    bool storiesDisabled = 14;
-    bool hasViewedOnboardingStory = 15;
+    optional bool storyViewReceiptsEnabled = 15;
-    bool storiesDisabled = 16;
+    bool hasSeenGroupStoryEducationSheet = 16;
-    optional bool storyViewReceiptsEnabled = 17;
+    bool hasCompletedUsernameOnboarding = 17;
-    bool hasReadOnboardingStory = 18;
+    PhoneNumberSharingMode phoneNumberSharingMode = 18;
-    bool hasSeenGroupStoryEducationSheet = 19;
-    bool hasCompletedUsernameOnboarding = 20;
-    PhoneNumberSharingMode phoneNumberSharingMode = 21;
   }
 
-  bytes aciIdentityPublicKey = 1;
+  bytes profileKey = 1;
-  bytes aciIdentityPrivateKey = 2;
+  optional string username = 2;
-  bytes pniIdentityPublicKey = 3;
+  UsernameLink usernameLink = 3;
-  bytes pniIdentityPrivateKey = 4;
+  string givenName = 4;
-  bytes profileKey = 5;
+  string familyName = 5;
-  optional string username = 6;
+  string avatarUrlPath = 6;
-  UsernameLink usernameLink = 7;
+  bytes subscriberId = 7;
-  string givenName = 8;
+  string subscriberCurrencyCode = 8;
-  string familyName = 9;
+  bool subscriptionManuallyCancelled = 9;
-  string avatarUrlPath = 10;
+  AccountSettings accountSettings = 10;
-  bytes subscriberId = 11;
-  string subscriberCurrencyCode = 12;
-  bool subscriptionManuallyCancelled = 13;
-  AccountSettings accountSettings = 14;
-  bytes aci = 15;
-  bytes pni = 16;
-  uint64 e164 = 17;
 }
 
 message Recipient {
@@ -94,29 +85,30 @@ message Recipient {
     Group group = 3;
     DistributionList distributionList = 4;
     Self self = 5;
+    ReleaseNotes releaseNotes = 6;
   }
 }
 
 message Contact {
+  enum Registered {
+    UNKNOWN = 0;
+    REGISTERED = 1;
+    NOT_REGISTERED = 2;
+  }
+
   optional bytes aci = 1; // should be 16 bytes
   optional bytes pni = 2; // should be 16 bytes
   optional string username = 3;
   optional uint64 e164 = 4;
   bool blocked = 5;
   bool hidden = 6;
-  enum Registered {
-    UNKNOWN = 0;
-    REGISTERED = 1;
-    NOT_REGISTERED = 2;
-  }
   Registered registered = 7;
   uint64 unregisteredTimestamp = 8;
   optional bytes profileKey = 9;
   bool profileSharing = 10;
   optional string profileGivenName = 11;
   optional string profileFamilyName = 12;
-  optional string profileJoinedName = 13;
+  bool hideStory = 13;
-  bool hideStory = 14;
 }
 
 message Group {
@@ -134,30 +126,34 @@ message Group {
 
 message Self {}
 
+message ReleaseNotes {}
+
 message Chat {
   uint64 id = 1; // generated id for reference only within this file
   uint64 recipientId = 2;
   bool archived = 3;
-  bool pinned = 4;
+  uint32 pinnedOrder = 4; // 0 = unpinned, otherwise chat is considered pinned and will be displayed in ascending order
-  uint64 expirationTimer = 5;
+  uint64 expirationTimerMs = 5;
-  uint64 muteUntil = 6;
+  uint64 muteUntilMs = 6;
   bool markedUnread = 7;
   bool dontNotifyForMentionsIfMuted = 8;
+  FilePointer wallpaper = 9;
 }
 
 message DistributionList {
+  enum PrivacyMode {
+    UNKNOWN = 0;
+    ONLY_WITH = 1;
+    ALL_EXCEPT = 2;
+    ALL = 3;
+  }
+
   string name = 1;
   bytes distributionId = 2; // distribution list ids are uuids
   bool allowReplies = 3;
   uint64 deletionTimestamp = 4;
-  bool isUnknown = 5;
+  PrivacyMode privacyMode = 5;
-  enum PrivacyMode {
+  repeated uint64 memberRecipientIds = 6; // generated recipient id
-    ONLY_WITH = 0;
-    ALL_EXCEPT = 1;
-    ALL = 2;
-  }
-  PrivacyMode privacyMode = 6;
-  repeated uint64 memberRecipientIds = 7; // generated recipient id
 }
 
 message Identity {
@@ -170,38 +166,41 @@ message Identity {
 }
 
 message Call {
-  uint64 callId = 1;
-  uint64 peerRecipientId = 2;
   enum Type {
-    AUDIO_CALL = 0;
+    UNKNOWN_TYPE = 0;
-    VIDEO_CALL = 1;
+    AUDIO_CALL = 1;
-    GROUP_CALL = 2;
+    VIDEO_CALL = 2;
-    AD_HOC_CALL = 3;
+    GROUP_CALL = 3;
+    AD_HOC_CALL = 4;
   }
+
+  enum Event {
+    UNKNOWN_EVENT = 0;
+    OUTGOING = 1; // 1:1 calls only
+    ACCEPTED = 2; // 1:1 and group calls. Group calls: You accepted a ring.
+    NOT_ACCEPTED = 3; // 1:1 calls only,
+    MISSED = 4; // 1:1 and group. Group calls: The remote ring has expired or was cancelled by the ringer.
+    DELETE = 5; // 1:1 and Group/Ad-Hoc Calls.
+    GENERIC_GROUP_CALL = 6; // Group/Ad-Hoc Calls only. Initial state
+    JOINED = 7; // Group Calls: User has joined the group call.
+    DECLINED = 8; // Group Calls: If you declined a ring.
+    OUTGOING_RING = 9; // Group Calls: If you are ringing a group.
+  }
+
+  uint64 callId = 1;
+  uint64 conversationRecipientId = 2;
   Type type = 3;
   bool outgoing = 4;
   uint64 timestamp = 5;
   uint64 ringerRecipientId = 6;
-  enum Event {
-    OUTGOING = 0; // 1:1 calls only
-    ACCEPTED = 1; // 1:1 and group calls. Group calls: You accepted a ring.
-    NOT_ACCEPTED = 2; // 1:1 calls only,
-    MISSED = 3; // 1:1 and group/ad-hoc calls. Group calls: The remote ring has expired or was cancelled by the ringer.
-    DELETE = 4; // 1:1 and Group/Ad-Hoc Calls.
-    GENERIC_GROUP_CALL = 5; // Group/Ad-Hoc Calls only. Initial state
-    JOINED = 6; // Group Calls: User has joined the group call.
-    RINGING = 7; // Group Calls: If a ring was requested by another user.
-    DECLINED = 8; // Group Calls: If you declined a ring.
-    OUTGOING_RING = 9; // Group Calls: If you are ringing a group.
-  }
   Event event = 7;
 }
 
 message ChatItem {
   message IncomingMessageDetails {
-    uint64 dateServerSent = 1;
+    uint64 dateReceived = 1;
-    bool read = 2;
+    uint64 dateServerSent = 2;
-    bool sealedSender = 3;
+    bool read = 3;
   }
 
   message OutgoingMessageDetails {
@@ -211,43 +210,45 @@ message ChatItem {
   uint64 chatId = 1;   // conversation id
   uint64 authorId = 2; // recipient id
   uint64 dateSent = 3;
-  uint64 dateReceived = 4;
+  bool sealedSender = 4;
-  optional uint64 expireStart = 5; // timestamp of when expiration timer started ticking down
+  optional uint64 expireStartDate = 5; // timestamp of when expiration timer started ticking down
-  optional uint64 expiresIn = 6; // how long timer of message is (ms)
+  optional uint64 expiresInMs = 6; // how long timer of message is (ms)
-  repeated ChatItem revisions = 7;
+  repeated ChatItem revisions = 7; // ordered from oldest to newest
   bool sms = 8;
 
   oneof directionalDetails {
-    IncomingMessageDetails incoming = 9;
+    IncomingMessageDetails incoming = 10;
-    OutgoingMessageDetails outgoing = 10;
+    OutgoingMessageDetails outgoing = 12;
   }
 
   oneof item {
-    StandardMessage standardMessage = 11;
+    StandardMessage standardMessage = 13;
-    ContactMessage contactMessage = 12;
+    ContactMessage contactMessage = 14;
-    VoiceMessage voiceMessage = 13;
+    VoiceMessage voiceMessage = 15;
-    StickerMessage stickerMessage = 14;
+    StickerMessage stickerMessage = 16;
-    RemoteDeletedMessage remoteDeletedMessage = 15;
+    RemoteDeletedMessage remoteDeletedMessage = 17;
-    UpdateMessage updateMessage = 16;
+    ChatUpdateMessage updateMessage = 18;
   }
 }
 
 message SendStatus {
   enum Status {
-    FAILED = 0;
+    UNKNOWN = 0;
-    PENDING = 1;
+    FAILED = 1;
-    SENT = 2;
+    PENDING = 2;
-    DELIVERED = 3;
+    SENT = 3;
-    READ = 4;
+    DELIVERED = 4;
-    VIEWED = 5;
+    READ = 5;
-    SKIPPED = 6; // e.g. user in group was blocked, so we skipped sending to them
+    VIEWED = 6;
+    SKIPPED = 7; // e.g. user in group was blocked, so we skipped sending to them
   }
+
   uint64 recipientId = 1;
   Status deliveryStatus = 2;
   bool networkFailure = 3;
   bool identityKeyMismatch = 4;
   bool sealedSender = 5;
-  uint64 timestamp = 6;
+  uint64 lastStatusUpdateTimestamp = 6; // the time the status was last updated -- if from a receipt, it should be the sentTime of the receipt
 }
 
 message Text {
@@ -258,9 +259,9 @@ message Text {
 message StandardMessage {
   optional Quote quote = 1;
   optional Text text = 2;
-  repeated AttachmentPointer attachments = 3;
+  repeated FilePointer attachments = 3;
-  optional LinkPreview linkPreview = 4;
+  repeated LinkPreview linkPreview = 4;
-  optional AttachmentPointer longText = 5;
+  optional FilePointer longText = 5;
   repeated Reaction reactions = 6;
 }
 
@@ -281,10 +282,11 @@ message ContactAttachment {
 
   message Phone {
     enum Type {
-      HOME = 0;
+      UNKNOWN = 0;
-      MOBILE = 1;
+      HOME = 1;
-      WORK = 2;
+      MOBILE = 2;
-      CUSTOM = 3;
+      WORK = 3;
+      CUSTOM = 4;
     }
 
     optional string value = 1;
@@ -294,10 +296,11 @@ message ContactAttachment {
 
   message Email {
     enum Type {
-      HOME = 0;
+      UNKNOWN = 0;
-      MOBILE = 1;
+      HOME = 1;
-      WORK = 2;
+      MOBILE = 2;
-      CUSTOM = 3;
+      WORK = 3;
+      CUSTOM = 4;
     }
 
     optional string value = 1;
@@ -307,9 +310,10 @@ message ContactAttachment {
 
   message PostalAddress {
     enum Type {
-      HOME = 0;
+      UNKNOWN = 0;
-      WORK = 1;
+      HOME = 1;
-      CUSTOM = 2;
+      WORK = 2;
+      CUSTOM = 3;
     }
 
     optional Type type = 1;
@@ -324,8 +328,7 @@ message ContactAttachment {
   }
 
   message Avatar {
-    optional AttachmentPointer avatar = 1;
+    FilePointer avatar = 1;
-    optional bool isProfile = 2;
   }
 
   optional Name name = 1;
@@ -338,13 +341,13 @@ message ContactAttachment {
 
 message DocumentMessage {
   Text text = 1;
-  AttachmentPointer document = 2;
+  FilePointer document = 2;
   repeated Reaction reactions = 3;
 }
 
 message VoiceMessage {
   optional Quote quote = 1;
-  AttachmentPointer audio = 2;
+  FilePointer audio = 2;
   repeated Reaction reactions = 3;
 }
 
@@ -356,11 +359,6 @@ message StickerMessage {
 // Tombstone for remote delete
 message RemoteDeletedMessage {}
 
-message ScheduledMessage {
-  ChatItem message = 1;
-  uint64 scheduledTime = 2;
-}
-
 message Sticker {
   bytes packId = 1;
   bytes packKey = 2;
@@ -371,37 +369,62 @@ message Sticker {
 message LinkPreview {
   string url = 1;
   optional string title = 2;
-  optional AttachmentPointer image = 3;
+  optional FilePointer image = 3;
   optional string description = 4;
   optional uint64 date = 5;
 }
 
-message AttachmentPointer {
+message FilePointer {
+  message BackupLocator {
+    string mediaName = 1;
+    uint32 cdnNumber = 2;
+  }
+
+  message AttachmentLocator {
+    string cdnKey = 1;
+    uint32 cdnNumber = 2;
+    uint64 uploadTimestamp = 3;
+  }
+
+  message LegacyAttachmentLocator {
+    fixed64 cdnId = 1;
+  }
+
+  // An attachment that was backed up without being downloaded.
+  // Its MediaName should be generated as “{sender_aci}_{cdn_attachment_key}”,
+  // but should eventually transition to a BackupLocator with mediaName
+  // being the content hash once it is downloaded.
+  message UndownloadedBackupLocator {
+    bytes senderAci = 1;
+    string cdnKey = 2;
+    uint32 cdnNumber = 3;
+  }
+
   enum Flags {
     VOICE_MESSAGE = 0;
     BORDERLESS = 1;
     GIF = 2;
   }
 
-  oneof attachmentIdentifier {
+  oneof locator {
-    fixed64 cdnId = 1;
+    BackupLocator backupLocator = 1;
-    string cdnKey = 2;
+    AttachmentLocator attachmentLocator= 2;
+    LegacyAttachmentLocator legacyAttachmentLocator = 3;
+    UndownloadedBackupLocator undownloadedBackupLocator = 4;
   }
 
-  optional string contentType = 3;
+  optional bytes key = 5;
-  optional bytes key = 4;
+  optional string contentType = 6;
-  optional uint32 size = 5;
+  optional uint32 size = 7;
-  optional bytes digest = 6;
+  optional bytes digest = 8;
-  optional bytes incrementalMac = 7;
+  optional bytes incrementalMac = 9;
-  optional bytes incrementalMacChunkSize = 8;
+  optional bytes incrementalMacChunkSize = 10;
-  optional string fileName = 9;
+  optional string fileName = 11;
-  optional uint32 flags = 10;
+  optional uint32 flags = 12;
-  optional uint32 width = 11;
+  optional uint32 width = 13;
-  optional uint32 height = 12;
+  optional uint32 height = 14;
-  optional string caption = 13;
+  optional string caption = 15;
-  optional string blurHash = 14;
+  optional string blurHash = 16;
-  optional uint64 uploadTimestamp = 15;
-  optional uint32 cdnNumber = 16;
 }
 
 message Quote {
@@ -414,16 +437,15 @@ message Quote {
   message QuotedAttachment {
     optional string contentType = 1;
     optional string fileName = 2;
-    optional AttachmentPointer thumbnail = 3;
+    optional FilePointer thumbnail = 3;
   }
 
-  uint64 targetSentTimestamp = 1;
+  optional uint64 targetSentTimestamp = 1; // null if the target message could not be found at time of quote insert
   uint64 authorId = 2;
   optional string text = 3;
   repeated QuotedAttachment attachments = 4;
   repeated BodyRange bodyRanges = 5;
   Type type = 6;
-  bool originalMessageMissing = 7;
 }
 
 message BodyRange {
@@ -440,7 +462,7 @@ message BodyRange {
   optional uint32 length = 2;
 
   oneof associatedValue {
-    string mentionAci = 3;
+    bytes mentionAci = 3;
     Style style = 4;
   }
 }
@@ -449,82 +471,85 @@ message Reaction {
   string emoji = 1;
   uint64 authorId = 2;
   uint64 sentTimestamp = 3;
-  uint64 receivedTimestamp = 4;
+  optional uint64 receivedTimestamp = 4;
+  uint64 sortOrder = 5; // A higher sort order means that a reaction is more recent
 }
 
-message UpdateMessage {
+message ChatUpdateMessage {
   oneof update {
-    SimpleUpdate simpleUpdate = 1;
+    SimpleChatUpdate simpleUpdate = 1;
-    GroupDescriptionUpdate groupDescription = 2;
+    GroupDescriptionChatUpdate groupDescription = 2;
-    ExpirationTimerChange expirationTimerChange = 3;
+    ExpirationTimerChatUpdate expirationTimerChange = 3;
-    ProfileChange profileChange = 4;
+    ProfileChangeChatUpdate profileChange = 4;
-    ThreadMergeEvent threadMerge = 5;
+    ThreadMergeChatUpdate threadMerge = 5;
-    SessionSwitchoverEvent sessionSwitchover = 6;
+    SessionSwitchoverChatUpdate sessionSwitchover = 6;
-    CallingMessage callingMessage = 7;
+    CallChatUpdate callingMessage = 7;
   }
 }
 
-message CallingMessage {
+message CallChatUpdate{
   oneof call {
     uint64 callId = 1; // maps to id of Call from call log
-    CallMessage callMessage = 2;
+    IndividualCallChatUpdate callMessage = 2;
-    GroupCallMessage groupCall = 3;
+    GroupCallChatUpdate groupCall = 3;
   }
 }
 
-message CallMessage {
+message IndividualCallChatUpdate {
   enum Type {
-    INCOMING_AUDIO_CALL = 0;
+    UNKNOWN = 0;
-    INCOMING_VIDEO_CALL = 1;
+    INCOMING_AUDIO_CALL = 1;
-    OUTGOING_AUDIO_CALL = 2;
+    INCOMING_VIDEO_CALL = 2;
-    OUTGOING_VIDEO_CALL = 3;
+    OUTGOING_AUDIO_CALL = 3;
-    MISSED_AUDIO_CALL = 4;
+    OUTGOING_VIDEO_CALL = 4;
-    MISSED_VIDEO_CALL = 5;
+    MISSED_AUDIO_CALL = 5;
+    MISSED_VIDEO_CALL = 6;
   }
 }
 
-message GroupCallMessage {
+message GroupCallChatUpdate {
-  bytes startedCallUuid = 1;
+  bytes startedCallAci = 1;
   uint64 startedCallTimestamp = 2;
-  repeated bytes inCallUuids = 3;
+  repeated bytes inCallAcis = 3;
-  bool isCallFull = 4;
 }
 
-message SimpleUpdate {
+message SimpleChatUpdate {
   enum Type {
-    JOINED_SIGNAL = 0;
+    UNKNOWN = 0;
-    IDENTITY_UPDATE = 1;
+    JOINED_SIGNAL = 1;
-    IDENTITY_VERIFIED = 2;
+    IDENTITY_UPDATE = 2;
-    IDENTITY_DEFAULT = 3; // marking as unverified
+    IDENTITY_VERIFIED = 3;
-    CHANGE_NUMBER = 4;
+    IDENTITY_DEFAULT = 4; // marking as unverified
-    BOOST_REQUEST = 5;
+    CHANGE_NUMBER = 5;
-    END_SESSION = 6;
+    BOOST_REQUEST = 6;
-    CHAT_SESSION_REFRESH = 7;
+    END_SESSION = 7;
-    BAD_DECRYPT = 8;
+    CHAT_SESSION_REFRESH = 8;
-    PAYMENTS_ACTIVATED = 9;
+    BAD_DECRYPT = 9;
-    PAYMENT_ACTIVATION_REQUEST = 10;
+    PAYMENTS_ACTIVATED = 10;
+    PAYMENT_ACTIVATION_REQUEST = 11;
   }
+
   Type type = 1;
 }
 
-message GroupDescriptionUpdate {
+message GroupDescriptionChatUpdate {
-  string body = 1;
+  string newDescription = 1;
 }
 
-message ExpirationTimerChange {
+message ExpirationTimerChatUpdate {
-  uint32 expiresIn = 1;
+  uint32 expiresInMs = 1;
 }
 
-message ProfileChange {
+message ProfileChangeChatUpdate {
   string previousName = 1;
   string newName = 2;
 }
 
-message ThreadMergeEvent {
+message ThreadMergeChatUpdate {
   uint64 previousE164 = 1;
 }
 
-message SessionSwitchoverEvent {
+message SessionSwitchoverChatUpdate {
   uint64 e164 = 1;
 }
 
@@ -537,6 +562,6 @@ message StickerPack {
 }
 
 message StickerPackSticker {
-  AttachmentPointer data = 1;
+  FilePointer data = 1;
   string emoji = 2;
 }

app/src/test/java/org/thoughtcrime/securesms/backup/v2/stream/EncryptedBackupReaderWriterTest.kt

@@ -0,0 +1,46 @@
+/*
+ * Copyright 2023 Signal Messenger, LLC
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+package org.thoughtcrime.securesms.backup.v2.stream
+
+import org.junit.Assert.assertEquals
+import org.junit.Test
+import org.thoughtcrime.securesms.backup.v2.proto.AccountData
+import org.thoughtcrime.securesms.backup.v2.proto.Frame
+import org.thoughtcrime.securesms.util.Util
+import org.whispersystems.signalservice.api.backup.BackupKey
+import org.whispersystems.signalservice.api.push.ServiceId.ACI
+import java.io.ByteArrayOutputStream
+import java.util.UUID
+
+class EncryptedBackupReaderWriterTest {
+
+  @Test
+  fun `can read back all of the frames we write`() {
+    val key = BackupKey(Util.getSecretBytes(32))
+    val aci = ACI.from(UUID.randomUUID())
+
+    val outputStream = ByteArrayOutputStream()
+
+    val frameCount = 10_000
+    EncryptedBackupWriter(key, aci, outputStream, append = { outputStream.write(it) }).use { writer ->
+      for (i in 0 until frameCount) {
+        writer.write(Frame(account = AccountData(username = "username-$i")))
+      }
+    }
+
+    val ciphertext: ByteArray = outputStream.toByteArray()
+
+    val frames: List<Frame> = EncryptedBackupReader(key, aci, ciphertext.size.toLong()) { ciphertext.inputStream() }.use { reader ->
+      reader.asSequence().toList()
+    }
+
+    assertEquals(frameCount, frames.size)
+
+    for (i in 0 until frameCount) {
+      assertEquals("username-$i", frames[i].account?.username)
+    }
+  }
+}

core-util-jvm/build.gradle.kts

@@ -18,4 +18,5 @@ java {
 
 dependencies {
   testImplementation(testLibs.junit.junit)
+  testImplementation(testLibs.assertj.core)
 }

core-util-jvm/src/main/java/org/signal/core/util/InputStreamExtensions.kt

@@ -0,0 +1,82 @@
+/*
+ * Copyright 2023 Signal Messenger, LLC
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+package org.signal.core.util
+
+import java.io.IOException
+import java.io.InputStream
+import kotlin.jvm.Throws
+
+/**
+ * Reads a 32-bit variable-length integer from the stream.
+ *
+ * The format uses one byte for each 7 bits of the integer, with the most significant bit (MSB) of each byte indicating whether more bytes need to be read.
+ * If the MSB is 0, it indicates the final byte. The actual integer value is constructed from the remaining 7 bits of each byte.
+ */
+fun InputStream.readVarInt32(): Int {
+  var result = 0
+
+  // We read 7 bits of the integer at a time, up to the full size of an integer (32 bits).
+  for (shift in 0 until 32 step 7) {
+    // Despite returning an int, the range of the returned value is 0..255, so it's just a byte.
+    // I believe it's an int just so it can return -1 when the stream ends.
+    val byte: Int = read()
+    if (byte < 0) {
+      return -1
+    }
+
+    val lowestSevenBits = byte and 0x7F
+    val shiftedBits = lowestSevenBits shl shift
+
+    result = result or shiftedBits
+
+    // If the MSB is 0, that means the varint is finished, and we have our full result
+    if (byte and 0x80 == 0) {
+      return result
+    }
+  }
+
+  throw IOException("Malformed varint!")
+}
+
+/**
+ * Reads the entire stream into a [ByteArray].
+ */
+@Throws(IOException::class)
+fun InputStream.readFully(autoClose: Boolean = true): ByteArray {
+  return StreamUtil.readFully(this, Integer.MAX_VALUE, autoClose)
+}
+
+/**
+ * Fills reads data from the stream into the [buffer] until it is full.
+ * Throws an [IOException] if the stream doesn't have enough data to fill the buffer.
+ */
+@Throws(IOException::class)
+fun InputStream.readFully(buffer: ByteArray) {
+  return StreamUtil.readFully(this, buffer)
+}
+
+/**
+ * Reads the specified number of bytes from the stream and returns it as a [ByteArray].
+ * Throws an [IOException] if the stream doesn't have that many bytes.
+ */
+@Throws(IOException::class)
+fun InputStream.readNBytesOrThrow(length: Int): ByteArray {
+  val buffer = ByteArray(length)
+  this.readFully(buffer)
+  return buffer
+}
+
+@Throws(IOException::class)
+fun InputStream.readLength(): Long? {
+  val buffer = ByteArray(4096)
+  var count = 0L
+
+  while (this.read(buffer).also { if (it > 0) count += it } != -1) {
+    // do nothing, all work is in the while condition
+  }
+
+  return count
+}

core-util-jvm/src/main/java/org/signal/core/util/OutputStreamExtensions.kt

@@ -0,0 +1,32 @@
+/*
+ * Copyright 2023 Signal Messenger, LLC
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+package org.signal.core.util
+
+import java.io.OutputStream
+
+/**
+ * Writes a 32-bit variable-length integer to the stream.
+ *
+ * The format uses one byte for each 7 bits of the integer, with the most significant bit (MSB) of each byte indicating whether more bytes need to be read.
+ */
+fun OutputStream.writeVarInt32(value: Int) {
+  var remaining = value
+
+  while (true) {
+    // We write 7 bits of the integer at a time
+    val lowestSevenBits = remaining and 0x7F
+    remaining = remaining ushr 7
+
+    if (remaining == 0) {
+      // If there are no more bits to write, we're done
+      write(lowestSevenBits)
+      return
+    } else {
+      // Otherwise, we need to write the next 7 bits, and set the MSB to 1 to indicate that there are more bits to come
+      write(lowestSevenBits or 0x80)
+    }
+  }
+}

core-util-jvm/src/main/java/org/signal/core/util/StreamUtil.java

@@ -1,6 +1,9 @@
-package org.signal.core.util;
+/*
+ * Copyright 2023 Signal Messenger, LLC
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
 
-import androidx.annotation.Nullable;
+package org.signal.core.util;
 
 import org.signal.core.util.logging.Log;
 
@@ -20,7 +23,7 @@ public final class StreamUtil {
 
   private StreamUtil() {}
 
-  public static void close(@Nullable Closeable closeable) {
+  public static void close(Closeable closeable) {
     if (closeable == null) return;
 
     try {
@@ -64,6 +67,10 @@ public final class StreamUtil {
   }
 
   public static byte[] readFully(InputStream in, int maxBytes) throws IOException {
+    return readFully(in, maxBytes, true);
+  }
+
+  public static byte[] readFully(InputStream in, int maxBytes, boolean closeWhenDone) throws IOException {
     ByteArrayOutputStream bout      = new ByteArrayOutputStream();
     byte[]                buffer    = new byte[4096];
     int                   totalRead = 0;
@@ -77,7 +84,9 @@ public final class StreamUtil {
       }
     }
 
+    if (closeWhenDone) {
       in.close();
+    }
 
     return bout.toByteArray();
   }

core-util-jvm/src/main/java/org/signal/core/util/stream/MacInputStream.kt

@@ -0,0 +1,43 @@
+/*
+ * Copyright 2023 Signal Messenger, LLC
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+package org.signal.core.util.stream
+
+import java.io.FilterInputStream
+import java.io.InputStream
+import javax.crypto.Mac
+
+/**
+ * Calculates a [Mac] as data is read from the target [InputStream].
+ * To get the final MAC, read the [mac] property after the stream has been fully read.
+ *
+ * Example:
+ * ```kotlin
+ * val stream = MacInputStream(myStream, myMac)
+ * stream.readFully()
+ * val mac = stream.mac.doFinal()
+ * ```
+ */
+class MacInputStream(val wrapped: InputStream, val mac: Mac) : FilterInputStream(wrapped) {
+  override fun read(): Int {
+    return wrapped.read().also { byte ->
+      if (byte >= 0) {
+        mac.update(byte.toByte())
+      }
+    }
+  }
+
+  override fun read(destination: ByteArray): Int {
+    return read(destination, 0, destination.size)
+  }
+
+  override fun read(destination: ByteArray, offset: Int, length: Int): Int {
+    return wrapped.read(destination, offset, length).also { bytesRead ->
+      if (bytesRead > 0) {
+        mac.update(destination, offset, bytesRead)
+      }
+    }
+  }
+}

core-util-jvm/src/main/java/org/signal/core/util/stream/MacOutputStream.kt

@@ -0,0 +1,37 @@
+/*
+ * Copyright 2023 Signal Messenger, LLC
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+package org.signal.core.util.stream
+
+import java.io.FilterOutputStream
+import java.io.OutputStream
+import javax.crypto.Mac
+
+/**
+ * Calculates a [Mac] as data is written to the target [OutputStream].
+ * To get the final MAC, read the [mac] property after the stream has been fully written.
+ *
+ * Example:
+ * ```kotlin
+ * val stream = MacOutputStream(myStream, myMac)
+ * // write data to stream
+ * val mac = stream.mac.doFinal()
+ * ```
+ */
+class MacOutputStream(val wrapped: OutputStream, val mac: Mac) : FilterOutputStream(wrapped) {
+  override fun write(byte: Int) {
+    wrapped.write(byte)
+    mac.update(byte.toByte())
+  }
+
+  override fun write(data: ByteArray) {
+    write(data, 0, data.size)
+  }
+
+  override fun write(data: ByteArray, offset: Int, length: Int) {
+    wrapped.write(data, offset, length)
+    mac.update(data, offset, length)
+  }
+}

core-util-jvm/src/main/java/org/signal/core/util/stream/TruncatingInputStream.kt

@@ -0,0 +1,58 @@
+/*
+ * Copyright 2023 Signal Messenger, LLC
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+package org.signal.core.util.stream
+
+import java.io.FilterInputStream
+import java.io.InputStream
+import java.lang.UnsupportedOperationException
+
+/**
+ * An [InputStream] that will read from the target [InputStream] until it reaches the end, or until it has read [maxBytes] bytes.
+ */
+class TruncatingInputStream(private val wrapped: InputStream, private val maxBytes: Long) : FilterInputStream(wrapped) {
+
+  private var bytesRead: Long = 0
+
+  override fun read(): Int {
+    if (bytesRead >= maxBytes) {
+      return -1
+    }
+
+    return wrapped.read().also {
+      if (it >= 0) {
+        bytesRead++
+      }
+    }
+  }
+
+  override fun read(destination: ByteArray): Int {
+    return read(destination, 0, destination.size)
+  }
+
+  override fun read(destination: ByteArray, offset: Int, length: Int): Int {
+    if (bytesRead >= maxBytes) {
+      return -1
+    }
+
+    val bytesRemaining: Long = maxBytes - bytesRead
+    val bytesToRead: Int = if (bytesRemaining > length) length else Math.toIntExact(bytesRemaining)
+    val bytesRead = wrapped.read(destination, offset, bytesToRead)
+
+    if (bytesRead > 0) {
+      this.bytesRead += bytesRead
+    }
+
+    return bytesRead
+  }
+
+  override fun skip(n: Long): Long {
+    throw UnsupportedOperationException()
+  }
+
+  override fun reset() {
+    throw UnsupportedOperationException()
+  }
+}

core-util-jvm/src/test/java/org/signal/core/util/VarInt32Tests.kt

@@ -0,0 +1,75 @@
+/*
+ * Copyright 2023 Signal Messenger, LLC
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+package org.signal.core.util
+
+import org.junit.Assert.assertEquals
+import org.junit.Ignore
+import org.junit.Test
+import java.io.ByteArrayOutputStream
+import java.util.concurrent.atomic.AtomicInteger
+import kotlin.random.Random
+
+class VarInt32Tests {
+
+  /**
+   * Tests a random sampling of integers. The faster and more practical version of [testAll].
+   */
+  @Test
+  fun testRandomSampling() {
+    val randomInts = (0..100_000).map { Random.nextInt() }
+
+    val bytes = ByteArrayOutputStream().use { outputStream ->
+      for (value in randomInts) {
+        outputStream.writeVarInt32(value)
+      }
+      outputStream
+    }.toByteArray()
+
+    bytes.inputStream().use { inputStream ->
+      for (value in randomInts) {
+        val read = inputStream.readVarInt32()
+        assertEquals(value, read)
+      }
+    }
+  }
+
+  /**
+   * Exhaustively checks reading and writing a varint for all possible integers.
+   * We can't keep everything in memory, so instead we use sequences to grab a million at a time,
+   * then run smaller chunks of those in parallel.
+   */
+  @Ignore("This test is very slow (over a minute). It was run once to verify correctness, but the random sampling test should be sufficient for catching regressions.")
+  @Test
+  fun testAll() {
+    val counter = AtomicInteger(0)
+
+    (Int.MIN_VALUE..Int.MAX_VALUE)
+      .asSequence()
+      .chunked(1_000_000)
+      .forEach { bigChunk ->
+        bigChunk
+          .chunked(100_000)
+          .parallelStream()
+          .forEach { smallChunk ->
+            println("Chunk ${counter.addAndGet(1)}")
+
+            val bytes = ByteArrayOutputStream().use { outputStream ->
+              for (value in smallChunk) {
+                outputStream.writeVarInt32(value)
+              }
+              outputStream
+            }.toByteArray()
+
+            bytes.inputStream().use { inputStream ->
+              for (value in smallChunk) {
+                val read = inputStream.readVarInt32()
+                assertEquals(value, read)
+              }
+            }
+          }
+      }
+  }
+}

core-util-jvm/src/test/java/org/signal/core/util/stream/MacInputStreamTest.kt

@@ -0,0 +1,56 @@
+/*
+ * Copyright 2023 Signal Messenger, LLC
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+package org.signal.core.util.stream
+
+import org.junit.Assert.assertArrayEquals
+import org.junit.Test
+import org.signal.core.util.readFully
+import java.io.InputStream
+import javax.crypto.Mac
+import javax.crypto.spec.SecretKeySpec
+import kotlin.random.Random
+
+class MacInputStreamTest {
+
+  @Test
+  fun `stream mac matches normal mac when reading via buffer`() {
+    testMacEquality { inputStream ->
+      inputStream.readFully()
+    }
+  }
+
+  @Test
+  fun `stream mac matches normal mac when reading one byte at a time`() {
+    testMacEquality { inputStream ->
+      var lastRead = inputStream.read()
+      while (lastRead != -1) {
+        lastRead = inputStream.read()
+      }
+    }
+  }
+
+  private fun testMacEquality(read: (InputStream) -> Unit) {
+    val data = Random.nextBytes(1_000)
+    val key = Random.nextBytes(32)
+
+    val mac1 = Mac.getInstance("HmacSHA256").apply {
+      init(SecretKeySpec(key, "HmacSHA256"))
+    }
+
+    val mac2 = Mac.getInstance("HmacSHA256").apply {
+      init(SecretKeySpec(key, "HmacSHA256"))
+    }
+
+    val expectedMac = mac1.doFinal(data)
+
+    val actualMac = MacInputStream(data.inputStream(), mac2).use { stream ->
+      read(stream)
+      stream.mac.doFinal()
+    }
+
+    assertArrayEquals(expectedMac, actualMac)
+  }
+}

core-util-jvm/src/test/java/org/signal/core/util/stream/MacOutputStreamTest.kt

@@ -0,0 +1,56 @@
+/*
+ * Copyright 2023 Signal Messenger, LLC
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+package org.signal.core.util.stream
+
+import org.junit.Assert.assertArrayEquals
+import org.junit.Test
+import org.signal.core.util.StreamUtil
+import java.io.ByteArrayOutputStream
+import java.io.OutputStream
+import javax.crypto.Mac
+import javax.crypto.spec.SecretKeySpec
+import kotlin.random.Random
+
+class MacOutputStreamTest {
+
+  @Test
+  fun `stream mac matches normal mac when writing via buffer`() {
+    testMacEquality { data, outputStream ->
+      StreamUtil.copy(data.inputStream(), outputStream)
+    }
+  }
+
+  @Test
+  fun `stream mac matches normal mac when writing one byte at a time`() {
+    testMacEquality { data, outputStream ->
+      for (byte in data) {
+        outputStream.write(byte.toInt())
+      }
+    }
+  }
+
+  private fun testMacEquality(write: (ByteArray, OutputStream) -> Unit) {
+    val data = Random.nextBytes(1_000)
+    val key = Random.nextBytes(32)
+
+    val mac1 = Mac.getInstance("HmacSHA256").apply {
+      init(SecretKeySpec(key, "HmacSHA256"))
+    }
+
+    val mac2 = Mac.getInstance("HmacSHA256").apply {
+      init(SecretKeySpec(key, "HmacSHA256"))
+    }
+
+    val expectedMac = mac1.doFinal(data)
+
+    val actualMac = MacOutputStream(ByteArrayOutputStream(), mac2).use { stream ->
+      write(data, stream)
+      stream.mac.doFinal()
+    }
+
+    assertArrayEquals(expectedMac, actualMac)
+  }
+}

core-util-jvm/src/test/java/org/signal/core/util/stream/TruncatingInputStreamTest.kt

@@ -0,0 +1,35 @@
+/*
+ * Copyright 2023 Signal Messenger, LLC
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+package org.signal.core.util.stream
+
+import org.junit.Assert.assertEquals
+import org.junit.Test
+import org.signal.core.util.readFully
+
+class TruncatingInputStreamTest {
+
+  @Test
+  fun `when I fully read the stream via a buffer, I should only get maxBytes`() {
+    val inputStream = TruncatingInputStream(ByteArray(100).inputStream(), maxBytes = 75)
+    val data = inputStream.readFully()
+
+    assertEquals(75, data.size)
+  }
+
+  @Test
+  fun `when I fully read the stream one byte at a time, I should only get maxBytes`() {
+    val inputStream = TruncatingInputStream(ByteArray(100).inputStream(), maxBytes = 75)
+
+    var count = 0
+    var lastRead = inputStream.read()
+    while (lastRead != -1) {
+      count++
+      lastRead = inputStream.read()
+    }
+
+    assertEquals(75, count)
+  }
+}

core-util/src/main/java/org/signal/core/util/ContentResolverExtensions.kt

@@ -0,0 +1,22 @@
+/*
+ * Copyright 2023 Signal Messenger, LLC
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+package org.signal.core.util
+
+import android.content.ContentResolver
+import android.net.Uri
+import android.provider.OpenableColumns
+import okio.IOException
+
+@Throws(IOException::class)
+fun ContentResolver.getLength(uri: Uri): Long? {
+  return this.query(uri, arrayOf(OpenableColumns.SIZE), null, null, null)?.use { cursor ->
+    if (cursor.moveToFirst()) {
+      cursor.requireLongOrNull(OpenableColumns.SIZE)
+    } else {
+      null
+    }
+  } ?: openInputStream(uri)?.use { it.readLength() }
+}

core-util/src/main/java/org/signal/core/util/InputStreamExtensions.kt

@@ -1,38 +0,0 @@
-/*
- * Copyright 2023 Signal Messenger, LLC
- * SPDX-License-Identifier: AGPL-3.0-only
- */
-
-package org.signal.core.util
-
-import java.io.IOException
-import java.io.InputStream
-import kotlin.jvm.Throws
-
-/**
- * Reads the entire stream into a [ByteArray].
- */
-@Throws(IOException::class)
-fun InputStream.readFully(): ByteArray {
-  return StreamUtil.readFully(this)
-}
-
-/**
- * Fills reads data from the stream into the [buffer] until it is full.
- * Throws an [IOException] if the stream doesn't have enough data to fill the buffer.
- */
-@Throws(IOException::class)
-fun InputStream.readFully(buffer: ByteArray) {
-  return StreamUtil.readFully(this, buffer)
-}
-
-/**
- * Reads the specified number of bytes from the stream and returns it as a [ByteArray].
- * Throws an [IOException] if the stream doesn't have that many bytes.
- */
-@Throws(IOException::class)
-fun InputStream.readNBytesOrThrow(length: Int): ByteArray {
-  val buffer: ByteArray = ByteArray(length)
-  this.readFully(buffer)
-  return buffer
-}

core-util/src/test/java/org/signal/core/util/InputStreamExtensionTests.kt

@@ -0,0 +1,22 @@
+/*
+ * Copyright 2023 Signal Messenger, LLC
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+package org.signal.core.util
+
+import org.junit.Assert.assertEquals
+import org.junit.Test
+import kotlin.random.Random
+
+class InputStreamExtensionTests {
+
+  @Test
+  fun `when I call readLength, it returns the correct length`() {
+    for (i in 1..10) {
+      val bytes = ByteArray(Random.nextInt(from = 512, until = 8092))
+      val length = bytes.inputStream().readLength()
+      assertEquals(bytes.size.toLong(), length)
+    }
+  }
+}

libsignal-service/src/main/java/org/whispersystems/signalservice/api/backup/BackupId.kt

@@ -0,0 +1,17 @@
+/*
+ * Copyright 2023 Signal Messenger, LLC
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+package org.whispersystems.signalservice.api.backup
+
+/**
+ * Safe typing around a backupId, which is a 16-byte array.
+ */
+@JvmInline
+value class BackupId(val value: ByteArray) {
+
+  init {
+    require(value.size == 16) { "BackupId must be 16 bytes!" }
+  }
+}

libsignal-service/src/main/java/org/whispersystems/signalservice/api/backup/BackupKey.kt

@@ -0,0 +1,40 @@
+/*
+ * Copyright 2023 Signal Messenger, LLC
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+package org.whispersystems.signalservice.api.backup
+
+import org.signal.libsignal.protocol.kdf.HKDF
+import org.whispersystems.signalservice.api.push.ServiceId.ACI
+
+/**
+ * Safe typing around a backup key, which is a 32-byte array.
+ */
+class BackupKey(val value: ByteArray) {
+  init {
+    require(value.size == 32) { "Backup key must be 32 bytes!" }
+  }
+
+  fun deriveSecrets(aci: ACI): KeyMaterial {
+    val backupId = BackupId(
+      HKDF.deriveSecrets(this.value, aci.toByteArray(), "20231003_Signal_Backups_GenerateBackupId".toByteArray(), 16)
+    )
+
+    val extendedKey = HKDF.deriveSecrets(this.value, backupId.value, "20231003_Signal_Backups_EncryptMessageBackup".toByteArray(), 80)
+
+    return KeyMaterial(
+      backupId = backupId,
+      macKey = extendedKey.copyOfRange(0, 32),
+      cipherKey = extendedKey.copyOfRange(32, 64),
+      iv = extendedKey.copyOfRange(64, 80)
+    )
+  }
+
+  class KeyMaterial(
+    val backupId: BackupId,
+    val macKey: ByteArray,
+    val cipherKey: ByteArray,
+    val iv: ByteArray
+  )
+}

libsignal-service/src/main/java/org/whispersystems/signalservice/api/kbs/MasterKey.java

@@ -1,5 +1,7 @@
 package org.whispersystems.signalservice.api.kbs;
 
+import org.signal.libsignal.protocol.kdf.HKDF;
+import org.whispersystems.signalservice.api.backup.BackupKey;
 import org.whispersystems.signalservice.api.storage.StorageKey;
 import org.whispersystems.signalservice.internal.util.Hex;
 import org.signal.core.util.Base64;
@@ -44,6 +46,10 @@ public final class MasterKey {
     return derive("Logging Key");
   }
 
+  public BackupKey deriveBackupKey() {
+    return new BackupKey(HKDF.deriveSecrets(masterKey, "20231003_Signal_Backups_GenerateBackupKey".getBytes(), 32));
+  }
+
   private byte[] derive(String keyName) {
     return hmacSha256(masterKey, StringUtil.utf8(keyName));
   }