diff --git a/app/src/androidTest/java/org/thoughtcrime/securesms/database/AttachmentTableTest.kt b/app/src/androidTest/java/org/thoughtcrime/securesms/database/AttachmentTableTest.kt index e50c43ce86..9bee3ab029 100644 --- a/app/src/androidTest/java/org/thoughtcrime/securesms/database/AttachmentTableTest.kt +++ b/app/src/androidTest/java/org/thoughtcrime/securesms/database/AttachmentTableTest.kt @@ -482,7 +482,8 @@ class AttachmentTableTest { quote = false, quoteTargetContentType = null, uuid = UUID.randomUUID(), - fileName = null + fileName = null, + localBackupKey = null ) } diff --git a/app/src/androidTest/java/org/thoughtcrime/securesms/messages/SyncMessageProcessorTest_synchronizeDeleteForMe.kt b/app/src/androidTest/java/org/thoughtcrime/securesms/messages/SyncMessageProcessorTest_synchronizeDeleteForMe.kt index 8cf2a04d4f..e42c0dd764 100644 --- a/app/src/androidTest/java/org/thoughtcrime/securesms/messages/SyncMessageProcessorTest_synchronizeDeleteForMe.kt +++ b/app/src/androidTest/java/org/thoughtcrime/securesms/messages/SyncMessageProcessorTest_synchronizeDeleteForMe.kt @@ -706,7 +706,8 @@ class SyncMessageProcessorTest_synchronizeDeleteForMe { thumbnailRestoreState = this.thumbnailRestoreState, archiveTransferState = this.archiveTransferState, uuid = uuid, - quoteTargetContentType = this.quoteTargetContentType + quoteTargetContentType = this.quoteTargetContentType, + metadata = null ) } diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index 095d8d861a..54c703909c 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -989,6 +989,10 @@ android:theme="@style/Signal.DayNight.NoActionBar" android:exported="false"/> + + { POINTER(PointerAttachment::class.java, "pointer"), TOMBSTONE(TombstoneAttachment::class.java, "tombstone"), URI(UriAttachment::class.java, "uri"), - ARCHIVED(ArchivedAttachment::class.java, "archived") + ARCHIVED(ArchivedAttachment::class.java, "archived"), + LOCAL_STICKER(LocalStickerAttachment::class.java, "local_sticker"), + WALLPAPER(WallpaperAttachment::class.java, "wallpaper") } @JvmStatic @@ -36,6 +38,8 @@ object AttachmentCreator : Parcelable.Creator { Subclass.TOMBSTONE -> TombstoneAttachment(source) Subclass.URI -> UriAttachment(source) Subclass.ARCHIVED -> ArchivedAttachment(source) + Subclass.LOCAL_STICKER -> LocalStickerAttachment(source) + Subclass.WALLPAPER -> WallpaperAttachment(source) } } diff --git a/app/src/main/java/org/thoughtcrime/securesms/attachments/AttachmentMetadata.kt b/app/src/main/java/org/thoughtcrime/securesms/attachments/AttachmentMetadata.kt new file mode 100644 index 0000000000..257b2dc851 --- /dev/null +++ b/app/src/main/java/org/thoughtcrime/securesms/attachments/AttachmentMetadata.kt @@ -0,0 +1,19 @@ +/* + * Copyright 2025 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ + +package org.thoughtcrime.securesms.attachments + +import android.os.Parcelable +import kotlinx.parcelize.Parcelize +import kotlinx.parcelize.RawValue + +/** + * Metadata for a specific attachment, specifically per data file. So there can be a + * many-to-one relationship from attachments to metadata. + */ +@Parcelize +class AttachmentMetadata( + val localBackupKey: @RawValue LocalBackupKey? +) : Parcelable diff --git a/app/src/main/java/org/thoughtcrime/securesms/attachments/DatabaseAttachment.kt b/app/src/main/java/org/thoughtcrime/securesms/attachments/DatabaseAttachment.kt index 1b78accc12..ef1f3db6bc 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/attachments/DatabaseAttachment.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/attachments/DatabaseAttachment.kt @@ -39,6 +39,10 @@ class DatabaseAttachment : Attachment { @JvmField val archiveTransferState: AttachmentTable.ArchiveTransferState + /** Metadata for this attachment, if null, no attempt was made to load the metadata and does not imply there is none */ + @JvmField + val metadata: AttachmentMetadata? + private val hasThumbnail: Boolean val displayOrder: Int @@ -76,7 +80,8 @@ class DatabaseAttachment : Attachment { thumbnailRestoreState: AttachmentTable.ThumbnailRestoreState, archiveTransferState: AttachmentTable.ArchiveTransferState, uuid: UUID?, - quoteTargetContentType: String? + quoteTargetContentType: String?, + metadata: AttachmentMetadata? ) : super( contentType = contentType, transferState = transferProgress, @@ -112,6 +117,7 @@ class DatabaseAttachment : Attachment { this.archiveCdn = archiveCdn this.thumbnailRestoreState = thumbnailRestoreState this.archiveTransferState = archiveTransferState + this.metadata = metadata } constructor(parcel: Parcel) : super(parcel) { @@ -124,6 +130,7 @@ class DatabaseAttachment : Attachment { archiveCdn = parcel.readInt().takeIf { it != NO_ARCHIVE_CDN } thumbnailRestoreState = AttachmentTable.ThumbnailRestoreState.deserialize(parcel.readInt()) archiveTransferState = AttachmentTable.ArchiveTransferState.deserialize(parcel.readInt()) + metadata = ParcelCompat.readParcelable(parcel, AttachmentMetadata::class.java.classLoader, AttachmentMetadata::class.java) } override fun writeToParcel(dest: Parcel, flags: Int) { @@ -137,6 +144,7 @@ class DatabaseAttachment : Attachment { dest.writeInt(archiveCdn ?: NO_ARCHIVE_CDN) dest.writeInt(thumbnailRestoreState.value) dest.writeInt(archiveTransferState.value) + dest.writeParcelable(metadata, 0) } override val uri: Uri? diff --git a/app/src/main/java/org/thoughtcrime/securesms/attachments/LocalBackupKey.kt b/app/src/main/java/org/thoughtcrime/securesms/attachments/LocalBackupKey.kt new file mode 100644 index 0000000000..9577c6e5c1 --- /dev/null +++ b/app/src/main/java/org/thoughtcrime/securesms/attachments/LocalBackupKey.kt @@ -0,0 +1,19 @@ +/* + * Copyright 2025 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ + +package org.thoughtcrime.securesms.attachments + +import okio.ByteString +import okio.ByteString.Companion.toByteString + +/** + * Combined key used to encrypt/decrypt attachments for local backups. + */ +@JvmInline +value class LocalBackupKey(val key: ByteArray) { + fun toByteString(): ByteString { + return key.toByteString() + } +} diff --git a/app/src/main/java/org/thoughtcrime/securesms/attachments/WallpaperAttachment.kt b/app/src/main/java/org/thoughtcrime/securesms/attachments/WallpaperAttachment.kt index a219b88d98..9aabf0aca2 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/attachments/WallpaperAttachment.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/attachments/WallpaperAttachment.kt @@ -5,6 +5,7 @@ package org.thoughtcrime.securesms.attachments +import android.os.Parcel import org.thoughtcrime.securesms.database.AttachmentTable import org.thoughtcrime.securesms.database.AttachmentTable.TransformProperties import org.thoughtcrime.securesms.util.MediaUtil @@ -12,34 +13,39 @@ import org.thoughtcrime.securesms.util.MediaUtil /** * A basically-empty [Attachment] that is solely used for inserting an attachment into the [AttachmentTable]. */ -class WallpaperAttachment() : Attachment( - contentType = MediaUtil.IMAGE_WEBP, - transferState = AttachmentTable.TRANSFER_PROGRESS_DONE, - size = 0, - fileName = null, - cdn = Cdn.CDN_0, - remoteLocation = null, - remoteKey = null, - remoteDigest = null, - incrementalDigest = null, - fastPreflightId = null, - voiceNote = false, - borderless = false, - videoGif = false, - width = 0, - height = 0, - incrementalMacChunkSize = 0, - quote = false, - quoteTargetContentType = null, - uploadTimestamp = 0, - caption = null, - stickerLocator = null, - blurHash = null, - audioHash = null, - transformProperties = TransformProperties.empty(), - uuid = null -) { +class WallpaperAttachment : Attachment { override val uri = null override val publicUri = null override val thumbnailUri = null + + constructor() : super( + contentType = MediaUtil.IMAGE_WEBP, + transferState = AttachmentTable.TRANSFER_PROGRESS_DONE, + size = 0, + fileName = null, + cdn = Cdn.CDN_0, + remoteLocation = null, + remoteKey = null, + remoteDigest = null, + incrementalDigest = null, + fastPreflightId = null, + voiceNote = false, + borderless = false, + videoGif = false, + width = 0, + height = 0, + incrementalMacChunkSize = 0, + quote = false, + quoteTargetContentType = null, + uploadTimestamp = 0, + caption = null, + stickerLocator = null, + blurHash = null, + audioHash = null, + transformProperties = TransformProperties.empty(), + uuid = null + ) + + @Suppress("unused") + constructor(parcel: Parcel) : super(parcel) } diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/BackupRepository.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/BackupRepository.kt index 97306b797b..5118d587bc 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/BackupRepository.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/BackupRepository.kt @@ -85,7 +85,6 @@ import org.thoughtcrime.securesms.backup.v2.stream.PlainTextBackupReader import org.thoughtcrime.securesms.backup.v2.stream.PlainTextBackupWriter import org.thoughtcrime.securesms.backup.v2.ui.BackupAlert import org.thoughtcrime.securesms.backup.v2.ui.subscription.MessageBackupsType -import org.thoughtcrime.securesms.backup.v2.util.ArchiveAttachmentInfo import org.thoughtcrime.securesms.components.settings.app.AppSettingsActivity import org.thoughtcrime.securesms.components.settings.app.subscription.RecurringInAppPaymentRepository import org.thoughtcrime.securesms.crypto.AttachmentSecretProvider @@ -770,24 +769,20 @@ object BackupRepository { append = { main.write(it) } ) - val maxBufferSize = 10_000 - var totalAttachmentCount = 0 - val attachmentInfos: MutableSet = mutableSetOf() - export( currentTime = System.currentTimeMillis(), isLocal = true, writer = writer, progressEmitter = localBackupProgressEmitter, cancellationSignal = cancellationSignal, - forTransfer = false, + backupMode = BackupMode.LOCAL, extraFrameOperation = null, messageInclusionCutoffTime = 0 ) { dbSnapshot -> val localArchivableAttachments = dbSnapshot .attachmentTable .getLocalArchivableAttachments() - .associateBy { MediaName.fromPlaintextHashAndRemoteKey(it.plaintextHash, it.remoteKey) } + .associateBy { MediaName.forLocalBackupFilename(it.plaintextHash, it.localBackupKey.key) } localBackupProgressEmitter.onAttachment(0, localArchivableAttachments.size.toLong()) @@ -834,7 +829,7 @@ object BackupRepository { currentTime = currentTime, isLocal = false, writer = writer, - forTransfer = false, + backupMode = BackupMode.REMOTE, progressEmitter = progressEmitter, cancellationSignal = cancellationSignal, extraFrameOperation = extraFrameOperation, @@ -865,7 +860,7 @@ object BackupRepository { currentTime = currentTime, isLocal = false, writer = writer, - forTransfer = true, + backupMode = BackupMode.LINK_SYNC, progressEmitter = progressEmitter, cancellationSignal = cancellationSignal, extraFrameOperation = null, @@ -882,7 +877,6 @@ object BackupRepository { messageBackupKey: MessageBackupKey = SignalStore.backup.messageBackupKey, plaintext: Boolean = false, currentTime: Long = System.currentTimeMillis(), - forTransfer: Boolean = false, progressEmitter: ExportProgressListener? = null, cancellationSignal: () -> Boolean = { false } ) { @@ -901,7 +895,7 @@ object BackupRepository { currentTime = currentTime, isLocal = false, writer = writer, - forTransfer = forTransfer, + backupMode = BackupMode.REMOTE, progressEmitter = progressEmitter, cancellationSignal = cancellationSignal, extraFrameOperation = null, @@ -925,7 +919,7 @@ object BackupRepository { currentTime: Long, isLocal: Boolean, writer: BackupExportWriter, - forTransfer: Boolean, + backupMode: BackupMode, messageInclusionCutoffTime: Long, progressEmitter: ExportProgressListener?, cancellationSignal: () -> Boolean, @@ -945,7 +939,7 @@ object BackupRepository { val selfAci = signalStoreSnapshot.accountValues.aci!! val selfRecipientId = dbSnapshot.recipientTable.getByAci(selfAci).get().toLong().let { RecipientId.from(it) } - val exportState = ExportState(backupTime = currentTime, forTransfer = forTransfer, selfRecipientId = selfRecipientId) + val exportState = ExportState(backupTime = currentTime, backupMode = backupMode, selfRecipientId = selfRecipientId) var frameCount = 0L @@ -2435,7 +2429,7 @@ data class ArchivedMediaObject(val mediaId: String, val cdn: Int) class ExportState( val backupTime: Long, - val forTransfer: Boolean, + val backupMode: BackupMode, val selfRecipientId: RecipientId ) { val recipientIds: MutableSet = hashSetOf() @@ -2507,6 +2501,18 @@ sealed interface RestoreTimestampResult { data object Failure : RestoreTimestampResult } +enum class BackupMode { + REMOTE, + LINK_SYNC, + LOCAL; + + val isLinkAndSync: Boolean + get() = this == LINK_SYNC + + val isLocalBackup: Boolean + get() = this == LOCAL +} + /** * Iterator that reads values from the given cursor. Expects that REMOTE_DIGEST is present and non-null, and ARCHIVE_CDN is present. * diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/LocalBackupV2Event.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/LocalBackupV2Event.kt index 51f5e38dde..c83fcd07fe 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/LocalBackupV2Event.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/LocalBackupV2Event.kt @@ -16,7 +16,6 @@ class LocalBackupV2Event(val type: Type, val count: Long = 0, val estimatedTotal CHAT_FOLDER, PROGRESS_MESSAGE, PROGRESS_ATTACHMENT, - PROGRESS_VERIFYING, FINISHED } } diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/exporters/ChatArchiveExporter.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/exporters/ChatArchiveExporter.kt index 36eae334ae..3d7e80162d 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/exporters/ChatArchiveExporter.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/exporters/ChatArchiveExporter.kt @@ -67,7 +67,8 @@ class ChatArchiveExporter(private val cursor: Cursor, private val db: SignalData db = db, chatColors = chatColors, chatColorId = customChatColorsId, - chatWallpaper = chatWallpaper + chatWallpaper = chatWallpaper, + backupMode = exportState.backupMode ) ) } diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/exporters/ChatItemArchiveExporter.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/exporters/ChatItemArchiveExporter.kt index 086ebc782d..94306802d3 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/exporters/ChatItemArchiveExporter.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/exporters/ChatItemArchiveExporter.kt @@ -36,6 +36,7 @@ import org.signal.core.util.requireString import org.signal.core.util.toByteArray import org.thoughtcrime.securesms.attachments.AttachmentId import org.thoughtcrime.securesms.attachments.DatabaseAttachment +import org.thoughtcrime.securesms.backup.v2.BackupMode import org.thoughtcrime.securesms.backup.v2.ExportOddities import org.thoughtcrime.securesms.backup.v2.ExportSkips import org.thoughtcrime.securesms.backup.v2.ExportState @@ -176,7 +177,7 @@ class ChatItemArchiveExporter( return buffer.remove() } - val extraData = fetchExtraMessageData(db, records.keys) + val extraData = fetchExtraMessageData(db = db, messageIds = records.keys) eventTimer.emit("extra-data") transformTimer.emit("ignore") @@ -368,7 +369,7 @@ class ChatItemArchiveExporter( } !record.sharedContacts.isNullOrEmpty() -> { - builder.contactMessage = record.toRemoteContactMessage(reactionRecords = extraData.reactionsById[id], attachments = extraData.attachmentsById[id]) ?: continue + builder.contactMessage = record.toRemoteContactMessage(reactionRecords = extraData.reactionsById[id], attachments = extraData.attachmentsById[id], backupMode = exportState.backupMode) ?: continue transformTimer.emit("contact") } @@ -382,7 +383,7 @@ class ChatItemArchiveExporter( Log.w(TAG, ExportSkips.directStoryReplyInNoteToSelf(record.dateSent)) continue } - builder.directStoryReplyMessage = record.toRemoteDirectStoryReplyMessage(reactionRecords = extraData.reactionsById[id], attachments = extraData.attachmentsById[record.id]) ?: continue + builder.directStoryReplyMessage = record.toRemoteDirectStoryReplyMessage(reactionRecords = extraData.reactionsById[id], attachments = extraData.attachmentsById[record.id], backupMode = exportState.backupMode) ?: continue transformTimer.emit("story") } @@ -430,7 +431,7 @@ class ChatItemArchiveExporter( val sticker = attachments?.firstOrNull { dbAttachment -> dbAttachment.isSticker } if (sticker?.stickerLocator != null) { - builder.stickerMessage = sticker.toRemoteStickerMessage(sentTimestamp = record.dateSent, reactions = extraData.reactionsById[id]) + builder.stickerMessage = sticker.toRemoteStickerMessage(sentTimestamp = record.dateSent, reactions = extraData.reactionsById[id], exportState.backupMode) } else { val standardMessage = record.toRemoteStandardMessage( exportState = exportState, @@ -521,7 +522,7 @@ class ChatItemArchiveExporter( val attachmentsFuture = executor.submitTyped { extraDataTimer.timeEvent("attachments") { - db.attachmentTable.getAttachmentsForMessages(messageIds, excludeTranscodingQuotes = true) + db.attachmentTable.getAttachmentsForMessagesArchive(messageIds) } } @@ -642,9 +643,9 @@ private fun BackupMessageRecord.toBasicChatItemBuilder(selfRecipientId: Recipien if (!MessageTypes.isExpirationTimerUpdate(record.type) && builder.expiresInMs != null && builder.expireStartDate != null) { val cutoffDuration = ChatItemArchiveExporter.EXPIRATION_CUTOFF.inWholeMilliseconds val expiresAt = builder.expireStartDate!! + builder.expiresInMs!! - val threshold = if (exportState.forTransfer) backupStartTime else backupStartTime + cutoffDuration + val threshold = if (exportState.backupMode.isLinkAndSync) backupStartTime else backupStartTime + cutoffDuration - if (expiresAt < threshold || (builder.expiresInMs!! <= cutoffDuration && !exportState.forTransfer)) { + if (expiresAt < threshold || (builder.expiresInMs!! <= cutoffDuration && !exportState.backupMode.isLinkAndSync)) { Log.w(TAG, ExportSkips.messageExpiresTooSoon(record.dateSent)) return null } @@ -954,22 +955,22 @@ private fun BackupMessageRecord.toRemoteLinkPreviews(attachments: List?, attachments: List?): ViewOnceMessage { - val attachment: MessageAttachment? = if (exportState.forTransfer) { + val attachment: MessageAttachment? = if (exportState.backupMode.isLinkAndSync) { attachments ?.firstOrNull() ?.takeUnless { !it.hasData && it.size == 0L && it.remoteDigest == null && it.width == 0 && it.height == 0 && it.blurHash == null } - ?.toRemoteMessageAttachment() + ?.toRemoteMessageAttachment(backupMode = exportState.backupMode) } else { null } @@ -980,13 +981,13 @@ private fun BackupMessageRecord.toRemoteViewOnceMessage(exportState: ExportState ) } -private fun BackupMessageRecord.toRemoteContactMessage(reactionRecords: List?, attachments: List?): ContactMessage? { +private fun BackupMessageRecord.toRemoteContactMessage(reactionRecords: List?, attachments: List?, backupMode: BackupMode): ContactMessage? { val sharedContact = toRemoteSharedContact(attachments) ?: return null return ContactMessage( contact = ContactAttachment( name = sharedContact.name.toRemote(), - avatar = (sharedContact.avatar?.attachment as? DatabaseAttachment)?.toRemoteMessageAttachment()?.pointer, + avatar = (sharedContact.avatar?.attachment as? DatabaseAttachment)?.toRemoteMessageAttachment(backupMode = backupMode)?.pointer, organization = sharedContact.organization ?: "", number = sharedContact.phoneNumbers.mapNotNull { phone -> ContactAttachment.Phone( @@ -1067,7 +1068,7 @@ private fun Contact.PostalAddress.Type.toRemote(): ContactAttachment.PostalAddre } } -private fun BackupMessageRecord.toRemoteDirectStoryReplyMessage(reactionRecords: List?, attachments: List?): DirectStoryReplyMessage? { +private fun BackupMessageRecord.toRemoteDirectStoryReplyMessage(reactionRecords: List?, attachments: List?, backupMode: BackupMode): DirectStoryReplyMessage? { if (this.body.isNullOrBlank()) { Log.w(TAG, ExportSkips.directStoryReplyHasNoBody(this.dateSent)) return null @@ -1089,7 +1090,7 @@ private fun BackupMessageRecord.toRemoteDirectStoryReplyMessage(reactionRecords: body = bodyText, bodyRanges = this.bodyRanges?.toRemoteBodyRanges(this.dateSent) ?: emptyList() ), - longText = longTextAttachment?.toRemoteFilePointer() + longText = longTextAttachment?.toRemoteFilePointer(backupMode = backupMode) ) } else { null @@ -1121,9 +1122,9 @@ private fun BackupMessageRecord.toRemoteStandardMessage(exportState: ExportState return StandardMessage( quote = this.toRemoteQuote(exportState, quotedAttachments), text = text.takeUnless { hasVoiceNote }, - attachments = messageAttachments.toRemoteAttachments().withFixedVoiceNotes(textPresent = text != null || longTextAttachment != null), - linkPreview = linkPreviews.map { it.toRemoteLinkPreview() }, - longText = longTextAttachment?.toRemoteFilePointer(), + attachments = messageAttachments.toRemoteAttachments(exportState.backupMode).withFixedVoiceNotes(textPresent = text != null || longTextAttachment != null), + linkPreview = linkPreviews.map { it.toRemoteLinkPreview(exportState.backupMode) }, + longText = longTextAttachment?.toRemoteFilePointer(backupMode = exportState.backupMode), reactions = reactionRecords.toRemote() ) } @@ -1194,7 +1195,7 @@ private fun BackupMessageRecord.toRemoteQuote(exportState: ExportState, attachme val attachments = if (remoteType == Quote.Type.VIEW_ONCE) { emptyList() } else { - attachments?.toRemoteQuoteAttachments() ?: emptyList() + attachments?.toRemoteQuoteAttachments(exportState.backupMode) ?: emptyList() } if (remoteType == Quote.Type.NORMAL && body == null && attachments.isEmpty()) { @@ -1250,7 +1251,7 @@ private fun PollRecord.toRemotePollMessage(reactionRecords: List ) } -private fun DatabaseAttachment.toRemoteStickerMessage(sentTimestamp: Long, reactions: List?): StickerMessage? { +private fun DatabaseAttachment.toRemoteStickerMessage(sentTimestamp: Long, reactions: List?, backupMode: BackupMode): StickerMessage? { val stickerLocator = this.stickerLocator!! val packId = try { @@ -1273,18 +1274,19 @@ private fun DatabaseAttachment.toRemoteStickerMessage(sentTimestamp: Long, react packKey = packKey.toByteString(), stickerId = stickerLocator.stickerId, emoji = stickerLocator.emoji, - data_ = this.toRemoteMessageAttachment().pointer + data_ = this.toRemoteMessageAttachment(backupMode = backupMode).pointer ), reactions = reactions.toRemote() ) } -private fun List.toRemoteQuoteAttachments(): List { +private fun List.toRemoteQuoteAttachments(backupMode: BackupMode): List { return this.map { attachment -> Quote.QuotedAttachment( contentType = attachment.quoteTargetContentType, fileName = attachment.fileName, thumbnail = attachment.toRemoteMessageAttachment( + backupMode = backupMode, flagOverride = MessageAttachment.Flag.NONE, contentTypeOverride = attachment.contentType ) @@ -1292,8 +1294,8 @@ private fun List.toRemoteQuoteAttachments(): List.toRemoteAttachments(): List { +private fun List.toRemoteAttachments(backupMode: BackupMode): List { return this.map { attachment -> - attachment.toRemoteMessageAttachment() + attachment.toRemoteMessageAttachment(backupMode = backupMode) } } diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/local/LocalArchiver.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/local/LocalArchiver.kt index 9677641ad7..9e020d5693 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/local/LocalArchiver.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/local/LocalArchiver.kt @@ -5,17 +5,22 @@ package org.thoughtcrime.securesms.backup.v2.local +import okio.ByteString.Companion.toByteString import org.greenrobot.eventbus.EventBus +import org.signal.core.models.backup.BackupId import org.signal.core.models.backup.MediaName -import org.signal.core.util.Base64 import org.signal.core.util.Stopwatch import org.signal.core.util.StreamUtil import org.signal.core.util.logging.Log +import org.signal.core.util.readFully +import org.thoughtcrime.securesms.attachments.AttachmentId import org.thoughtcrime.securesms.backup.v2.BackupRepository import org.thoughtcrime.securesms.backup.v2.LocalBackupV2Event import org.thoughtcrime.securesms.backup.v2.local.proto.FilesFrame import org.thoughtcrime.securesms.backup.v2.local.proto.Metadata import org.thoughtcrime.securesms.database.AttachmentTable +import org.thoughtcrime.securesms.keyvalue.SignalStore +import org.thoughtcrime.securesms.util.Util import org.whispersystems.signalservice.api.crypto.AttachmentCipherOutputStream import org.whispersystems.signalservice.api.crypto.AttachmentCipherStreamUtil import org.whispersystems.signalservice.internal.crypto.PaddingInputStream @@ -23,8 +28,12 @@ import java.io.IOException import java.io.InputStream import java.io.OutputStream import java.util.Collections +import javax.crypto.Cipher +import javax.crypto.spec.IvParameterSpec +import javax.crypto.spec.SecretKeySpec -typealias ArchiveResult = org.signal.core.util.Result +typealias ArchiveResult = org.signal.core.util.Result +typealias RestoreResult = org.signal.core.util.Result /** * Handle importing and exporting folder-based archives using backupv2 format. @@ -34,6 +43,8 @@ object LocalArchiver { private val TAG = Log.tag(LocalArchiver::class) private const val VERSION = 1 + private const val MAX_CREATE_FAILURES = 10 + /** * Export archive to the provided [snapshotFileSystem] and store new files in [filesFileSystem]. */ @@ -44,12 +55,15 @@ object LocalArchiver { var mainStream: OutputStream? = null var filesStream: OutputStream? = null + val createFailures: MutableSet = Collections.synchronizedSet(HashSet()) + val readWriteFailures: MutableSet = Collections.synchronizedSet(HashSet()) + try { - metadataStream = snapshotFileSystem.metadataOutputStream() ?: return ArchiveResult.failure(FailureCause.METADATA_STREAM) - metadataStream.use { it.write(Metadata(VERSION).encode()) } + metadataStream = snapshotFileSystem.metadataOutputStream() ?: return ArchiveResult.failure(ArchiveFailure.MetadataStream) + metadataStream.use { it.write(Metadata(version = VERSION, backupId = getEncryptedBackupId()).encode()) } stopwatch.split("metadata") - mainStream = snapshotFileSystem.mainOutputStream() ?: return ArchiveResult.failure(FailureCause.MAIN_STREAM) + mainStream = snapshotFileSystem.mainOutputStream() ?: return ArchiveResult.failure(ArchiveFailure.MainStream) Log.i(TAG, "Listing all current files") val allFiles = filesFileSystem.allFiles() @@ -59,11 +73,11 @@ object LocalArchiver { Log.i(TAG, "Starting frame export") BackupRepository.exportForLocalBackup(mainStream, LocalExportProgressListener(), cancellationSignal) { attachment, source -> - if (cancellationSignal()) { + if (cancellationSignal() || createFailures.size > MAX_CREATE_FAILURES) { return@exportForLocalBackup } - val mediaName = MediaName.fromPlaintextHashAndRemoteKey(attachment.plaintextHash, attachment.remoteKey) + val mediaName = MediaName.forLocalBackupFilename(attachment.plaintextHash, attachment.localBackupKey.key) mediaNames.add(mediaName) @@ -73,23 +87,21 @@ object LocalArchiver { } source()?.use { sourceStream -> - val combinedKey = Base64.decode(attachment.remoteKey) val destination: OutputStream? = filesFileSystem.fileOutputStream(mediaName) if (destination == null) { - Log.w(TAG, "Unable to create output file for attachment") - // todo [local-backup] should we abort here? + Log.w(TAG, "Unable to create output file for ${attachment.attachmentId}") + createFailures.add(attachment.attachmentId) } else { - // todo [local-backup] but deal with attachment disappearing/deleted by normal app use try { PaddingInputStream(sourceStream, attachment.size).use { input -> - AttachmentCipherOutputStream(combinedKey, null, destination).use { output -> - StreamUtil.copy(input, output) + AttachmentCipherOutputStream(attachment.localBackupKey.key, null, destination).use { output -> + StreamUtil.copy(input, output, false, false) } } } catch (e: IOException) { - Log.w(TAG, "Unable to save attachment", e) - // todo [local-backup] should we abort here? + Log.w(TAG, "Unable to save ${attachment.attachmentId}", e) + readWriteFailures.add(attachment.attachmentId) } } } @@ -97,7 +109,11 @@ object LocalArchiver { } stopwatch.split("frames-and-files") - filesStream = snapshotFileSystem.filesOutputStream() ?: return ArchiveResult.failure(FailureCause.FILES_STREAM) + if (createFailures.size > MAX_CREATE_FAILURES) { + return ArchiveResult.failure(ArchiveFailure.TooManyCreateFailures(createFailures)) + } + + filesStream = snapshotFileSystem.filesOutputStream() ?: return ArchiveResult.failure(ArchiveFailure.FilesStream) ArchivedFilesWriter(filesStream).use { writer -> mediaNames.forEach { name -> writer.write(FilesFrame(mediaName = name.name)) } } @@ -109,22 +125,56 @@ object LocalArchiver { } if (cancellationSignal()) { - return ArchiveResult.failure(FailureCause.CANCELLED) + return ArchiveResult.failure(ArchiveFailure.Cancelled) } - return ArchiveResult.success(Unit) + return if (createFailures.isNotEmpty() || readWriteFailures.isNotEmpty()) { + ArchiveResult.success(ArchiveSuccess.PartialSuccess(createFailures, readWriteFailures)) + } else { + ArchiveResult.success(ArchiveSuccess.FullSuccess) + } + } + + private fun getEncryptedBackupId(): Metadata.EncryptedBackupId { + val metadataKey = SignalStore.backup.messageBackupKey.deriveLocalBackupMetadataKey() + val iv = Util.getSecretBytes(12) + val backupId = SignalStore.backup.messageBackupKey.deriveBackupId(SignalStore.account.requireAci()) + + val cipher = Cipher.getInstance("AES/CTR/NoPadding") + cipher.init(Cipher.ENCRYPT_MODE, SecretKeySpec(metadataKey, "AES"), IvParameterSpec(iv)) + val cipherText = cipher.doFinal(backupId.value) + + return Metadata.EncryptedBackupId(iv = iv.toByteString(), encryptedId = cipherText.toByteString()) } /** * Import archive data from a folder on the system. Does not restore attachments. */ - fun import(snapshotFileSystem: SnapshotFileSystem, selfData: BackupRepository.SelfData): ArchiveResult { + fun import(snapshotFileSystem: SnapshotFileSystem, selfData: BackupRepository.SelfData): RestoreResult { var metadataStream: InputStream? = null try { - metadataStream = snapshotFileSystem.metadataInputStream() ?: return ArchiveResult.failure(FailureCause.METADATA_STREAM) + metadataStream = snapshotFileSystem.metadataInputStream() ?: return RestoreResult.failure(RestoreFailure.MetadataStream) + val metadata = Metadata.ADAPTER.decode(metadataStream.readFully(autoClose = false)) - val mainStreamLength = snapshotFileSystem.mainLength() ?: return ArchiveResult.failure(FailureCause.MAIN_STREAM) + if (metadata.version > VERSION) { + Log.w(TAG, "Local backup version does not match, bailing supported: $VERSION backup: ${metadata.version}") + return RestoreResult.failure(RestoreFailure.VersionMismatch(metadata.version, VERSION)) + } + + if (metadata.backupId == null) { + Log.w(TAG, "Local backup metadata missing encrypted backup id") + return RestoreResult.failure(RestoreFailure.BackupIdMissing) + } + + val backupId = decryptBackupId(metadata.backupId) + + if (!backupId.value.contentEquals(SignalStore.backup.messageBackupKey.deriveBackupId(SignalStore.account.requireAci()).value)) { + Log.w(TAG, "Local backup metadata backup id does not match derived backup id, likely from another account") + return RestoreResult.failure(RestoreFailure.BackupIdMismatch) + } + + val mainStreamLength = snapshotFileSystem.mainLength() ?: return ArchiveResult.failure(RestoreFailure.MainStream) BackupRepository.importLocal( mainStreamFactory = { snapshotFileSystem.mainInputStream()!! }, @@ -135,18 +185,52 @@ object LocalArchiver { metadataStream?.close() } - return ArchiveResult.success(Unit) + return RestoreResult.success(RestoreSuccess.FullSuccess) + } + + private fun decryptBackupId(encryptedBackupId: Metadata.EncryptedBackupId): BackupId { + val metadataKey = SignalStore.backup.messageBackupKey.deriveLocalBackupMetadataKey() + val iv = encryptedBackupId.iv.toByteArray() + val backupIdCipher = encryptedBackupId.encryptedId.toByteArray() + + val cipher = Cipher.getInstance("AES/CTR/NoPadding") + cipher.init(Cipher.DECRYPT_MODE, SecretKeySpec(metadataKey, "AES"), IvParameterSpec(iv)) + val plaintext = cipher.doFinal(backupIdCipher) + + return BackupId(plaintext) } private val AttachmentTable.LocalArchivableAttachment.cipherLength: Long get() = AttachmentCipherStreamUtil.getCiphertextLength(PaddingInputStream.getPaddedSize(size)) - enum class FailureCause { - METADATA_STREAM, MAIN_STREAM, FILES_STREAM, CANCELLED + sealed interface ArchiveSuccess { + data object FullSuccess : ArchiveSuccess + data class PartialSuccess(val createFailures: Set, val readWriteFailures: Set) : ArchiveSuccess + } + + sealed interface ArchiveFailure { + data object MetadataStream : ArchiveFailure + data object MainStream : ArchiveFailure + data object FilesStream : ArchiveFailure + data object Cancelled : ArchiveFailure + data class TooManyCreateFailures(val attachmentId: Set) : ArchiveFailure + } + + sealed interface RestoreSuccess { + data object FullSuccess : RestoreSuccess + } + + sealed interface RestoreFailure { + data object MetadataStream : RestoreFailure + data object MainStream : RestoreFailure + data object Cancelled : RestoreFailure + data object BackupIdMissing : RestoreFailure + data object BackupIdMismatch : RestoreFailure + data class VersionMismatch(val backupVersion: Int, val supportedVersion: Int) : RestoreFailure } private class LocalExportProgressListener : BackupRepository.ExportProgressListener { - private var lastAttachmentUpdate: Long = 0 + private var lastVerboseUpdate: Long = 0 override fun onAccount() { EventBus.getDefault().post(LocalBackupV2Event(LocalBackupV2Event.Type.PROGRESS_ACCOUNT)) @@ -177,15 +261,16 @@ object LocalArchiver { } override fun onMessage(currentProgress: Long, approximateCount: Long) { - if (currentProgress == 0L) { - EventBus.getDefault().post(LocalBackupV2Event(LocalBackupV2Event.Type.PROGRESS_MESSAGE)) + if (lastVerboseUpdate > System.currentTimeMillis() || lastVerboseUpdate + 1000 < System.currentTimeMillis() || currentProgress >= approximateCount) { + EventBus.getDefault().post(LocalBackupV2Event(LocalBackupV2Event.Type.PROGRESS_MESSAGE, currentProgress, approximateCount)) + lastVerboseUpdate = System.currentTimeMillis() } } override fun onAttachment(currentProgress: Long, totalCount: Long) { - if (lastAttachmentUpdate > System.currentTimeMillis() || lastAttachmentUpdate + 1000 < System.currentTimeMillis() || currentProgress >= totalCount) { + if (lastVerboseUpdate > System.currentTimeMillis() || lastVerboseUpdate + 1000 < System.currentTimeMillis() || currentProgress >= totalCount) { EventBus.getDefault().post(LocalBackupV2Event(LocalBackupV2Event.Type.PROGRESS_ATTACHMENT, currentProgress, totalCount)) - lastAttachmentUpdate = System.currentTimeMillis() + lastVerboseUpdate = System.currentTimeMillis() } } } diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/AccountDataArchiveProcessor.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/AccountDataArchiveProcessor.kt index ece0c3b286..36e1856a80 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/AccountDataArchiveProcessor.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/AccountDataArchiveProcessor.kt @@ -142,7 +142,8 @@ object AccountDataArchiveProcessor { db = db, chatColors = chatColors, chatColorId = chatColors?.id?.takeIf { it.isValid(exportState) } ?: ChatColors.Id.NotSet, - chatWallpaper = chatWallpaper + chatWallpaper = chatWallpaper, + backupMode = exportState.backupMode ) ), donationSubscriberData = donationSubscriber?.toSubscriberData(signalStore.inAppPaymentValues.isDonationSubscriptionManuallyCancelled()), diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/util/ArchiveConverterExtensions.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/util/ArchiveConverterExtensions.kt index ffe267b557..19f3b01499 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/util/ArchiveConverterExtensions.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/util/ArchiveConverterExtensions.kt @@ -20,6 +20,7 @@ import org.thoughtcrime.securesms.attachments.Cdn import org.thoughtcrime.securesms.attachments.DatabaseAttachment import org.thoughtcrime.securesms.attachments.PointerAttachment import org.thoughtcrime.securesms.attachments.TombstoneAttachment +import org.thoughtcrime.securesms.backup.v2.BackupMode import org.thoughtcrime.securesms.backup.v2.ExportState import org.thoughtcrime.securesms.backup.v2.proto.FilePointer import org.thoughtcrime.securesms.conversation.colors.AvatarColor @@ -78,7 +79,8 @@ fun FilePointer?.toLocalAttachment( quote = quote, quoteTargetContentType = quoteTargetContentType, uuid = UuidUtil.fromByteStringOrNull(uuid), - fileName = fileName + fileName = fileName, + localBackupKey = this.locatorInfo.localKey?.toByteArray() ) } AttachmentType.TRANSIT -> { @@ -133,10 +135,7 @@ fun FilePointer?.toLocalAttachment( } } -/** - * @param mediaArchiveEnabled True if this user has enable media backup, otherwise false. - */ -fun DatabaseAttachment.toRemoteFilePointer(contentTypeOverride: String? = null): FilePointer { +fun DatabaseAttachment.toRemoteFilePointer(contentTypeOverride: String? = null, backupMode: BackupMode): FilePointer { val builder = FilePointer.Builder() builder.contentType = contentTypeOverride ?: this.contentType?.takeUnless { it.isBlank() } builder.incrementalMac = this.incrementalDigest?.takeIf { it.isNotEmpty() && this.incrementalMacChunkSize > 0 }?.toByteString() @@ -146,12 +145,12 @@ fun DatabaseAttachment.toRemoteFilePointer(contentTypeOverride: String? = null): builder.height = this.height.takeIf { it > 0 } builder.caption = this.caption builder.blurHash = this.blurHash?.hash - builder.locatorInfo = this.toLocatorInfo() + builder.locatorInfo = this.toLocatorInfo(backupMode) return builder.build() } -fun DatabaseAttachment.toLocatorInfo(): FilePointer.LocatorInfo { +fun DatabaseAttachment.toLocatorInfo(backupMode: BackupMode): FilePointer.LocatorInfo { val attachmentType = this.toRemoteAttachmentType() if (attachmentType == AttachmentType.INVALID) { @@ -183,6 +182,14 @@ fun DatabaseAttachment.toLocatorInfo(): FilePointer.LocatorInfo { AttachmentType.INVALID -> Unit } + if (backupMode.isLocalBackup && this.dataHash != null && this.metadata?.localBackupKey != null) { + if (locatorBuilder.plaintextHash == null) { + locatorBuilder.plaintextHash = Base64.decode(this.dataHash).toByteString() + } + + locatorBuilder.localKey = this.metadata.localBackupKey.toByteString() + } + return locatorBuilder.build() } diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/util/ChatStyleConverter.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/util/ChatStyleConverter.kt index 7fc8db9566..6a5613e4c7 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/util/ChatStyleConverter.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/util/ChatStyleConverter.kt @@ -7,6 +7,7 @@ package org.thoughtcrime.securesms.backup.v2.util import org.signal.core.util.logging.Log import org.thoughtcrime.securesms.attachments.AttachmentId +import org.thoughtcrime.securesms.backup.v2.BackupMode import org.thoughtcrime.securesms.backup.v2.ImportState import org.thoughtcrime.securesms.backup.v2.proto.ChatStyle import org.thoughtcrime.securesms.backup.v2.proto.FilePointer @@ -34,7 +35,8 @@ object ChatStyleConverter { db: SignalDatabase, chatColors: ChatColors?, chatColorId: ChatColors.Id, - chatWallpaper: Wallpaper? + chatWallpaper: Wallpaper?, + backupMode: BackupMode ): ChatStyle? { if (chatColors == null && chatWallpaper == null) { return null @@ -72,7 +74,7 @@ object ChatStyleConverter { chatStyleBuilder.wallpaperPreset = chatWallpaper.linearGradient.toRemoteWallpaperPreset() } chatWallpaper.file_ != null -> { - chatStyleBuilder.wallpaperPhoto = chatWallpaper.file_.toFilePointer(db) + chatStyleBuilder.wallpaperPhoto = chatWallpaper.file_.toFilePointer(db, backupMode) } } @@ -251,10 +253,10 @@ private fun Wallpaper.LinearGradient.toRemoteWallpaperPreset(): ChatStyle.Wallpa } } -private fun Wallpaper.File.toFilePointer(db: SignalDatabase): FilePointer? { +private fun Wallpaper.File.toFilePointer(db: SignalDatabase, backupMode: BackupMode): FilePointer? { val attachmentId: AttachmentId = UriUtil.parseOrNull(this.uri)?.let { PartUriParser(it).partId } ?: return null val attachment = db.attachmentTable.getAttachment(attachmentId) - return attachment?.toRemoteFilePointer() + return attachment?.toRemoteFilePointer(backupMode = backupMode) } private fun ChatStyle.Builder.hasBubbleColorSet(): Boolean { diff --git a/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/backups/BackupsSettingsFragment.kt b/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/backups/BackupsSettingsFragment.kt index ac7b3e8776..1e9726c669 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/backups/BackupsSettingsFragment.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/backups/BackupsSettingsFragment.kt @@ -104,6 +104,7 @@ class BackupsSettingsFragment : ComposeFragment() { } }, onOnDeviceBackupsRowClick = { findNavController().safeNavigate(R.id.action_backupsSettingsFragment_to_backupsPreferenceFragment) }, + onNewOnDeviceBackupsRowClick = { findNavController().safeNavigate(R.id.action_backupsSettingsFragment_to_internalLocalBackupFragment) }, onBackupTierInternalOverrideChanged = { viewModel.onBackupTierInternalOverrideChanged(it) } ) } @@ -115,6 +116,7 @@ private fun BackupsSettingsContent( onNavigationClick: () -> Unit = {}, onBackupsRowClick: () -> Unit = {}, onOnDeviceBackupsRowClick: () -> Unit = {}, + onNewOnDeviceBackupsRowClick: () -> Unit = {}, onBackupTierInternalOverrideChanged: (MessageBackupTier?) -> Unit = {} ) { Scaffolds.Settings( @@ -232,6 +234,16 @@ private fun BackupsSettingsContent( onClick = onOnDeviceBackupsRowClick ) } + + if (backupsSettingsState.showNewLocalBackup) { + item { + Rows.TextRow( + text = "INTERNAL ONLY - New Local Backup", + label = "Use new local backup format", + onClick = onNewOnDeviceBackupsRowClick + ) + } + } } } } diff --git a/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/backups/BackupsSettingsState.kt b/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/backups/BackupsSettingsState.kt index fb79fd6bd1..bd834f3f3b 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/backups/BackupsSettingsState.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/backups/BackupsSettingsState.kt @@ -17,5 +17,6 @@ data class BackupsSettingsState( val backupState: BackupState, val lastBackupAt: Duration = SignalStore.backup.lastBackupTime.milliseconds, val showBackupTierInternalOverride: Boolean = false, - val backupTierInternalOverride: MessageBackupTier? = null + val backupTierInternalOverride: MessageBackupTier? = null, + val showNewLocalBackup: Boolean = false ) diff --git a/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/backups/BackupsSettingsViewModel.kt b/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/backups/BackupsSettingsViewModel.kt index 18d8404231..f13dc40be6 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/backups/BackupsSettingsViewModel.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/backups/BackupsSettingsViewModel.kt @@ -21,6 +21,7 @@ import org.thoughtcrime.securesms.keyvalue.SignalStore import org.thoughtcrime.securesms.recipients.Recipient import org.thoughtcrime.securesms.storage.StorageSyncHelper import org.thoughtcrime.securesms.util.Environment +import org.thoughtcrime.securesms.util.RemoteConfig import kotlin.time.Duration.Companion.milliseconds class BackupsSettingsViewModel : ViewModel() { @@ -45,7 +46,8 @@ class BackupsSettingsViewModel : ViewModel() { backupState = enabledState, lastBackupAt = SignalStore.backup.lastBackupTime.milliseconds, showBackupTierInternalOverride = Environment.IS_STAGING, - backupTierInternalOverride = SignalStore.backup.backupTierInternalOverride + backupTierInternalOverride = SignalStore.backup.backupTierInternalOverride, + showNewLocalBackup = RemoteConfig.internalUser || Environment.IS_NIGHTLY ) } } diff --git a/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/backups/local/InternalNewLocalBackupCreateFragment.kt b/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/backups/local/InternalNewLocalBackupCreateFragment.kt new file mode 100644 index 0000000000..9f8291d6d0 --- /dev/null +++ b/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/backups/local/InternalNewLocalBackupCreateFragment.kt @@ -0,0 +1,276 @@ +/* + * Copyright 2025 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ + +package org.thoughtcrime.securesms.components.settings.app.backups.local + +import android.app.Activity +import android.content.Context +import android.content.Intent +import android.net.Uri +import android.os.Build +import android.os.Bundle +import android.provider.DocumentsContract +import android.widget.Toast +import androidx.activity.result.ActivityResultLauncher +import androidx.activity.result.contract.ActivityResultContracts +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.lazy.LazyColumn +import androidx.compose.runtime.Composable +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember +import androidx.compose.runtime.setValue +import androidx.compose.ui.Modifier +import androidx.compose.ui.graphics.vector.ImageVector +import androidx.compose.ui.platform.LocalContext +import androidx.compose.ui.res.vectorResource +import androidx.lifecycle.compose.collectAsStateWithLifecycle +import kotlinx.coroutines.flow.map +import org.greenrobot.eventbus.EventBus +import org.greenrobot.eventbus.Subscribe +import org.greenrobot.eventbus.ThreadMode +import org.signal.core.ui.compose.DayNightPreviews +import org.signal.core.ui.compose.Previews +import org.signal.core.ui.compose.Rows +import org.signal.core.ui.compose.Scaffolds +import org.signal.core.util.logging.Log +import org.thoughtcrime.securesms.R +import org.thoughtcrime.securesms.backup.v2.LocalBackupV2Event +import org.thoughtcrime.securesms.compose.ComposeFragment +import org.thoughtcrime.securesms.conversation.v2.registerForLifecycle +import org.thoughtcrime.securesms.jobs.LocalBackupJob +import org.thoughtcrime.securesms.keyvalue.SignalStore +import org.thoughtcrime.securesms.service.LocalBackupListener +import org.thoughtcrime.securesms.util.DateUtils +import org.thoughtcrime.securesms.util.StorageUtil +import org.thoughtcrime.securesms.util.formatHours +import java.time.LocalTime +import java.util.Locale + +/** + * App settings internal screen for enabling and creating new local backups. + */ +class InternalNewLocalBackupCreateFragment : ComposeFragment() { + + private val TAG = Log.tag(InternalNewLocalBackupCreateFragment::class) + + private lateinit var chooseBackupLocationLauncher: ActivityResultLauncher + + private var createStatus by mutableStateOf("None") + private val directoryFlow = SignalStore.backup.newLocalBackupsDirectoryFlow.map { if (Build.VERSION.SDK_INT >= 24 && it != null) StorageUtil.getDisplayPath(requireContext(), Uri.parse(it)) else it } + + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) + + chooseBackupLocationLauncher = registerForActivityResult( + ActivityResultContracts.StartActivityForResult() + ) { result -> + if (result.resultCode == Activity.RESULT_OK && result.data?.data != null) { + handleBackupLocationSelected(result.data!!.data!!) + } else { + Log.w(TAG, "Backup location selection cancelled or failed") + } + } + + EventBus.getDefault().registerForLifecycle(subscriber = this, lifecycleOwner = this) + } + + @Subscribe(threadMode = ThreadMode.MAIN) + fun onEvent(event: LocalBackupV2Event) { + createStatus = "${event.type}: ${event.count} / ${event.estimatedTotalCount}" + } + + @Composable + override fun FragmentContent() { + val context = LocalContext.current + val backupsEnabled by SignalStore.backup.newLocalBackupsEnabledFlow.collectAsStateWithLifecycle(SignalStore.backup.newLocalBackupsEnabled) + val selectedDirectory by directoryFlow.collectAsStateWithLifecycle(SignalStore.backup.newLocalBackupsDirectory) + val lastBackupTime by SignalStore.backup.newLocalBackupsLastBackupTimeFlow.collectAsStateWithLifecycle(SignalStore.backup.newLocalBackupsLastBackupTime) + val lastBackupTimeString = remember(lastBackupTime) { calculateLastBackupTimeString(context, lastBackupTime) } + val backupTime = remember { LocalTime.of(SignalStore.settings.backupHour, SignalStore.settings.backupMinute).formatHours(requireContext()) } + + InternalLocalBackupScreen( + backupsEnabled = backupsEnabled, + selectedDirectory = selectedDirectory, + lastBackupTimeString = lastBackupTimeString, + backupTime = backupTime, + createStatus = createStatus, + callback = CallbackImpl() + ) + } + + private fun launchBackupDirectoryPicker() { + val intent = Intent(Intent.ACTION_OPEN_DOCUMENT_TREE) + + if (Build.VERSION.SDK_INT >= 26) { + val latestDirectory = SignalStore.settings.latestSignalBackupDirectory + if (latestDirectory != null) { + intent.putExtra(DocumentsContract.EXTRA_INITIAL_URI, latestDirectory) + } + } + + intent.addFlags( + Intent.FLAG_GRANT_PERSISTABLE_URI_PERMISSION or + Intent.FLAG_GRANT_WRITE_URI_PERMISSION or + Intent.FLAG_GRANT_READ_URI_PERMISSION + ) + + try { + Log.d(TAG, "Launching backup directory picker") + chooseBackupLocationLauncher.launch(intent) + } catch (e: Exception) { + Log.w(TAG, "Failed to launch backup directory picker", e) + Toast.makeText(requireContext(), R.string.BackupDialog_no_file_picker_available, Toast.LENGTH_LONG).show() + } + } + + private fun handleBackupLocationSelected(uri: Uri) { + Log.i(TAG, "Backup location selected: $uri") + + val takeFlags = Intent.FLAG_GRANT_READ_URI_PERMISSION or Intent.FLAG_GRANT_WRITE_URI_PERMISSION + + requireContext().contentResolver.takePersistableUriPermission(uri, takeFlags) + + SignalStore.backup.newLocalBackupsDirectory = uri.toString() + + Toast.makeText(requireContext(), "Directory selected: $uri", Toast.LENGTH_SHORT).show() + } + + private fun calculateLastBackupTimeString(context: Context, lastBackupTimestamp: Long): String { + return if (lastBackupTimestamp > 0) { + val relativeTime = DateUtils.getDatelessRelativeTimeSpanFormattedDate( + context, + Locale.getDefault(), + lastBackupTimestamp + ) + + if (relativeTime.isRelative) { + relativeTime.value + } else { + val day = DateUtils.getDayPrecisionTimeString(context, Locale.getDefault(), lastBackupTimestamp) + val time = relativeTime.value + + context.getString(R.string.RemoteBackupsSettingsFragment__s_at_s, day, time) + } + } else { + context.getString(R.string.RemoteBackupsSettingsFragment__never) + } + } + + private inner class CallbackImpl : Callback { + override fun onNavigationClick() { + requireActivity().onBackPressedDispatcher.onBackPressed() + } + + override fun onToggleBackupsClick(enabled: Boolean) { + SignalStore.backup.newLocalBackupsEnabled = enabled + if (enabled) { + LocalBackupListener.schedule(requireContext()) + } + } + + override fun onSelectDirectoryClick() { + launchBackupDirectoryPicker() + } + + override fun onEnqueueBackupClick() { + createStatus = "Starting..." + LocalBackupJob.enqueueArchive(false) + } + } +} + +private interface Callback { + fun onNavigationClick() + fun onToggleBackupsClick(enabled: Boolean) + fun onSelectDirectoryClick() + fun onEnqueueBackupClick() + + object Empty : Callback { + override fun onNavigationClick() = Unit + override fun onToggleBackupsClick(enabled: Boolean) = Unit + override fun onSelectDirectoryClick() = Unit + override fun onEnqueueBackupClick() = Unit + } +} + +@Composable +private fun InternalLocalBackupScreen( + backupsEnabled: Boolean = false, + selectedDirectory: String? = null, + lastBackupTimeString: String = "Never", + backupTime: String = "Unknown", + createStatus: String = "None", + callback: Callback +) { + Scaffolds.Settings( + title = "New Local Backups", + navigationIcon = ImageVector.vectorResource(R.drawable.symbol_arrow_start_24), + onNavigationClick = callback::onNavigationClick + ) { paddingValues -> + LazyColumn( + modifier = Modifier.padding(paddingValues) + ) { + item { + Rows.ToggleRow( + checked = backupsEnabled, + text = "Enable New Local Backups", + label = if (backupsEnabled) "Backups are enabled" else "Backups are disabled", + onCheckChanged = callback::onToggleBackupsClick + ) + } + + item { + Rows.TextRow( + text = "Last Backup", + label = lastBackupTimeString + ) + } + + item { + Rows.TextRow( + text = "Backup Schedule Time (same as v1)", + label = backupTime + ) + } + + item { + Rows.TextRow( + text = "Select Backup Directory", + label = selectedDirectory ?: "No directory selected", + onClick = callback::onSelectDirectoryClick + ) + } + + item { + Rows.TextRow( + text = "Create Backup Now", + label = "Enqueue LocalArchiveJob", + onClick = callback::onEnqueueBackupClick + ) + } + + item { + Rows.TextRow( + text = "Create Status", + label = createStatus + ) + } + } + } +} + +@DayNightPreviews +@Composable +fun InternalLocalBackupScreenPreview() { + Previews.Preview { + InternalLocalBackupScreen( + backupsEnabled = true, + selectedDirectory = "/storage/emulated/0/Signal/Backups", + lastBackupTimeString = "1 hour ago", + callback = Callback.Empty + ) + } +} diff --git a/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/backup/InternalBackupPlaygroundFragment.kt b/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/backup/InternalBackupPlaygroundFragment.kt index 38e0c1aa19..bc78c71a84 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/backup/InternalBackupPlaygroundFragment.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/backup/InternalBackupPlaygroundFragment.kt @@ -84,6 +84,7 @@ import org.thoughtcrime.securesms.jobs.BackupRestoreMediaJob import org.thoughtcrime.securesms.jobs.LocalBackupJob import org.thoughtcrime.securesms.keyvalue.BackupValues import org.thoughtcrime.securesms.keyvalue.SignalStore +import org.thoughtcrime.securesms.registration.ui.restore.local.InternalNewLocalRestoreActivity import org.thoughtcrime.securesms.util.Util class InternalBackupPlaygroundFragment : ComposeFragment() { @@ -192,7 +193,7 @@ class InternalBackupPlaygroundFragment : ComposeFragment() { savePlaintextCopyLauncher.launch(intent) }, - onExportNewStyleLocalBackupClicked = { LocalBackupJob.enqueueArchive() }, + onExportNewStyleLocalBackupClicked = { LocalBackupJob.enqueueArchive(false) }, onWipeDataAndRestoreFromRemoteClicked = { MaterialAlertDialogBuilder(context) .setTitle("Are you sure?") @@ -229,7 +230,9 @@ class InternalBackupPlaygroundFragment : ComposeFragment() { MaterialAlertDialogBuilder(context) .setTitle("Are you sure?") .setMessage("After you choose a file to import, this will delete all of your chats, then restore them from the file! Only do this on a test device!") - .setPositiveButton("Wipe and restore") { _, _ -> viewModel.import(SignalStore.settings.signalBackupDirectory!!) } + .setPositiveButton("Wipe and restore") { _, _ -> + startActivity(InternalNewLocalRestoreActivity.getIntent(context, finish = false)) + } .show() }, onDeleteRemoteBackup = { diff --git a/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/backup/InternalBackupPlaygroundViewModel.kt b/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/backup/InternalBackupPlaygroundViewModel.kt index 14187b9724..504e08f359 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/backup/InternalBackupPlaygroundViewModel.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/backup/InternalBackupPlaygroundViewModel.kt @@ -5,7 +5,6 @@ package org.thoughtcrime.securesms.components.settings.app.internal.backup -import android.net.Uri import androidx.compose.runtime.MutableState import androidx.compose.runtime.State import androidx.compose.runtime.mutableStateOf @@ -43,11 +42,6 @@ import org.thoughtcrime.securesms.backup.v2.BackupRepository import org.thoughtcrime.securesms.backup.v2.DebugBackupMetadata import org.thoughtcrime.securesms.backup.v2.MessageBackupTier import org.thoughtcrime.securesms.backup.v2.RemoteRestoreResult -import org.thoughtcrime.securesms.backup.v2.local.ArchiveFileSystem -import org.thoughtcrime.securesms.backup.v2.local.ArchiveResult -import org.thoughtcrime.securesms.backup.v2.local.LocalArchiver -import org.thoughtcrime.securesms.backup.v2.local.LocalArchiver.FailureCause -import org.thoughtcrime.securesms.backup.v2.local.SnapshotFileSystem import org.thoughtcrime.securesms.backup.v2.stream.EncryptedBackupReader import org.thoughtcrime.securesms.backup.v2.stream.EncryptedBackupReader.Companion.MAC_SIZE import org.thoughtcrime.securesms.database.AttachmentTable @@ -55,7 +49,6 @@ import org.thoughtcrime.securesms.database.AttachmentTable.DebugAttachmentStats import org.thoughtcrime.securesms.database.SignalDatabase import org.thoughtcrime.securesms.dependencies.AppDependencies import org.thoughtcrime.securesms.jobs.BackupMessagesJob -import org.thoughtcrime.securesms.jobs.RestoreLocalAttachmentJob import org.thoughtcrime.securesms.keyvalue.SignalStore import org.thoughtcrime.securesms.net.SignalNetwork import org.thoughtcrime.securesms.providers.BlobProvider @@ -193,29 +186,6 @@ class InternalBackupPlaygroundViewModel : ViewModel() { } } - fun import(uri: Uri) { - _state.value = _state.value.copy(statusMessage = "Importing new-style local backup...") - - val self = Recipient.self() - val selfData = BackupRepository.SelfData(self.aci.get(), self.pni.get(), self.e164.get(), ProfileKey(self.profileKey)) - - disposables += Single.fromCallable { - val archiveFileSystem = ArchiveFileSystem.fromUri(AppDependencies.application, uri)!! - val snapshotInfo = archiveFileSystem.listSnapshots().firstOrNull() ?: return@fromCallable ArchiveResult.failure(FailureCause.MAIN_STREAM) - val snapshotFileSystem = SnapshotFileSystem(AppDependencies.application, snapshotInfo.file) - - LocalArchiver.import(snapshotFileSystem, selfData) - - val mediaNameToFileInfo = archiveFileSystem.filesFileSystem.allFiles() - RestoreLocalAttachmentJob.enqueueRestoreLocalAttachmentsJobs(mediaNameToFileInfo) - } - .subscribeOn(Schedulers.io()) - .observeOn(AndroidSchedulers.mainThread()) - .subscribeBy { - _state.value = _state.value.copy(statusMessage = "New-style local backup import complete!") - } - } - fun haltAllJobs() { ArchiveUploadProgress.cancel() diff --git a/app/src/main/java/org/thoughtcrime/securesms/database/AttachmentMetadataTable.kt b/app/src/main/java/org/thoughtcrime/securesms/database/AttachmentMetadataTable.kt new file mode 100644 index 0000000000..610f539758 --- /dev/null +++ b/app/src/main/java/org/thoughtcrime/securesms/database/AttachmentMetadataTable.kt @@ -0,0 +1,126 @@ +/* + * Copyright 2025 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ + +package org.thoughtcrime.securesms.database + +import android.content.Context +import android.database.Cursor +import org.signal.core.util.delete +import org.signal.core.util.insertInto +import org.signal.core.util.logging.Log +import org.signal.core.util.readToList +import org.signal.core.util.readToSingleLongOrNull +import org.signal.core.util.requireBlob +import org.signal.core.util.requireNonNullString +import org.signal.core.util.select +import org.signal.core.util.update +import org.signal.core.util.withinTransaction +import org.thoughtcrime.securesms.attachments.AttachmentMetadata +import org.thoughtcrime.securesms.attachments.LocalBackupKey +import org.thoughtcrime.securesms.util.Util + +/** + * Metadata for various attachments. There is a many-to-one relationship with the Attachment table as this metadata + * represents data about a specific data file (plaintext hash). + */ +class AttachmentMetadataTable(context: Context, databaseHelper: SignalDatabase) : DatabaseTable(context, databaseHelper) { + companion object { + private val TAG = Log.tag(AttachmentMetadataTable::class) + + const val TABLE_NAME = "attachment_metadata" + const val ID = "_id" + const val PLAINTEXT_HASH = "plaintext_hash" + const val LOCAL_BACKUP_KEY = "local_backup_key" + + const val CREATE_TABLE = """ + CREATE TABLE $TABLE_NAME ( + $ID INTEGER PRIMARY KEY, + $PLAINTEXT_HASH TEXT NOT NULL, + $LOCAL_BACKUP_KEY BLOB DEFAULT NULL, + UNIQUE ($PLAINTEXT_HASH) + ) + """ + + val PROJECTION = arrayOf(LOCAL_BACKUP_KEY) + + /** + * Attempts to load metadata from the cursor if present. Returns null iff the cursor contained no + * metadata columns (i.e., no join in the original query). If there are columns, but they are null, the contents of the + * returned [AttachmentMetadata] will be null. + */ + fun getMetadata(cursor: Cursor, localBackupKeyColumn: String = LOCAL_BACKUP_KEY): AttachmentMetadata? { + if (cursor.getColumnIndex(localBackupKeyColumn) >= 0) { + val localBackupKey = cursor.requireBlob(localBackupKeyColumn)?.let { LocalBackupKey(it) } + return AttachmentMetadata(localBackupKey) + } + return null + } + } + + fun insert(plaintextHash: String, localBackupKey: ByteArray): Long { + val rowId = writableDatabase + .insertInto(TABLE_NAME) + .values(PLAINTEXT_HASH to plaintextHash, LOCAL_BACKUP_KEY to localBackupKey) + .run(conflictStrategy = SQLiteDatabase.CONFLICT_IGNORE) + + if (rowId > 0) { + return rowId + } + + return readableDatabase + .select(ID) + .from(TABLE_NAME) + .where("$PLAINTEXT_HASH = ?", plaintextHash) + .run() + .readToSingleLongOrNull()!! + } + + fun cleanup() { + writableDatabase + .delete(TABLE_NAME) + .where("$ID NOT IN (SELECT DISTINCT ${AttachmentTable.METADATA_ID} FROM ${AttachmentTable.TABLE_NAME})") + .run() + } + + fun insertNewKeysForExistingAttachments() { + writableDatabase.withinTransaction { + do { + val hashes: List = readableDatabase + .select("DISTINCT ${AttachmentTable.DATA_HASH_END}") + .from(AttachmentTable.TABLE_NAME) + .where("${AttachmentTable.DATA_HASH_END} IS NOT NULL AND ${AttachmentTable.DATA_FILE} IS NOT NULL AND ${AttachmentTable.METADATA_ID} IS NULL") + .limit(1000) + .run() + .readToList { it.requireNonNullString(AttachmentTable.DATA_HASH_END) } + + if (hashes.isNotEmpty()) { + val newKeys: List> = hashes.zip(hashes.map { Util.getSecretBytes(64) }) + + newKeys.forEach { (hash, key) -> + var rowId = writableDatabase + .insertInto(TABLE_NAME) + .values(PLAINTEXT_HASH to hash, LOCAL_BACKUP_KEY to key) + .run(conflictStrategy = SQLiteDatabase.CONFLICT_IGNORE) + + if (rowId == -1L) { + rowId = readableDatabase + .select(ID) + .from(TABLE_NAME) + .where("$PLAINTEXT_HASH = ?", hash) + .run() + .readToSingleLongOrNull()!! + } + + writableDatabase + .update(AttachmentTable.TABLE_NAME) + .values(AttachmentTable.METADATA_ID to rowId) + .where("${AttachmentTable.DATA_HASH_END} = ?", hash) + .run() + } + } + } while (hashes.isNotEmpty()) + } + } +} diff --git a/app/src/main/java/org/thoughtcrime/securesms/database/AttachmentTable.kt b/app/src/main/java/org/thoughtcrime/securesms/database/AttachmentTable.kt index 32cddc69f3..f7166e9815 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/database/AttachmentTable.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/database/AttachmentTable.kt @@ -60,6 +60,7 @@ import org.signal.core.util.requireBoolean import org.signal.core.util.requireInt import org.signal.core.util.requireIntOrNull import org.signal.core.util.requireLong +import org.signal.core.util.requireLongOrNull import org.signal.core.util.requireNonNullBlob import org.signal.core.util.requireNonNullString import org.signal.core.util.requireObject @@ -73,6 +74,7 @@ import org.thoughtcrime.securesms.attachments.Attachment import org.thoughtcrime.securesms.attachments.AttachmentId import org.thoughtcrime.securesms.attachments.Cdn import org.thoughtcrime.securesms.attachments.DatabaseAttachment +import org.thoughtcrime.securesms.attachments.LocalBackupKey import org.thoughtcrime.securesms.attachments.LocalStickerAttachment import org.thoughtcrime.securesms.attachments.WallpaperAttachment import org.thoughtcrime.securesms.audio.AudioHash @@ -191,6 +193,7 @@ class AttachmentTable( const val ATTACHMENT_UUID = "attachment_uuid" const val OFFLOAD_RESTORED_AT = "offload_restored_at" const val QUOTE_TARGET_CONTENT_TYPE = "quote_target_content_type" + const val METADATA_ID = "metadata_id" const val ATTACHMENT_JSON_ALIAS = "attachment_json" @@ -249,6 +252,8 @@ class AttachmentTable( ATTACHMENT_UUID ) + private val PROJECTION_WITH_METADATA = PROJECTION.map { if (it == ID) "$TABLE_NAME.$ID" else it }.toTypedArray() + AttachmentMetadataTable.PROJECTION + @JvmField val CREATE_TABLE = """ CREATE TABLE $TABLE_NAME ( @@ -293,7 +298,8 @@ class AttachmentTable( $ATTACHMENT_UUID TEXT DEFAULT NULL, $OFFLOAD_RESTORED_AT INTEGER DEFAULT 0, $QUOTE_TARGET_CONTENT_TYPE TEXT DEFAULT NULL, - $ARCHIVE_THUMBNAIL_TRANSFER_STATE INTEGER DEFAULT ${ArchiveTransferState.NONE.value} + $ARCHIVE_THUMBNAIL_TRANSFER_STATE INTEGER DEFAULT ${ArchiveTransferState.NONE.value}, + $METADATA_ID INTEGER DEFAULT NULL REFERENCES ${AttachmentMetadataTable.TABLE_NAME} (${AttachmentMetadataTable.ID}) ) """ @@ -309,11 +315,12 @@ class AttachmentTable( "CREATE INDEX IF NOT EXISTS $DATA_HASH_REMOTE_KEY_INDEX ON $TABLE_NAME ($DATA_HASH_END, $REMOTE_KEY);", "CREATE INDEX IF NOT EXISTS $DATA_FILE_INDEX ON $TABLE_NAME ($DATA_FILE);", "CREATE INDEX IF NOT EXISTS attachment_archive_transfer_state ON $TABLE_NAME ($ARCHIVE_TRANSFER_STATE);", - "CREATE INDEX IF NOT EXISTS attachment_remote_digest_index ON $TABLE_NAME ($REMOTE_DIGEST);" + "CREATE INDEX IF NOT EXISTS attachment_remote_digest_index ON $TABLE_NAME ($REMOTE_DIGEST);", + "CREATE INDEX IF NOT EXISTS attachment_metadata_id ON $TABLE_NAME ($METADATA_ID);" ) private val DATA_FILE_INFO_PROJECTION = arrayOf( - ID, DATA_FILE, DATA_SIZE, DATA_RANDOM, DATA_HASH_START, DATA_HASH_END, TRANSFORM_PROPERTIES, UPLOAD_TIMESTAMP, ARCHIVE_CDN, ARCHIVE_TRANSFER_STATE, THUMBNAIL_FILE, THUMBNAIL_RESTORE_STATE, THUMBNAIL_RANDOM + ID, DATA_FILE, DATA_SIZE, DATA_RANDOM, DATA_HASH_START, DATA_HASH_END, TRANSFORM_PROPERTIES, UPLOAD_TIMESTAMP, ARCHIVE_CDN, ARCHIVE_TRANSFER_STATE, THUMBNAIL_FILE, THUMBNAIL_RESTORE_STATE, THUMBNAIL_RANDOM, METADATA_ID ) private const val QUOTE_THUMBNAIL_DIMEN = 200 @@ -325,6 +332,8 @@ class AttachmentTable( /** Indicates a quote from a free-tier backup restore is pending potential reconstruction from a parent attachment. */ const val QUOTE_PENDING_RECONSTRUCTION = 3 + private val TABLE_NAME_WITH_METADTA = "$TABLE_NAME LEFT JOIN ${AttachmentMetadataTable.TABLE_NAME} ON $TABLE_NAME.$METADATA_ID = ${AttachmentMetadataTable.TABLE_NAME}.${AttachmentMetadataTable.ID}" + @JvmStatic @Throws(IOException::class) fun newDataFile(context: Context): File { @@ -472,13 +481,15 @@ class AttachmentTable( .firstOrNull() } - fun getAttachmentIdByPlaintextHashAndRemoteKey(plaintextHash: ByteArray, remoteKey: ByteArray): AttachmentId? { + fun getAttachmentWithMetadata(attachmentId: AttachmentId): DatabaseAttachment? { return readableDatabase - .select(ID) - .from("$TABLE_NAME INDEXED BY $DATA_HASH_REMOTE_KEY_INDEX") - .where("$DATA_HASH_END = ? AND $REMOTE_KEY = ?", Base64.encodeWithPadding(plaintextHash), Base64.encodeWithPadding(remoteKey)) + .select(*PROJECTION_WITH_METADATA) + .from(TABLE_NAME_WITH_METADTA) + .where("$TABLE_NAME.$ID = ?", attachmentId.id) .run() - .readToSingleObject { AttachmentId(it.requireLong(ID)) } + .readToList { it.readAttachments() } + .flatten() + .firstOrNull() } fun getAttachmentsForMessage(mmsId: Long): List { @@ -492,23 +503,37 @@ class AttachmentTable( .flatten() } - @JvmOverloads - fun getAttachmentsForMessages(mmsIds: Collection, excludeTranscodingQuotes: Boolean = false): Map> { + fun getAttachmentsForMessagesArchive(mmsIds: Collection): Map> { if (mmsIds.isEmpty()) { return emptyMap() } val query = SqlUtil.buildFastCollectionQuery(MESSAGE_ID, mmsIds) - val where = if (excludeTranscodingQuotes) { - "(${query.where}) AND $QUOTE != $QUOTE_PENDING_TRANSCODE" - } else { - query.where + val where = "(${query.where}) AND $QUOTE != $QUOTE_PENDING_TRANSCODE" + + return readableDatabase + .select(*PROJECTION_WITH_METADATA) + .from(TABLE_NAME_WITH_METADTA) + .where(where, query.whereArgs) + .orderBy("$TABLE_NAME.$ID ASC") + .run() + .groupBy { cursor -> + val attachment = cursor.readAttachment() + attachment.mmsId to attachment + } + } + + fun getAttachmentsForMessages(mmsIds: Collection): Map> { + if (mmsIds.isEmpty()) { + return emptyMap() } + val query = SqlUtil.buildFastCollectionQuery(MESSAGE_ID, mmsIds) + return readableDatabase .select(*PROJECTION) .from(TABLE_NAME) - .where(where, query.whereArgs) + .where(query.where, query.whereArgs) .orderBy("$ID ASC") .run() .groupBy { cursor -> @@ -559,38 +584,20 @@ class AttachmentTable( .flatten() } - fun getLocalArchivableAttachment(plaintextHash: String, remoteKey: String): LocalArchivableAttachment? { - return readableDatabase - .select(*PROJECTION) - .from(TABLE_NAME) - .where("$DATA_HASH_END = ? AND $REMOTE_KEY = ?") - .orderBy("$ID DESC") - .limit(1) - .run() - .readToSingleObject { - LocalArchivableAttachment( - file = File(it.requireNonNullString(DATA_FILE)), - random = it.requireNonNullBlob(DATA_RANDOM), - size = it.requireLong(DATA_SIZE), - remoteKey = Base64.decode(it.requireNonNullString(REMOTE_KEY)), - plaintextHash = Base64.decode(it.requireNonNullString(DATA_HASH_END)) - ) - } - } - fun getLocalArchivableAttachments(): List { return readableDatabase - .select(*PROJECTION) - .from(TABLE_NAME) - .where("$REMOTE_KEY IS NOT NULL AND $DATA_HASH_END IS NOT NULL AND $DATA_FILE IS NOT NULL") - .orderBy("$ID DESC") + .select(*PROJECTION_WITH_METADATA) + .from(TABLE_NAME_WITH_METADTA) + .where("$DATA_HASH_END IS NOT NULL AND $DATA_FILE IS NOT NULL AND ${AttachmentMetadataTable.TABLE_NAME}.${AttachmentMetadataTable.LOCAL_BACKUP_KEY} IS NOT NULL") + .orderBy("$TABLE_NAME.$ID DESC") .run() .readToList { LocalArchivableAttachment( + attachmentId = AttachmentId(it.requireLong(ID)), file = File(it.requireNonNullString(DATA_FILE)), random = it.requireNonNullBlob(DATA_RANDOM), size = it.requireLong(DATA_SIZE), - remoteKey = Base64.decode(it.requireNonNullString(REMOTE_KEY)), + localBackupKey = AttachmentMetadataTable.getMetadata(it)!!.localBackupKey!!, plaintextHash = Base64.decode(it.requireNonNullString(DATA_HASH_END)) ) } @@ -646,26 +653,6 @@ class AttachmentTable( } } - fun getRestorableAttachments(batchSize: Int): List { - return readableDatabase - .select(ID, MESSAGE_ID, DATA_SIZE, DATA_HASH_END, REMOTE_KEY, STICKER_PACK_ID) - .from(TABLE_NAME) - .where("$TRANSFER_STATE = ?", TRANSFER_NEEDS_RESTORE) - .limit(batchSize) - .orderBy("$ID DESC") - .run() - .readToList { - RestorableAttachment( - attachmentId = AttachmentId(it.requireLong(ID)), - mmsId = it.requireLong(MESSAGE_ID), - size = it.requireLong(DATA_SIZE), - plaintextHash = it.requireString(DATA_HASH_END)?.let { hash -> Base64.decode(hash) }, - remoteKey = it.requireString(REMOTE_KEY)?.let { key -> Base64.decode(key) }, - stickerPackId = it.requireString(STICKER_PACK_ID) - ) - } - } - fun getRestorableOptimizedAttachments(): List { return readableDatabase .select(ID, MESSAGE_ID, DATA_SIZE, DATA_HASH_END, REMOTE_KEY, STICKER_PACK_ID) @@ -685,6 +672,25 @@ class AttachmentTable( } } + fun getRestorableLocalAttachments(batchSize: Int): List { + return readableDatabase + .select("$TABLE_NAME.$ID", MESSAGE_ID, DATA_HASH_END, STICKER_PACK_ID, AttachmentMetadataTable.LOCAL_BACKUP_KEY) + .from(TABLE_NAME_WITH_METADTA) + .where("$TRANSFER_STATE = ?", TRANSFER_NEEDS_RESTORE) + .limit(batchSize) + .orderBy("$TABLE_NAME.$ID DESC") + .run() + .readToList { + LocalRestorableAttachment( + attachmentId = AttachmentId(it.requireLong(ID)), + mmsId = it.requireLong(MESSAGE_ID), + plaintextHash = it.requireString(DATA_HASH_END)?.let { hash -> Base64.decode(hash) }, + localBackupKey = it.requireBlob(AttachmentMetadataTable.LOCAL_BACKUP_KEY)?.let { key -> LocalBackupKey(key) }, + stickerPackId = it.requireString(STICKER_PACK_ID) + ) + } + } + fun getRemainingRestorableAttachmentSize(): Long { return readableDatabase .rawQuery( @@ -1239,6 +1245,8 @@ class AttachmentTable( .where("$MESSAGE_ID = ?", mmsId) .run() + SignalDatabase.attachmentMetadata.cleanup() + AppDependencies.databaseObserver.notifyAttachmentDeletedObservers() deleteCount @@ -1315,11 +1323,14 @@ class AttachmentTable( TRANSFER_STATE to TRANSFER_PROGRESS_DONE, ARCHIVE_TRANSFER_STATE to ArchiveTransferState.NONE.value, BLUR_HASH to null, - CONTENT_TYPE to MediaUtil.VIEW_ONCE + CONTENT_TYPE to MediaUtil.VIEW_ONCE, + METADATA_ID to null ) .where("$MESSAGE_ID = ?", messageId) .run() + SignalDatabase.attachmentMetadata.cleanup() + AppDependencies.databaseObserver.notifyAttachmentDeletedObservers() val threadId = messages.getThreadIdForMessage(messageId) @@ -1357,6 +1368,8 @@ class AttachmentTable( .where("$ID = ?", id.id) .run() + SignalDatabase.attachmentMetadata.cleanup() + if (filePath != null && isSafeToDeleteDataFile(filePath, id)) { filePathsToDelete += filePath contentType?.let { contentTypesToDelete += it } @@ -1490,6 +1503,7 @@ class AttachmentTable( Log.d(TAG, "[deleteAllAttachments]") writableDatabase.deleteAll(TABLE_NAME) + SignalDatabase.attachmentMetadata.cleanup() FileUtils.deleteDirectoryContents(context.getDir(DIRECTORY, Context.MODE_PRIVATE)) @@ -1647,6 +1661,7 @@ class AttachmentTable( values.put(DATA_RANDOM, hashMatch.random) values.put(DATA_HASH_START, hashMatch.hashEnd) values.put(DATA_HASH_END, hashMatch.hashEnd) + values.put(METADATA_ID, hashMatch.metadataId) if (archiveRestore) { // We aren't getting an updated remote key/mediaName when restoring, can reuse values.put(ARCHIVE_CDN, hashMatch.archiveCdn) @@ -2295,7 +2310,8 @@ class AttachmentTable( archiveCdn = if (jsonObject.isNull(ARCHIVE_CDN)) null else jsonObject.getInt(ARCHIVE_CDN), thumbnailRestoreState = ThumbnailRestoreState.deserialize(jsonObject.getInt(THUMBNAIL_RESTORE_STATE)), archiveTransferState = ArchiveTransferState.deserialize(jsonObject.getInt(ARCHIVE_TRANSFER_STATE)), - uuid = UuidUtil.parseOrNull(jsonObject.getString(ATTACHMENT_UUID)) + uuid = UuidUtil.parseOrNull(jsonObject.getString(ATTACHMENT_UUID)), + metadata = null ) } } @@ -2764,6 +2780,12 @@ class AttachmentTable( val attachmentId: AttachmentId = writableDatabase.withinTransaction { db -> val plaintextHash = attachment.plaintextHash.takeIf { it.isNotEmpty() }?.let { Base64.encodeWithPadding(it) } + val metadataId: Long? = if (plaintextHash != null && attachment.localBackupKey != null) { + SignalDatabase.attachmentMetadata.insert(plaintextHash, attachment.localBackupKey) + } else { + null + } + val contentValues = ContentValues().apply { put(MESSAGE_ID, messageId) put(CONTENT_TYPE, attachment.contentType) @@ -2808,6 +2830,10 @@ class AttachmentTable( } else { putNull(REMOTE_INCREMENTAL_DIGEST) } + + if (metadataId != null && metadataId > 0) { + put(METADATA_ID, metadataId) + } } val rowId = db.insert(TABLE_NAME, null, contentValues) @@ -2870,6 +2896,7 @@ class AttachmentTable( put(THUMBNAIL_RANDOM, dataFileInfo.thumbnailRandom) put(THUMBNAIL_FILE, dataFileInfo.thumbnailFile) put(ATTACHMENT_UUID, stickerAttachment.uuid?.toString()) + put(METADATA_ID, dataFileInfo.metadataId) } val rowId = db.insert(TABLE_NAME, null, contentValues) @@ -2911,7 +2938,7 @@ class AttachmentTable( attachmentId = AttachmentId(rowId) } - return attachmentId as AttachmentId + return attachmentId } /** @@ -2994,6 +3021,7 @@ class AttachmentTable( contentValues.put(DATA_RANDOM, hashMatch.random) contentValues.put(DATA_HASH_START, fileWriteResult.hash) contentValues.put(DATA_HASH_END, hashMatch.hashEnd) + contentValues.put(METADATA_ID, hashMatch.metadataId) if (hashMatch.transformProperties.skipTransform) { Log.i(TAG, "[insertAttachmentWithData] The hash match has a DATA_HASH_END and skipTransform=true, so skipping transform of the new file as well. (MessageId: $messageId, ${attachment.uri})") @@ -3249,7 +3277,8 @@ class AttachmentTable( archiveCdn = cursor.requireIntOrNull(ARCHIVE_CDN), thumbnailRestoreState = ThumbnailRestoreState.deserialize(cursor.requireInt(THUMBNAIL_RESTORE_STATE)), archiveTransferState = ArchiveTransferState.deserialize(cursor.requireInt(ARCHIVE_TRANSFER_STATE)), - uuid = UuidUtil.parseOrNull(cursor.requireString(ATTACHMENT_UUID)) + uuid = UuidUtil.parseOrNull(cursor.requireString(ATTACHMENT_UUID)), + metadata = AttachmentMetadataTable.getMetadata(cursor) ) } @@ -3278,7 +3307,8 @@ class AttachmentTable( archiveTransferState = this.requireInt(ARCHIVE_TRANSFER_STATE), thumbnailFile = this.requireString(THUMBNAIL_FILE), thumbnailRandom = this.requireBlob(THUMBNAIL_RANDOM), - thumbnailRestoreState = this.requireInt(THUMBNAIL_RESTORE_STATE) + thumbnailRestoreState = this.requireInt(THUMBNAIL_RESTORE_STATE), + metadataId = this.requireLongOrNull(METADATA_ID) ) } @@ -3649,7 +3679,8 @@ class AttachmentTable( val archiveTransferState: Int, val thumbnailFile: String?, val thumbnailRandom: ByteArray?, - val thumbnailRestoreState: Int + val thumbnailRestoreState: Int, + val metadataId: Long? ) @VisibleForTesting @@ -3840,11 +3871,12 @@ class AttachmentTable( class SyncAttachment(val id: AttachmentId, val uuid: UUID?, val digest: ByteArray?, val plaintextHash: String?) class LocalArchivableAttachment( + val attachmentId: AttachmentId, val file: File, val random: ByteArray, val size: Long, val plaintextHash: ByteArray, - val remoteKey: ByteArray + val localBackupKey: LocalBackupKey ) data class RestorableAttachment( @@ -3864,6 +3896,22 @@ class AttachmentTable( } } + data class LocalRestorableAttachment( + val attachmentId: AttachmentId, + val mmsId: Long, + val plaintextHash: ByteArray?, + val localBackupKey: LocalBackupKey?, + val stickerPackId: String? + ) { + override fun equals(other: Any?): Boolean { + return this === other || attachmentId == (other as? RestorableAttachment)?.attachmentId + } + + override fun hashCode(): Int { + return attachmentId.hashCode() + } + } + data class DebugAttachmentStats( val totalAttachmentRows: Long = 0L, val totalUniqueMediaNamesEligibleForUpload: Long = 0L, diff --git a/app/src/main/java/org/thoughtcrime/securesms/database/SignalDatabase.kt b/app/src/main/java/org/thoughtcrime/securesms/database/SignalDatabase.kt index 8089b55751..3fdbd58016 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/database/SignalDatabase.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/database/SignalDatabase.kt @@ -82,6 +82,7 @@ open class SignalDatabase(private val context: Application, databaseSecret: Data val backupMediaSnapshotTable: BackupMediaSnapshotTable = BackupMediaSnapshotTable(context, this) val pollTable: PollTables = PollTables(context, this) val lastResortKeyTuples: LastResortKeyTupleTable = LastResortKeyTupleTable(context, this) + val attachmentMetadataTable: AttachmentMetadataTable = AttachmentMetadataTable(context, this) override fun onOpen(db: net.zetetic.database.sqlcipher.SQLiteDatabase) { db.setForeignKeyConstraintsEnabled(true) @@ -152,6 +153,7 @@ open class SignalDatabase(private val context: Application, databaseSecret: Data executeStatements(db, PollTables.CREATE_TABLE) db.execSQL(BackupMediaSnapshotTable.CREATE_TABLE) db.execSQL(LastResortKeyTupleTable.CREATE_TABLE) + db.execSQL(AttachmentMetadataTable.CREATE_TABLE) executeStatements(db, RecipientTable.CREATE_INDEXS) executeStatements(db, MessageTable.CREATE_INDEXS) @@ -597,5 +599,10 @@ open class SignalDatabase(private val context: Application, databaseSecret: Data @get:JvmName("lastResortKeyTuples") val lastResortKeyTuples: LastResortKeyTupleTable get() = instance!!.lastResortKeyTuples + + @get:JvmStatic + @get:JvmName("attachmentMetadata") + val attachmentMetadata: AttachmentMetadataTable + get() = instance!!.attachmentMetadataTable } } diff --git a/app/src/main/java/org/thoughtcrime/securesms/database/helpers/SignalDatabaseMigrations.kt b/app/src/main/java/org/thoughtcrime/securesms/database/helpers/SignalDatabaseMigrations.kt index f1f00edcc0..b8c1a28827 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/database/helpers/SignalDatabaseMigrations.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/database/helpers/SignalDatabaseMigrations.kt @@ -152,6 +152,7 @@ import org.thoughtcrime.securesms.database.helpers.migration.V295_AddLastRestore import org.thoughtcrime.securesms.database.helpers.migration.V296_RemovePollVoteConstraint import org.thoughtcrime.securesms.database.helpers.migration.V297_AddPinnedMessageColumns import org.thoughtcrime.securesms.database.helpers.migration.V298_DoNotBackupReleaseNotes +import org.thoughtcrime.securesms.database.helpers.migration.V299_AddAttachmentMetadataTable import org.thoughtcrime.securesms.database.SQLiteDatabase as SignalSqliteDatabase /** @@ -310,10 +311,11 @@ object SignalDatabaseMigrations { 295 to V295_AddLastRestoreKeyTypeTableIfMissingMigration, 296 to V296_RemovePollVoteConstraint, 297 to V297_AddPinnedMessageColumns, - 298 to V298_DoNotBackupReleaseNotes + 298 to V298_DoNotBackupReleaseNotes, + 299 to V299_AddAttachmentMetadataTable ) - const val DATABASE_VERSION = 298 + const val DATABASE_VERSION = 299 @JvmStatic fun migrate(context: Application, db: SignalSqliteDatabase, oldVersion: Int, newVersion: Int) { diff --git a/app/src/main/java/org/thoughtcrime/securesms/database/helpers/migration/V299_AddAttachmentMetadataTable.kt b/app/src/main/java/org/thoughtcrime/securesms/database/helpers/migration/V299_AddAttachmentMetadataTable.kt new file mode 100644 index 0000000000..418d311e4c --- /dev/null +++ b/app/src/main/java/org/thoughtcrime/securesms/database/helpers/migration/V299_AddAttachmentMetadataTable.kt @@ -0,0 +1,32 @@ +/* + * Copyright 2024 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ + +package org.thoughtcrime.securesms.database.helpers.migration + +import android.app.Application +import org.thoughtcrime.securesms.database.SQLiteDatabase + +/** + * We need to keep track of the local backup key + */ +@Suppress("ClassName") +object V299_AddAttachmentMetadataTable : SignalDatabaseMigration { + + override fun migrate(context: Application, db: SQLiteDatabase, oldVersion: Int, newVersion: Int) { + db.execSQL( + """ + CREATE TABLE attachment_metadata ( + _id INTEGER PRIMARY KEY, + plaintext_hash TEXT NOT NULL, + local_backup_key BLOB DEFAULT NULL, + UNIQUE (plaintext_hash) + ) + """ + ) + + db.execSQL("ALTER TABLE attachment ADD COLUMN metadata_id INTEGER DEFAULT NULL REFERENCES attachment_metadata (_id)") + db.execSQL("CREATE INDEX IF NOT EXISTS attachment_metadata_id ON attachment (metadata_id)") + } +} diff --git a/app/src/main/java/org/thoughtcrime/securesms/jobs/LocalArchiveJob.kt b/app/src/main/java/org/thoughtcrime/securesms/jobs/LocalArchiveJob.kt index 84cb3e3bee..8ecaa8a173 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/jobs/LocalArchiveJob.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/jobs/LocalArchiveJob.kt @@ -1,9 +1,9 @@ package org.thoughtcrime.securesms.jobs +import android.net.Uri import org.greenrobot.eventbus.EventBus import org.greenrobot.eventbus.Subscribe import org.greenrobot.eventbus.ThreadMode -import org.signal.core.util.Result import org.signal.core.util.Stopwatch import org.signal.core.util.logging.Log import org.thoughtcrime.securesms.R @@ -13,13 +13,12 @@ import org.thoughtcrime.securesms.backup.v2.LocalBackupV2Event import org.thoughtcrime.securesms.backup.v2.local.ArchiveFileSystem import org.thoughtcrime.securesms.backup.v2.local.LocalArchiver import org.thoughtcrime.securesms.backup.v2.local.SnapshotFileSystem +import org.thoughtcrime.securesms.database.SignalDatabase import org.thoughtcrime.securesms.jobmanager.Job import org.thoughtcrime.securesms.keyvalue.SignalStore import org.thoughtcrime.securesms.notifications.NotificationChannels import org.thoughtcrime.securesms.service.GenericForegroundService import org.thoughtcrime.securesms.service.NotificationController -import org.thoughtcrime.securesms.util.BackupUtil -import org.thoughtcrime.securesms.util.StorageUtil import java.io.IOException /** @@ -69,17 +68,11 @@ class LocalArchiveJob internal constructor(parameters: Parameters) : Job(paramet val stopwatch = Stopwatch("archive-export") - val archiveFileSystem = if (BackupUtil.isUserSelectionRequired(context)) { - val backupDirectoryUri = SignalStore.settings.signalBackupDirectory - - if (backupDirectoryUri == null || backupDirectoryUri.path == null) { - throw IOException("Backup Directory has not been selected!") - } - - ArchiveFileSystem.fromUri(context, backupDirectoryUri) - } else { - ArchiveFileSystem.fromFile(context, StorageUtil.getOrCreateBackupV2Directory()) + val backupDirectoryUri = SignalStore.backup.newLocalBackupsDirectory?.let { Uri.parse(it) } + if (backupDirectoryUri == null || backupDirectoryUri.path == null) { + throw IOException("Backup Directory has not been selected!") } + val archiveFileSystem = ArchiveFileSystem.fromUri(context, backupDirectoryUri) if (archiveFileSystem == null) { BackupFileIOError.ACCESS_ERROR.postNotification(context) @@ -95,6 +88,8 @@ class LocalArchiveJob internal constructor(parameters: Parameters) : Job(paramet stopwatch.split("create-snapshot") try { + SignalDatabase.attachmentMetadata.insertNewKeysForExistingAttachments() + try { val result = LocalArchiver.export(snapshotFileSystem, archiveFileSystem.filesFileSystem, stopwatch, cancellationSignal = { isCanceled }) Log.i(TAG, "Archive finished with result: $result") @@ -108,22 +103,10 @@ class LocalArchiveJob internal constructor(parameters: Parameters) : Job(paramet stopwatch.split("archive-create") - // todo [local-backup] verify local backup - EventBus.getDefault().post(LocalBackupV2Event(LocalBackupV2Event.Type.PROGRESS_VERIFYING)) - val valid = true - - stopwatch.split("archive-verify") - - if (valid) { - snapshotFileSystem.finalize() - stopwatch.split("archive-finalize") - } else { - BackupFileIOError.VERIFICATION_FAILED.postNotification(context) - } + snapshotFileSystem.finalize() + stopwatch.split("archive-finalize") EventBus.getDefault().post(LocalBackupV2Event(LocalBackupV2Event.Type.FINISHED)) - - stopwatch.stop(TAG) } catch (e: BackupCanceledException) { EventBus.getDefault().post(LocalBackupV2Event(LocalBackupV2Event.Type.FINISHED)) Log.w(TAG, "Archive cancelled") @@ -148,6 +131,8 @@ class LocalArchiveJob internal constructor(parameters: Parameters) : Job(paramet stopwatch.split("delete-unused") stopwatch.stop(TAG) + + SignalStore.backup.newLocalBackupsLastBackupTime = System.currentTimeMillis() } finally { notification?.close() EventBus.getDefault().unregister(updater) diff --git a/app/src/main/java/org/thoughtcrime/securesms/jobs/LocalBackupJob.java b/app/src/main/java/org/thoughtcrime/securesms/jobs/LocalBackupJob.java index 6e64a2a726..14c0bbc157 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/jobs/LocalBackupJob.java +++ b/app/src/main/java/org/thoughtcrime/securesms/jobs/LocalBackupJob.java @@ -36,6 +36,7 @@ import java.io.IOException; import java.text.SimpleDateFormat; import java.util.Date; import java.util.Locale; +import java.util.concurrent.TimeUnit; public final class LocalBackupJob extends BaseJob { @@ -65,11 +66,12 @@ public final class LocalBackupJob extends BaseJob { } } - public static void enqueueArchive() { + public static void enqueueArchive(boolean delay) { JobManager jobManager = AppDependencies.getJobManager(); Parameters.Builder parameters = new Parameters.Builder() .setQueue(QUEUE) .setMaxInstancesForFactory(1) + .setInitialDelay(delay ? TimeUnit.MINUTES.toMillis(30) : 0) .setMaxAttempts(3); jobManager.add(new LocalArchiveJob(parameters.build())); diff --git a/app/src/main/java/org/thoughtcrime/securesms/jobs/RestoreLocalAttachmentJob.kt b/app/src/main/java/org/thoughtcrime/securesms/jobs/RestoreLocalAttachmentJob.kt index 2ff521d0e7..ea4e945108 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/jobs/RestoreLocalAttachmentJob.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/jobs/RestoreLocalAttachmentJob.kt @@ -16,7 +16,7 @@ import org.thoughtcrime.securesms.attachments.AttachmentId import org.thoughtcrime.securesms.backup.v2.ArchiveRestoreProgress import org.thoughtcrime.securesms.backup.v2.local.ArchiveFileSystem import org.thoughtcrime.securesms.database.AttachmentTable -import org.thoughtcrime.securesms.database.AttachmentTable.RestorableAttachment +import org.thoughtcrime.securesms.database.AttachmentTable.LocalRestorableAttachment import org.thoughtcrime.securesms.database.SignalDatabase import org.thoughtcrime.securesms.dependencies.AppDependencies import org.thoughtcrime.securesms.jobmanager.Job @@ -47,15 +47,14 @@ class RestoreLocalAttachmentJob private constructor( val jobManager = AppDependencies.jobManager do { - val possibleRestorableAttachments: List = SignalDatabase.attachments.getRestorableAttachments(500) + val possibleRestorableAttachments: List = SignalDatabase.attachments.getRestorableLocalAttachments(500) val notRestorableAttachments = ArrayList(possibleRestorableAttachments.size) val restoreAttachmentJobs: MutableList = ArrayList(possibleRestorableAttachments.size) possibleRestorableAttachments .forEachIndexed { index, attachment -> - val fileInfo = if (attachment.plaintextHash != null && attachment.remoteKey != null) { - val mediaName = MediaName.fromPlaintextHashAndRemoteKey(attachment.plaintextHash, attachment.remoteKey).name - mediaNameToFileInfo[mediaName] + val fileInfo = if (attachment.plaintextHash != null && attachment.localBackupKey != null) { + mediaNameToFileInfo[MediaName.forLocalBackupFilename(attachment.plaintextHash, attachment.localBackupKey.key).name] } else { null } @@ -90,7 +89,7 @@ class RestoreLocalAttachmentJob private constructor( } } - private constructor(queue: String, attachment: RestorableAttachment, info: DocumentFileInfo) : this( + private constructor(queue: String, attachment: LocalRestorableAttachment, info: DocumentFileInfo) : this( Parameters.Builder() .setQueue(queue) .setLifespan(Parameters.IMMORTAL) @@ -122,15 +121,15 @@ class RestoreLocalAttachmentJob private constructor( override fun run(): Result { Log.i(TAG, "onRun() messageId: $messageId attachmentId: $attachmentId") - val attachment = SignalDatabase.attachments.getAttachment(attachmentId) + val attachment = SignalDatabase.attachments.getAttachmentWithMetadata(attachmentId) if (attachment == null) { Log.w(TAG, "attachment no longer exists.") return Result.failure() } - if (attachment.remoteDigest == null || attachment.remoteKey == null) { - Log.w(TAG, "Attachment no longer has a remote digest or key") + if (attachment.dataHash == null || attachment.metadata?.localBackupKey == null) { + Log.w(TAG, "Attachment no longer has a plaintext hash or local backup key") return Result.failure() } @@ -144,7 +143,6 @@ class RestoreLocalAttachmentJob private constructor( return Result.success() } - val combinedKey = Base64.decode(attachment.remoteKey) val streamSupplier = StreamSupplier { ArchiveFileSystem.openInputStream(context, restoreUri) ?: throw IOException("Unable to open stream") } try { @@ -154,10 +152,9 @@ class RestoreLocalAttachmentJob private constructor( streamSupplier = streamSupplier, streamLength = size, plaintextLength = attachment.size, - combinedKeyMaterial = combinedKey, - integrityCheck = IntegrityCheck.forEncryptedDigestAndPlaintextHash( - encryptedDigest = attachment.remoteDigest, - plaintextHash = attachment.dataHash + combinedKeyMaterial = attachment.metadata.localBackupKey.key, + integrityCheck = IntegrityCheck.forPlaintextHash( + plaintextHash = Base64.decode(attachment.dataHash) ), incrementalDigest = null, incrementalMacChunkSize = 0 diff --git a/app/src/main/java/org/thoughtcrime/securesms/keyvalue/BackupValues.kt b/app/src/main/java/org/thoughtcrime/securesms/keyvalue/BackupValues.kt index bb358523c5..58408861c7 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/keyvalue/BackupValues.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/keyvalue/BackupValues.kt @@ -99,6 +99,10 @@ class BackupValues(store: KeyValueStore) : SignalStoreValues(store) { private const val KEY_MESSAGE_CUTOFF_DURATION = "backup.message_cutoff_duration" private const val KEY_LAST_USED_MESSAGE_CUTOFF_TIME = "backup.last_used_message_cutoff_time" + private const val KEY_NEW_LOCAL_BACKUPS_ENABLED = "backup.new_local_backups_enabled" + private const val KEY_NEW_LOCAL_BACKUPS_DIRECTORY = "backup.new_local_backups_directory" + private const val KEY_NEW_LOCAL_BACKUPS_LAST_BACKUP_TIME = "backup.new_local_backups_last_backup_time" + private val cachedCdnCredentialsExpiresIn: Duration = 12.hours private val lock = ReentrantLock() @@ -441,6 +445,27 @@ class BackupValues(store: KeyValueStore) : SignalStoreValues(store) { */ var lastUsedMessageCutoffTime: Long by longValue(KEY_LAST_USED_MESSAGE_CUTOFF_TIME, 0) + /** + * True if the new local backup system is enabled, otherwise false. + */ + private val newLocalBackupsEnabledValue = booleanValue(KEY_NEW_LOCAL_BACKUPS_ENABLED, false) + var newLocalBackupsEnabled: Boolean by newLocalBackupsEnabledValue + val newLocalBackupsEnabledFlow: Flow by lazy { newLocalBackupsEnabledValue.toFlow() } + + /** + * The directory URI path selected for new local backups. + */ + private val newLocalBackupsDirectoryValue = stringValue(KEY_NEW_LOCAL_BACKUPS_DIRECTORY, null as String?) + var newLocalBackupsDirectory: String? by newLocalBackupsDirectoryValue + val newLocalBackupsDirectoryFlow: Flow by lazy { newLocalBackupsDirectoryValue.toFlow() } + + /** + * The timestamp of the last successful new local backup. + */ + private val newLocalBackupsLastBackupTimeValue = longValue(KEY_NEW_LOCAL_BACKUPS_LAST_BACKUP_TIME, -1) + var newLocalBackupsLastBackupTime: Long by newLocalBackupsLastBackupTimeValue + val newLocalBackupsLastBackupTimeFlow: Flow by lazy { newLocalBackupsLastBackupTimeValue.toFlow() } + /** * When we are told by the server that we are out of storage space, we should show * UX treatment to make the user aware of this. diff --git a/app/src/main/java/org/thoughtcrime/securesms/keyvalue/RestoreDecisionStateExt.kt b/app/src/main/java/org/thoughtcrime/securesms/keyvalue/RestoreDecisionStateExt.kt index c47eb03abd..ac945a1a80 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/keyvalue/RestoreDecisionStateExt.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/keyvalue/RestoreDecisionStateExt.kt @@ -32,6 +32,15 @@ val RestoreDecisionState.isWantingManualRemoteRestore: Boolean else -> false } +/** Has the user indicated they want a manual local v2 restore but not via quick restore. */ +val RestoreDecisionState.isWantingNewLocalBackupRestore: Boolean + get() = when (this.decisionState) { + RestoreDecisionState.State.INTEND_TO_RESTORE -> { + this.intendToRestoreData?.fromLocalV2 == true && !this.intendToRestoreData.hasOldDevice + } + else -> false + } + val RestoreDecisionState.includeDeviceToDeviceTransfer: Boolean get() = when (this.decisionState) { RestoreDecisionState.State.INTEND_TO_RESTORE -> { @@ -49,10 +58,10 @@ val RestoreDecisionState.Companion.Start: RestoreDecisionState get() = RestoreDecisionState(RestoreDecisionState.State.START) /** Helper to create a [RestoreDecisionState.State.INTEND_TO_RESTORE] with appropriate data. */ -fun RestoreDecisionState.Companion.intendToRestore(hasOldDevice: Boolean, fromRemote: Boolean?): RestoreDecisionState { +fun RestoreDecisionState.Companion.intendToRestore(hasOldDevice: Boolean, fromRemote: Boolean?, fromLocalV2: Boolean? = null): RestoreDecisionState { return RestoreDecisionState( decisionState = RestoreDecisionState.State.INTEND_TO_RESTORE, - intendToRestoreData = RestoreDecisionState.IntendToRestoreData(hasOldDevice = hasOldDevice, fromRemote = fromRemote) + intendToRestoreData = RestoreDecisionState.IntendToRestoreData(hasOldDevice = hasOldDevice, fromRemote = fromRemote, fromLocalV2 = fromLocalV2) ) } diff --git a/app/src/main/java/org/thoughtcrime/securesms/registration/ui/RegistrationViewModel.kt b/app/src/main/java/org/thoughtcrime/securesms/registration/ui/RegistrationViewModel.kt index b7bd834d92..5b7122acf3 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/registration/ui/RegistrationViewModel.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/registration/ui/RegistrationViewModel.kt @@ -987,8 +987,8 @@ class RegistrationViewModel : ViewModel() { SignalStore.registration.restoreDecisionState = RestoreDecisionState.Start } - fun intendToRestore(hasOldDevice: Boolean, fromRemote: Boolean? = null) { - SignalStore.registration.restoreDecisionState = RestoreDecisionState.intendToRestore(hasOldDevice, fromRemote) + fun intendToRestore(hasOldDevice: Boolean, fromRemote: Boolean? = null, fromLocalV2: Boolean? = null) { + SignalStore.registration.restoreDecisionState = RestoreDecisionState.intendToRestore(hasOldDevice, fromRemote, fromLocalV2) } fun skipRestore() { diff --git a/app/src/main/java/org/thoughtcrime/securesms/registration/ui/phonenumber/EnterPhoneNumberFragment.kt b/app/src/main/java/org/thoughtcrime/securesms/registration/ui/phonenumber/EnterPhoneNumberFragment.kt index cd324c4687..4c67881a73 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/registration/ui/phonenumber/EnterPhoneNumberFragment.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/registration/ui/phonenumber/EnterPhoneNumberFragment.kt @@ -562,6 +562,7 @@ class EnterPhoneNumberFragment : LoggingFragment(R.layout.fragment_registration_ EnterPhoneNumberMode.RESTART_AFTER_COLLECTION -> startNormalRegistration() EnterPhoneNumberMode.COLLECT_FOR_MANUAL_SIGNAL_BACKUPS_RESTORE -> findNavController().safeNavigate(EnterPhoneNumberFragmentDirections.goToEnterBackupKey()) + EnterPhoneNumberMode.COLLECT_FOR_LOCAL_V2_SIGNAL_BACKUPS_RESTORE -> findNavController().safeNavigate(EnterPhoneNumberFragmentDirections.goToInternalNewLocalBackupRestore()) } } diff --git a/app/src/main/java/org/thoughtcrime/securesms/registration/ui/phonenumber/EnterPhoneNumberMode.kt b/app/src/main/java/org/thoughtcrime/securesms/registration/ui/phonenumber/EnterPhoneNumberMode.kt index 643bad0e2f..e2e06f9be9 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/registration/ui/phonenumber/EnterPhoneNumberMode.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/registration/ui/phonenumber/EnterPhoneNumberMode.kt @@ -12,9 +12,12 @@ enum class EnterPhoneNumberMode { /** Normal registration start, collect number to verify */ NORMAL, - /** User pre-selected restore/transfer flow, collect number to re-register and restore with */ + /** User pre-selected restore/transfer flow, collect number to re-register and restore via remote */ COLLECT_FOR_MANUAL_SIGNAL_BACKUPS_RESTORE, + /** User pre-selected restore/transfer flow, collect number to re-register and restore via local backup v2 */ + COLLECT_FOR_LOCAL_V2_SIGNAL_BACKUPS_RESTORE, + /** User reversed decision on restore and needs to resume normal re-register but automatically start verify */ RESTART_AFTER_COLLECTION } diff --git a/app/src/main/java/org/thoughtcrime/securesms/registration/ui/restore/SelectManualRestoreMethodFragment.kt b/app/src/main/java/org/thoughtcrime/securesms/registration/ui/restore/SelectManualRestoreMethodFragment.kt index 52ae555d91..5acef1107c 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/registration/ui/restore/SelectManualRestoreMethodFragment.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/registration/ui/restore/SelectManualRestoreMethodFragment.kt @@ -20,11 +20,13 @@ import androidx.fragment.app.activityViewModels import androidx.navigation.fragment.findNavController import org.signal.core.ui.compose.Dialogs import org.signal.core.util.logging.Log +import org.thoughtcrime.securesms.BuildConfig import org.thoughtcrime.securesms.R import org.thoughtcrime.securesms.compose.ComposeFragment import org.thoughtcrime.securesms.registration.ui.RegistrationViewModel import org.thoughtcrime.securesms.registration.ui.phonenumber.EnterPhoneNumberMode import org.thoughtcrime.securesms.restore.RestoreActivity +import org.thoughtcrime.securesms.util.Environment import org.thoughtcrime.securesms.util.navigation.safeNavigate /** @@ -55,8 +57,16 @@ class SelectManualRestoreMethodFragment : ComposeFragment() { override fun FragmentContent() { var showSkipRestoreWarning by remember { mutableStateOf(false) } + val restoreMethods = remember { + if (Environment.IS_NIGHTLY || BuildConfig.DEBUG) { + listOf(RestoreMethod.FROM_SIGNAL_BACKUPS, RestoreMethod.FROM_LOCAL_BACKUP_V1, RestoreMethod.FROM_LOCAL_BACKUP_V2) + } else { + listOf(RestoreMethod.FROM_SIGNAL_BACKUPS, RestoreMethod.FROM_LOCAL_BACKUP_V1) + } + } + SelectRestoreMethodScreen( - restoreMethods = listOf(RestoreMethod.FROM_SIGNAL_BACKUPS, RestoreMethod.FROM_LOCAL_BACKUP_V1), + restoreMethods = restoreMethods, onRestoreMethodClicked = this::startRestoreMethod, onSkip = { showSkipRestoreWarning = true @@ -92,7 +102,11 @@ class SelectManualRestoreMethodFragment : ComposeFragment() { localBackupRestore.launch(RestoreActivity.getLocalRestoreIntent(requireContext())) } RestoreMethod.FROM_OLD_DEVICE -> error("Device transfer not supported in manual restore flow") - RestoreMethod.FROM_LOCAL_BACKUP_V2 -> error("Not currently supported") + RestoreMethod.FROM_LOCAL_BACKUP_V2 -> { + sharedViewModel.clearPreviousRegistrationState() + sharedViewModel.intendToRestore(hasOldDevice = false, fromRemote = false, fromLocalV2 = true) + findNavController().safeNavigate(SelectManualRestoreMethodFragmentDirections.goToEnterPhoneNumber(EnterPhoneNumberMode.COLLECT_FOR_LOCAL_V2_SIGNAL_BACKUPS_RESTORE)) + } } } } diff --git a/app/src/main/java/org/thoughtcrime/securesms/registration/ui/restore/local/InternalNewLocalBackupRestore.kt b/app/src/main/java/org/thoughtcrime/securesms/registration/ui/restore/local/InternalNewLocalBackupRestore.kt new file mode 100644 index 0000000000..88e8e67bcd --- /dev/null +++ b/app/src/main/java/org/thoughtcrime/securesms/registration/ui/restore/local/InternalNewLocalBackupRestore.kt @@ -0,0 +1,312 @@ +/* + * Copyright 2025 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ + +package org.thoughtcrime.securesms.registration.ui.restore.local + +import android.app.Activity +import android.content.Intent +import android.net.Uri +import android.os.Build +import android.os.Bundle +import android.provider.DocumentsContract +import android.widget.Toast +import androidx.activity.result.ActivityResultLauncher +import androidx.activity.result.contract.ActivityResultContracts +import androidx.compose.foundation.clickable +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Spacer +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.height +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.text.KeyboardActions +import androidx.compose.foundation.text.KeyboardOptions +import androidx.compose.material3.LocalTextStyle +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.Text +import androidx.compose.material3.TextField +import androidx.compose.runtime.Composable +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember +import androidx.compose.runtime.setValue +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.focus.FocusRequester +import androidx.compose.ui.focus.focusRequester +import androidx.compose.ui.layout.onGloballyPositioned +import androidx.compose.ui.platform.LocalSoftwareKeyboardController +import androidx.compose.ui.res.stringResource +import androidx.compose.ui.text.input.ImeAction +import androidx.compose.ui.text.input.KeyboardCapitalization +import androidx.compose.ui.text.input.KeyboardType +import androidx.compose.ui.unit.dp +import androidx.compose.ui.unit.sp +import androidx.fragment.app.activityViewModels +import androidx.lifecycle.compose.collectAsStateWithLifecycle +import kotlinx.coroutines.flow.map +import org.signal.core.models.AccountEntropyPool +import org.signal.core.ui.compose.Buttons +import org.signal.core.ui.compose.DayNightPreviews +import org.signal.core.ui.compose.Previews +import org.signal.core.util.logging.Log +import org.thoughtcrime.securesms.R +import org.thoughtcrime.securesms.compose.ComposeFragment +import org.thoughtcrime.securesms.fonts.MonoTypeface +import org.thoughtcrime.securesms.keyvalue.SignalStore +import org.thoughtcrime.securesms.registration.ui.RegistrationViewModel +import org.thoughtcrime.securesms.registration.ui.restore.AccountEntropyPoolVerification +import org.thoughtcrime.securesms.registration.ui.restore.BackupKeyVisualTransformation +import org.thoughtcrime.securesms.registration.ui.restore.attachBackupKeyAutoFillHelper +import org.thoughtcrime.securesms.registration.ui.restore.backupKeyAutoFillHelper +import org.thoughtcrime.securesms.registration.ui.shared.RegistrationScreen +import org.thoughtcrime.securesms.util.StorageUtil + +/** + * Internal only registration screen to collect backup folder and AEP. Actual restore will happen + * post-registration when the app re-routes to [org.thoughtcrime.securesms.restore.RestoreActivity] and then + * [InternalNewLocalRestoreActivity]. Yay implicit navigation! + */ +class InternalNewLocalBackupRestore : ComposeFragment() { + + private val TAG = Log.tag(InternalNewLocalBackupRestore::class) + + private val sharedViewModel by activityViewModels() + + private lateinit var chooseBackupLocationLauncher: ActivityResultLauncher + private val directoryFlow = SignalStore.backup.newLocalBackupsDirectoryFlow.map { if (Build.VERSION.SDK_INT >= 24 && it != null) StorageUtil.getDisplayPath(requireContext(), Uri.parse(it)) else it } + + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) + + chooseBackupLocationLauncher = registerForActivityResult( + ActivityResultContracts.StartActivityForResult() + ) { result -> + if (result.resultCode == Activity.RESULT_OK && result.data?.data != null) { + handleBackupLocationSelected(result.data!!.data!!) + } else { + Log.w(TAG, "Backup location selection cancelled or failed") + } + } + } + + @Composable + override fun FragmentContent() { + val selectedDirectory: String? by directoryFlow.collectAsStateWithLifecycle(SignalStore.backup.newLocalBackupsDirectory) + + InternalNewLocalBackupRestoreScreen( + selectedDirectory = selectedDirectory, + callback = CallbackImpl() + ) + } + + private fun launchBackupDirectoryPicker() { + val intent = Intent(Intent.ACTION_OPEN_DOCUMENT_TREE) + + if (Build.VERSION.SDK_INT >= 26) { + val currentDirectory = SignalStore.backup.newLocalBackupsDirectory + if (currentDirectory != null) { + intent.putExtra(DocumentsContract.EXTRA_INITIAL_URI, Uri.parse(currentDirectory)) + } + } + + intent.addFlags( + Intent.FLAG_GRANT_PERSISTABLE_URI_PERMISSION or + Intent.FLAG_GRANT_WRITE_URI_PERMISSION or + Intent.FLAG_GRANT_READ_URI_PERMISSION + ) + + try { + Log.d(TAG, "Launching backup directory picker") + chooseBackupLocationLauncher.launch(intent) + } catch (e: Exception) { + Log.w(TAG, "Failed to launch backup directory picker", e) + Toast.makeText(requireContext(), R.string.BackupDialog_no_file_picker_available, Toast.LENGTH_LONG).show() + } + } + + private fun handleBackupLocationSelected(uri: Uri) { + Log.i(TAG, "Backup location selected: $uri") + + val takeFlags = Intent.FLAG_GRANT_READ_URI_PERMISSION or Intent.FLAG_GRANT_WRITE_URI_PERMISSION + + requireContext().contentResolver.takePersistableUriPermission(uri, takeFlags) + + SignalStore.backup.newLocalBackupsDirectory = uri.toString() + + Toast.makeText(requireContext(), "Directory selected: $uri", Toast.LENGTH_SHORT).show() + } + + private inner class CallbackImpl : Callback { + override fun onSelectDirectoryClick() { + launchBackupDirectoryPicker() + } + + override fun onRestoreClick(backupKey: String) { + sharedViewModel.registerWithBackupKey( + context = requireContext(), + backupKey = backupKey, + e164 = null, + pin = null, + aciIdentityKeyPair = null, + pniIdentityKeyPair = null + ) + } + } +} + +private interface Callback { + fun onSelectDirectoryClick() + fun onRestoreClick(backupKey: String) + + object Empty : Callback { + override fun onSelectDirectoryClick() = Unit + override fun onRestoreClick(backupKey: String) = Unit + } +} + +@Composable +private fun InternalNewLocalBackupRestoreScreen( + selectedDirectory: String? = null, + callback: Callback +) { + var backupKey by remember { mutableStateOf("") } + var isBackupKeyValid by remember { mutableStateOf(false) } + var aepValidationError by remember { mutableStateOf(null) } + + val visualTransform = remember { BackupKeyVisualTransformation(chunkSize = 4) } + val keyboardController = LocalSoftwareKeyboardController.current + val focusRequester = remember { FocusRequester() } + var requestFocus by remember { mutableStateOf(true) } + + val autoFillHelper = backupKeyAutoFillHelper { newValue -> + backupKey = newValue + val (valid, error) = AccountEntropyPoolVerification.verifyAEP( + backupKey = newValue, + changed = true, + previousAEPValidationError = aepValidationError + ) + isBackupKeyValid = valid + aepValidationError = error + } + + RegistrationScreen( + title = "Local Backup V2 Restore", + subtitle = null, + bottomContent = { + Buttons.LargeTonal( + onClick = { callback.onRestoreClick(backupKey) }, + enabled = isBackupKeyValid && aepValidationError == null && selectedDirectory != null, + modifier = Modifier.align(Alignment.CenterEnd) + ) { + Text(text = "Restore") + } + } + ) { + Column( + modifier = Modifier.fillMaxWidth() + ) { + DirectorySelectionRow( + selectedDirectory = selectedDirectory, + onClick = callback::onSelectDirectoryClick + ) + + Spacer(modifier = Modifier.height(24.dp)) + + TextField( + value = backupKey, + onValueChange = { value -> + val newKey = AccountEntropyPool.removeIllegalCharacters(value).take(AccountEntropyPool.LENGTH + 16).lowercase() + val (valid, error) = AccountEntropyPoolVerification.verifyAEP( + backupKey = newKey, + changed = backupKey != newKey, + previousAEPValidationError = aepValidationError + ) + backupKey = newKey + isBackupKeyValid = valid + aepValidationError = error + autoFillHelper.onValueChanged(newKey) + }, + label = { + Text(text = stringResource(id = R.string.EnterBackupKey_backup_key)) + }, + textStyle = LocalTextStyle.current.copy( + fontFamily = MonoTypeface.fontFamily(), + lineHeight = 36.sp + ), + keyboardOptions = KeyboardOptions( + keyboardType = KeyboardType.Password, + capitalization = KeyboardCapitalization.None, + imeAction = ImeAction.Done, + autoCorrectEnabled = false + ), + keyboardActions = KeyboardActions( + onDone = { + if (isBackupKeyValid && aepValidationError == null && selectedDirectory != null) { + keyboardController?.hide() + callback.onRestoreClick(backupKey) + } + } + ), + supportingText = { aepValidationError?.ValidationErrorMessage() }, + isError = aepValidationError != null, + minLines = 4, + visualTransformation = visualTransform, + modifier = Modifier + .fillMaxWidth() + .focusRequester(focusRequester) + .attachBackupKeyAutoFillHelper(autoFillHelper) + .onGloballyPositioned { + if (requestFocus) { + focusRequester.requestFocus() + requestFocus = false + } + } + ) + } + } +} + +@Composable +private fun DirectorySelectionRow( + selectedDirectory: String?, + onClick: () -> Unit +) { + Column( + modifier = Modifier + .fillMaxWidth() + .clickable(onClick = onClick) + .padding(horizontal = 16.dp, vertical = 12.dp) + ) { + Text( + text = "Select Backup Directory", + style = MaterialTheme.typography.bodyLarge + ) + Text( + text = selectedDirectory ?: "No directory selected", + style = MaterialTheme.typography.bodyMedium, + color = MaterialTheme.colorScheme.onSurfaceVariant + ) + } +} + +@Composable +private fun AccountEntropyPoolVerification.AEPValidationError.ValidationErrorMessage() { + when (this) { + is AccountEntropyPoolVerification.AEPValidationError.TooLong -> Text(text = stringResource(R.string.EnterBackupKey_too_long_error, this.count, this.max)) + AccountEntropyPoolVerification.AEPValidationError.Invalid -> Text(text = stringResource(R.string.EnterBackupKey_invalid_backup_key_error)) + AccountEntropyPoolVerification.AEPValidationError.Incorrect -> Text(text = stringResource(R.string.EnterBackupKey_incorrect_backup_key_error)) + } +} + +@DayNightPreviews +@Composable +private fun InternalNewLocalBackupRestoreScreenPreview() { + Previews.Preview { + InternalNewLocalBackupRestoreScreen( + selectedDirectory = "/storage/emulated/0/Signal/Backups", + callback = Callback.Empty + ) + } +} diff --git a/app/src/main/java/org/thoughtcrime/securesms/registration/ui/restore/local/InternalNewLocalRestoreActivity.kt b/app/src/main/java/org/thoughtcrime/securesms/registration/ui/restore/local/InternalNewLocalRestoreActivity.kt new file mode 100644 index 0000000000..7ec5f62a3f --- /dev/null +++ b/app/src/main/java/org/thoughtcrime/securesms/registration/ui/restore/local/InternalNewLocalRestoreActivity.kt @@ -0,0 +1,151 @@ +/* + * Copyright 2025 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ + +package org.thoughtcrime.securesms.registration.ui.restore.local + +import android.content.Context +import android.content.Intent +import android.net.Uri +import android.os.Bundle +import android.widget.Toast +import androidx.activity.compose.setContent +import androidx.compose.foundation.layout.padding +import androidx.compose.material3.CircularProgressIndicator +import androidx.compose.material3.Surface +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.setValue +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.unit.dp +import androidx.lifecycle.lifecycleScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext +import org.greenrobot.eventbus.EventBus +import org.greenrobot.eventbus.Subscribe +import org.greenrobot.eventbus.ThreadMode +import org.signal.core.ui.compose.DayNightPreviews +import org.signal.core.ui.compose.Previews +import org.signal.core.util.Result +import org.signal.libsignal.zkgroup.profiles.ProfileKey +import org.thoughtcrime.securesms.BaseActivity +import org.thoughtcrime.securesms.MainActivity +import org.thoughtcrime.securesms.backup.v2.BackupRepository +import org.thoughtcrime.securesms.backup.v2.RestoreV2Event +import org.thoughtcrime.securesms.backup.v2.local.ArchiveFileSystem +import org.thoughtcrime.securesms.backup.v2.local.LocalArchiver +import org.thoughtcrime.securesms.backup.v2.local.SnapshotFileSystem +import org.thoughtcrime.securesms.compose.SignalTheme +import org.thoughtcrime.securesms.conversation.v2.registerForLifecycle +import org.thoughtcrime.securesms.database.model.databaseprotos.RestoreDecisionState +import org.thoughtcrime.securesms.dependencies.AppDependencies +import org.thoughtcrime.securesms.jobs.RestoreLocalAttachmentJob +import org.thoughtcrime.securesms.keyvalue.Completed +import org.thoughtcrime.securesms.keyvalue.SignalStore +import org.thoughtcrime.securesms.recipients.Recipient +import org.thoughtcrime.securesms.registration.ui.restore.StorageServiceRestore +import org.thoughtcrime.securesms.registration.ui.shared.RegistrationScreen +import org.thoughtcrime.securesms.registration.util.RegistrationUtil + +/** + * Internal only. On launch, attempt to import the most recent backup located in [SignalStore.backup].newLocalBackupsDirectory. + */ +class InternalNewLocalRestoreActivity : BaseActivity() { + companion object { + fun getIntent(context: Context, finish: Boolean = true): Intent = Intent(context, InternalNewLocalRestoreActivity::class.java).apply { putExtra("finish", finish) } + } + + private var restoreStatus by mutableStateOf("Unknown") + + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) + + lifecycleScope.launch(Dispatchers.IO) { + restoreStatus = "Starting..." + + val self = Recipient.self() + val selfData = BackupRepository.SelfData(self.aci.get(), self.pni.get(), self.e164.get(), ProfileKey(self.profileKey)) + + val archiveFileSystem = ArchiveFileSystem.fromUri(AppDependencies.application, Uri.parse(SignalStore.backup.newLocalBackupsDirectory!!))!! + val snapshotInfo = archiveFileSystem.listSnapshots().first() + val snapshotFileSystem = SnapshotFileSystem(AppDependencies.application, snapshotInfo.file) + + val result = LocalArchiver.import(snapshotFileSystem, selfData) + + if (result is Result.Success) { + restoreStatus = "Success! Finishing" + val mediaNameToFileInfo = archiveFileSystem.filesFileSystem.allFiles() + RestoreLocalAttachmentJob.enqueueRestoreLocalAttachmentsJobs(mediaNameToFileInfo) + + SignalStore.registration.restoreDecisionState = RestoreDecisionState.Completed + + SignalStore.backup.backupSecretRestoreRequired = false + StorageServiceRestore.restore() + + withContext(Dispatchers.Main) { + Toast.makeText(this@InternalNewLocalRestoreActivity, "Local backup restored!", Toast.LENGTH_SHORT).show() + RegistrationUtil.maybeMarkRegistrationComplete() + startActivity(MainActivity.clearTop(this@InternalNewLocalRestoreActivity)) + if (intent.getBooleanExtra("finish", false)) { + finishAffinity() + } + } + } else { + restoreStatus = "Backup failed" + Toast.makeText(this@InternalNewLocalRestoreActivity, "Local backup failed", Toast.LENGTH_SHORT).show() + } + } + + setContent { + SignalTheme { + Surface { + InternalNewLocalRestoreScreen( + status = restoreStatus + ) + } + } + } + + EventBus.getDefault().registerForLifecycle(subscriber = this, lifecycleOwner = this) + } + + @Subscribe(threadMode = ThreadMode.MAIN) + fun onEvent(restoreEvent: RestoreV2Event) { + this.restoreStatus = "${restoreEvent.type}: ${restoreEvent.count} / ${restoreEvent.estimatedTotalCount}" + } +} + +@Composable +private fun InternalNewLocalRestoreScreen( + status: String = "" +) { + RegistrationScreen( + title = "Internal - Local Restore", + subtitle = null, + bottomContent = { } + ) { + Text( + text = status, + modifier = Modifier + .align(Alignment.CenterHorizontally) + .padding(bottom = 16.dp) + ) + + CircularProgressIndicator( + modifier = Modifier.align(Alignment.CenterHorizontally) + ) + } +} + +@DayNightPreviews +@Composable +private fun InternalNewLocalRestorePreview() { + Previews.Preview { + InternalNewLocalRestoreScreen(status = "Importing...") + } +} diff --git a/app/src/main/java/org/thoughtcrime/securesms/restore/RestoreActivity.kt b/app/src/main/java/org/thoughtcrime/securesms/restore/RestoreActivity.kt index e76c5299ab..768fdee03b 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/restore/RestoreActivity.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/restore/RestoreActivity.kt @@ -24,14 +24,18 @@ import org.signal.core.util.ThreadUtil import org.signal.core.util.getParcelableExtraCompat import org.signal.core.util.logging.Log import org.thoughtcrime.securesms.BaseActivity +import org.thoughtcrime.securesms.BuildConfig import org.thoughtcrime.securesms.MainActivity import org.thoughtcrime.securesms.PassphraseRequiredActivity import org.thoughtcrime.securesms.R import org.thoughtcrime.securesms.RestoreDirections import org.thoughtcrime.securesms.keyvalue.SignalStore import org.thoughtcrime.securesms.keyvalue.isWantingManualRemoteRestore +import org.thoughtcrime.securesms.keyvalue.isWantingNewLocalBackupRestore import org.thoughtcrime.securesms.registration.ui.restore.RemoteRestoreActivity +import org.thoughtcrime.securesms.registration.ui.restore.local.InternalNewLocalRestoreActivity import org.thoughtcrime.securesms.util.DynamicNoActionBarTheme +import org.thoughtcrime.securesms.util.Environment import org.thoughtcrime.securesms.util.TextSecurePreferences import org.thoughtcrime.securesms.util.navigation.safeNavigate @@ -79,6 +83,8 @@ class RestoreActivity : BaseActivity() { if (SignalStore.registration.restoreDecisionState.isWantingManualRemoteRestore) { Log.i(TAG, "User has no available restore methods but previously wanted a remote restore, navigating immediately.") startActivity(RemoteRestoreActivity.getIntent(this, isOnlyOption = true)) + } else if (SignalStore.registration.restoreDecisionState.isWantingNewLocalBackupRestore && (BuildConfig.DEBUG || Environment.IS_NIGHTLY)) { + startActivity(InternalNewLocalRestoreActivity.getIntent(this)) } else { Log.i(TAG, "No restore methods available, skipping") sharedViewModel.skipRestore() diff --git a/app/src/main/java/org/thoughtcrime/securesms/service/LocalBackupListener.java b/app/src/main/java/org/thoughtcrime/securesms/service/LocalBackupListener.java index f7888530fb..1232f0ad9a 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/service/LocalBackupListener.java +++ b/app/src/main/java/org/thoughtcrime/securesms/service/LocalBackupListener.java @@ -34,11 +34,15 @@ public class LocalBackupListener extends PersistentAlarmManagerListener { LocalBackupJob.enqueue(false); } + if (SignalStore.backup().getNewLocalBackupsEnabled()) { + LocalBackupJob.enqueueArchive(SignalStore.settings().isBackupEnabled()); + } + return setNextBackupTimeToIntervalFromNow(context); } public static void schedule(Context context) { - if (SignalStore.settings().isBackupEnabled()) { + if (SignalStore.settings().isBackupEnabled() || SignalStore.backup().getNewLocalBackupsEnabled()) { new LocalBackupListener().onReceive(context, getScheduleIntent()); } } diff --git a/app/src/main/protowire/Database.proto b/app/src/main/protowire/Database.proto index d04f85a668..c5f864d4bb 100644 --- a/app/src/main/protowire/Database.proto +++ b/app/src/main/protowire/Database.proto @@ -604,6 +604,7 @@ message RestoreDecisionState { message IntendToRestoreData { bool hasOldDevice = 1; optional bool fromRemote = 2; + optional bool fromLocalV2 = 3; } State decisionState = 1; diff --git a/app/src/main/protowire/LocalArchive.proto b/app/src/main/protowire/LocalArchive.proto index 2233a1e9c4..5381d79071 100644 --- a/app/src/main/protowire/LocalArchive.proto +++ b/app/src/main/protowire/LocalArchive.proto @@ -3,9 +3,18 @@ syntax = "proto3"; package signal.backup.local; option java_package = "org.thoughtcrime.securesms.backup.v2.local.proto"; +option swift_prefix = "LocalBackupProto_"; message Metadata { + message EncryptedBackupId { + bytes iv = 1; // 12 bytes, randomly generated + bytes encryptedId = 2; // AES-256-CTR, key = local backup metadata key, message = backup ID bytes + // local backup metadata key = hkdf(input: K_B, info: UTF8("20241011_SIGNAL_LOCAL_BACKUP_METADATA_KEY"), length: 32) + // No hash of the ID; if it's decrypted incorrectly, the main backup will fail to decrypt anyway. + } + uint32 version = 1; + EncryptedBackupId backupId = 2; // used to decrypt the backup file knowing only the Account Entropy Pool } message FilesFrame { diff --git a/app/src/main/res/navigation/app_settings_with_change_number.xml b/app/src/main/res/navigation/app_settings_with_change_number.xml index 8444d4f198..73c9823922 100644 --- a/app/src/main/res/navigation/app_settings_with_change_number.xml +++ b/app/src/main/res/navigation/app_settings_with_change_number.xml @@ -1142,6 +1142,14 @@ app:popEnterAnim="@anim/fragment_close_enter" app:popExitAnim="@anim/fragment_close_exit" /> + + + + + diff --git a/app/src/main/res/navigation/registration_v3.xml b/app/src/main/res/navigation/registration_v3.xml index 35c09ede4d..587d5554cf 100644 --- a/app/src/main/res/navigation/registration_v3.xml +++ b/app/src/main/res/navigation/registration_v3.xml @@ -222,6 +222,14 @@ app:popEnterAnim="@anim/nav_default_pop_enter_anim" app:popExitAnim="@anim/nav_default_pop_exit_anim" /> + + @@ -243,6 +251,12 @@ + + + + >): PluginResult { + val api = parameters["api"]?.firstOrNull() + + if (api == null) { + val apiButtons = apis.keys.joinToString("\n") { apiName -> + """ +
+ + +
+ """.trimIndent() + } + + val apiScripts = apis.keys.joinToString("\n") { apiName -> + """ + document.getElementById('btn_$apiName').addEventListener('click', async function() { + const resultSpan = document.getElementById('result_$apiName'); + resultSpan.textContent = 'Loading...'; + + try { + const response = await fetch('$PATH?api=$apiName'); + if (!response.ok) { + const errorText = await response.text(); + resultSpan.textContent = 'Error: ' + errorText; + return; + } + const data = await response.json(); + resultSpan.textContent = 'Result: ' + JSON.stringify(data, null, 2); + } catch (error) { + resultSpan.textContent = 'Error: ' + error.message; + } + }); + """.trimIndent() + } + + val html = """ +

Available APIs

+ $apiButtons + + + """.trimIndent() + + return PluginResult.RawHtmlResult(html) + } + + return apis[api]?.invoke(parameters) ?: PluginResult.ErrorResult.notFound(message = "not found") + } + + private fun localBackups(parameters: Map>): PluginResult { + // Check if cache bust is requested + val cacheBust = parameters["cacheBust"]?.firstOrNull() == "true" + if (cacheBust) { + Log.i(TAG, "Cache bust requested, clearing local backups cache") + PluginCache.clearBackupCache() + } + + if (PluginCache.localBackups != null) { + return PluginResult.JsonResult(JsonUtil.toJson(PluginCache.localBackups)) + } + + val fs = PluginCache.getArchiveFileSystem() ?: return PluginResult.ErrorResult(message = "Unable to load archive file system! Ensure backup directory is configured.") + + val snapshots = fs.listSnapshots() + + PluginCache.localBackups = LocalBackups( + snapshots.map { LocalBackup(name = "${it.name} - ${it.timestamp}", it.timestamp) } + ) + + return PluginResult.JsonResult(JsonUtil.toJson(PluginCache.localBackups)) + } + + data class LocalBackups @JsonCreator constructor(@field:JsonProperty val backups: List) + + data class LocalBackup @JsonCreator constructor(@field:JsonProperty val name: String, @field:JsonProperty val timestamp: Long) +} diff --git a/app/src/spinner/java/org/thoughtcrime/securesms/BackupPlugin.kt b/app/src/spinner/java/org/thoughtcrime/securesms/BackupPlugin.kt new file mode 100644 index 0000000000..733d9d5743 --- /dev/null +++ b/app/src/spinner/java/org/thoughtcrime/securesms/BackupPlugin.kt @@ -0,0 +1,648 @@ +/* + * Copyright 2025 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ + +package org.thoughtcrime.securesms + +import okio.ByteString +import org.signal.core.util.bytes +import org.signal.core.util.decodeOrNull +import org.signal.core.util.logging.Log +import org.signal.libsignal.zkgroup.profiles.ProfileKey +import org.signal.spinner.Plugin +import org.signal.spinner.PluginResult +import org.thoughtcrime.securesms.backup.v2.BackupRepository +import org.thoughtcrime.securesms.backup.v2.local.SnapshotFileSystem +import org.thoughtcrime.securesms.backup.v2.proto.AccountData +import org.thoughtcrime.securesms.backup.v2.proto.BackupDebugInfo +import org.thoughtcrime.securesms.backup.v2.proto.FilePointer +import org.thoughtcrime.securesms.backup.v2.stream.EncryptedBackupReader +import org.thoughtcrime.securesms.dependencies.AppDependencies +import org.thoughtcrime.securesms.keyvalue.SignalStore +import org.thoughtcrime.securesms.net.SignalNetwork +import org.thoughtcrime.securesms.providers.BlobProvider +import org.thoughtcrime.securesms.recipients.Recipient +import org.whispersystems.signalservice.api.NetworkResult +import org.whispersystems.signalservice.api.svr.SvrBApi +import java.io.IOException +import java.lang.StringBuilder +import java.text.SimpleDateFormat +import java.util.Date +import java.util.Locale + +class BackupPlugin : Plugin { + companion object { + private val TAG = Log.tag(BackupPlugin::class.java) + const val PATH = "/backups" + } + + override val name: String = "Backups" + override val path: String = PATH + + override fun get(parameters: Map>): PluginResult { + val page = """ +

Remote Backup

+

${getRemoteBackup()} +

Local Backups

+

+ ${getLocalBackups()} +

+

Selected Backup

+ ${if (parameters.containsKey("remoteBackup")) getSelectedRemoteBackup() else getSelectedLocalBackup(parameters["localBackup"])} + """.trimIndent() + + return PluginResult.RawHtmlResult(page) + } + + private fun getSelectedRemoteBackup(): String { + Log.d(TAG, "Downloading file...") + val tempBackupFile = BlobProvider.getInstance().forNonAutoEncryptingSingleSessionOnDisk(AppDependencies.application) + + when (val result = BackupRepository.downloadBackupFile(tempBackupFile)) { + is NetworkResult.Success -> Log.i(TAG, "Download successful") + else -> { + Log.w(TAG, "Failed to download backup file", result.getCause()) + return result.getCause().toString() + } + } + + val forwardSecrecyMetadata = tempBackupFile.inputStream().use { EncryptedBackupReader.readForwardSecrecyMetadata(it) } + if (forwardSecrecyMetadata == null) { + return "Failed to read forward secrecy metadata!" + } + + val svrBAuth = when (val result = BackupRepository.getSvrBAuth()) { + is NetworkResult.Success -> result.result + else -> return "Failed to read forward secrecy metadata!" + } + + val forwardSecrecyToken = when (val result = SignalNetwork.svrB.restore(svrBAuth, SignalStore.backup.messageBackupKey, forwardSecrecyMetadata)) { + is SvrBApi.RestoreResult.Success -> result.data.forwardSecrecyToken + else -> return "Failed to read forward secrecy metadata! $result" + } + + val self = Recipient.self() + val selfData = BackupRepository.SelfData(self.aci.get(), self.pni.get(), self.e164.get(), ProfileKey(self.profileKey)) + + val backupKey = SignalStore.backup.messageBackupKey + + val frameReader = EncryptedBackupReader.createForSignalBackup( + key = backupKey, + aci = selfData.aci, + forwardSecrecyToken = forwardSecrecyToken, + length = tempBackupFile.length(), + dataStream = { tempBackupFile.inputStream() } + ) + + val output = StringBuilder() + + frameReader.use { reader -> + dumpBackupData(reader, output) + } + + return output.toString() + } + + private fun getSelectedLocalBackup(key: List?): String { + if (key?.size != 1) { + return "No selection" + } + + val timestamp = key.first().toLongOrNull() ?: return "Timestamp invalid" + + val fs = PluginCache.getArchiveFileSystem() ?: return "Unable to load archive file system! Ensure backup directory is configured." + + val snapshot = fs.listSnapshots().firstOrNull { it.timestamp == timestamp } ?: return "No snapshot for timestamp $timestamp" + val snapshotFS = SnapshotFileSystem(AppDependencies.application, snapshot.file) + + val self = Recipient.self() + val selfData = BackupRepository.SelfData(self.aci.get(), self.pni.get(), self.e164.get(), ProfileKey(self.profileKey)) + + val backupKey = SignalStore.backup.messageBackupKey + + val frameReader = try { + EncryptedBackupReader.createForLocalOrLinking( + key = backupKey, + aci = selfData.aci, + length = snapshotFS.mainLength()!!, + dataStream = { snapshotFS.mainInputStream()!! } + ) + } catch (e: IOException) { + return "Unable to import local archive: $e" + } + + val output = StringBuilder() + + frameReader.use { reader -> + dumpBackupData(reader, output) + } + + return output.toString() + } + + private fun dumpBackupData(reader: EncryptedBackupReader, output: StringBuilder) { + val header = reader.getHeader() + if (header != null) { + val debugInfoRow = if (header.debugInfo.size > 0) { + val debugInfo = BackupDebugInfo.ADAPTER.decodeOrNull(header.debugInfo.toByteArray()) + """Debug Info${formatDebugInfo(debugInfo)}""" + } else { + "" + } + + output.append( + """ +
Header
+ + + + + + + + $debugInfoRow +
FieldValue
Version${header.version}
Backup Time${formatTimestamp(header.backupTimeMs)}
Media Root Backup Key${formatBytes(header.mediaRootBackupKey)}
Current App Version${header.currentAppVersion ?: "N/A"}
First App Version${header.firstAppVersion ?: "N/A"}
+ """.trimIndent() + ) + } else { + output.append("

No header found

") + } + + // Collect frame data on the fly + Log.i(TAG, "Starting to process backup frames") + var accountData: AccountData? = null + + // Store only essential data + val recipientNames = mutableMapOf() + val chatInfos = mutableMapOf() + + // Statistics counters + var contactCount = 0 + var groupCount = 0 + var distributionListCount = 0 + var selfCount = 0 + var releaseNotesCount = 0 + var callLinkCount = 0 + var totalRecipients = 0 + var totalFrames = 0 + + val attachmentCounters = AttachmentCounters() + + val startTime = System.currentTimeMillis() + + for (frame in reader) { + totalFrames++ + + if (totalFrames % 10000 == 0) { + Log.i(TAG, "Processed $totalFrames frames...") + } + + when { + frame.account != null -> { + accountData = frame.account + Log.i(TAG, "Found account data") + } + frame.recipient != null -> { + totalRecipients++ + val recipient = frame.recipient + recipientNames[recipient.id] = extractRecipientName(recipient) + + when { + recipient.contact != null -> contactCount++ + recipient.group != null -> groupCount++ + recipient.distributionList != null -> distributionListCount++ + recipient.self != null -> selfCount++ + recipient.releaseNotes != null -> releaseNotesCount++ + recipient.callLink != null -> callLinkCount++ + } + } + frame.chat != null -> { + val chat = frame.chat + chatInfos[chat.id] = ChatInfo( + chatId = chat.id, + recipientId = chat.recipientId, + archived = chat.archived, + pinnedOrder = chat.pinnedOrder, + messageCount = 0, + attachmentCount = 0 + ) + + // Count wallpaper photo + chat.style?.wallpaperPhoto?.let { wallpaper -> + processFilePointer(wallpaper, attachmentCounters, FilePointerType.WALLPAPER) + } + } + frame.chatItem != null -> { + val item = frame.chatItem + chatInfos[item.chatId]?.let { chatInfo -> + chatInfo.messageCount++ + chatInfo.attachmentCount += item.standardMessage?.attachments?.size ?: 0 + } + + // Process StandardMessage file pointers + item.standardMessage?.let { msg -> + // Link preview images + msg.linkPreview.forEach { preview -> + preview.image?.let { image -> + processFilePointer(image, attachmentCounters, FilePointerType.LINK_PREVIEW) + } + } + + // Standard attachments + msg.attachments.forEach { attachment -> + attachment.pointer?.let { pointer -> + processFilePointer(pointer, attachmentCounters, FilePointerType.ATTACHMENT) + } + } + + // Long text + msg.longText?.let { longText -> + processFilePointer(longText, attachmentCounters, FilePointerType.LONG_TEXT) + } + + // Quote attachments + msg.quote?.attachments?.forEach { quotedAttachment -> + quotedAttachment.thumbnail?.pointer?.let { pointer -> + processFilePointer(pointer, attachmentCounters, FilePointerType.QUOTE) + } + } + } + + // Process StickerMessage file pointers + item.stickerMessage?.sticker?.data_?.let { stickerData -> + processFilePointer(stickerData, attachmentCounters, FilePointerType.STICKER) + } + + // Process ContactMessage file pointers + item.contactMessage?.contact?.avatar?.let { avatar -> + processFilePointer(avatar, attachmentCounters, FilePointerType.CONTACT_AVATAR) + } + } + } + } + + val elapsed = System.currentTimeMillis() - startTime + Log.i(TAG, "Finished processing $totalFrames frames in ${elapsed}ms") + + // Format account data + if (accountData != null) { + output.append(formatAccountData(accountData)) + } + + // Format recipients summary + output.append( + formatRecipientsSummary( + totalRecipients = totalRecipients, + contactCount = contactCount, + groupCount = groupCount, + distributionListCount = distributionListCount, + selfCount = selfCount, + releaseNotesCount = releaseNotesCount, + callLinkCount = callLinkCount + ) + ) + + // Format attachments summary + output.append(formatAttachmentsSummary(attachmentCounters)) + + // Format chats summary + output.append(formatChatsSummary(chatInfos, recipientNames)) + } + + private fun processFilePointer( + filePointer: FilePointer, + counters: AttachmentCounters, + type: FilePointerType + ) { + // Increment total attachments for any file pointer + counters.total++ + + // Check for plaintextHash + val hasPlaintextHash = filePointer.locatorInfo?.plaintextHash != null && filePointer.locatorInfo.plaintextHash.size > 0 + if (hasPlaintextHash) { + counters.withPlaintextHash++ + } else { + // Track missing plaintextHash by type + counters.withoutPlaintextHashByType[type] = counters.withoutPlaintextHashByType.getOrDefault(type, 0) + 1 + } + + // Check for localKey + val hasLocalKey = filePointer.locatorInfo?.localKey != null && filePointer.locatorInfo.localKey.size > 0 + if (hasLocalKey) { + counters.withLocalKey++ + } else { + // Track missing local keys by type + counters.withoutLocalKeyByType[type] = counters.withoutLocalKeyByType.getOrDefault(type, 0) + 1 + } + } + + private fun formatTimestamp(timestampMs: Long?): String { + if (timestampMs == null) return "N/A" + return try { + val date = Date(timestampMs) + val formatter = SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.US) + "${formatter.format(date)} (${timestampMs}ms)" + } catch (_: Exception) { + timestampMs.toString() + } + } + + private fun formatBytes(bytes: ByteString?): String { + if (bytes == null || bytes.size == 0) return "N/A" + return if (bytes.size <= 32) { + bytes.hex() + } else { + "${bytes.hex().take(64)}... (${bytes.size} bytes)" + } + } + + private fun formatDebugInfo(debugInfo: BackupDebugInfo?): String { + if (debugInfo == null) return "N/A" + + val attachmentDetailsHtml = debugInfo.attachmentDetails?.let { details -> + """ + + + + + + + + +
Attachment Details
Not Started${details.notStartedCount}
Upload In Progress${details.uploadInProgressCount}
Copy Pending${details.copyPendingCount}
Finished${details.finishedCount}
Permanent Failure${details.permanentFailureCount}
Temporary Failure${details.temporaryFailureCount}
+ """.trimIndent() + } ?: "No attachment details" + + val debugLogUrlHtml = if (debugInfo.debuglogUrl.isNotEmpty()) { + "${debugInfo.debuglogUrl}" + } else { + "N/A" + } + + return """ +
+ Debug Log URL: $debugLogUrlHtml
+ Using Paid Tier: ${debugInfo.usingPaidTier}
+ $attachmentDetailsHtml +
+ """.trimIndent() + } + + private fun formatAccountData(account: AccountData): String { + return """ +
Account Details
+ + + + + + + + + +
FieldValue
Given Name${account.givenName}
Family Name${account.familyName}
Username${account.username ?: "N/A"}
Profile Key${formatBytes(account.profileKey)}
Avatar URL Path${account.avatarUrlPath.ifEmpty { "N/A" }}
Bio Text${account.bioText.ifEmpty { "N/A" }}
Bio Emoji${account.bioEmoji.ifEmpty { "N/A" }}
+ """.trimIndent() + } + + private fun formatRecipientsSummary( + totalRecipients: Int, + contactCount: Int, + groupCount: Int, + distributionListCount: Int, + selfCount: Int, + releaseNotesCount: Int, + callLinkCount: Int + ): String { + return """ +
Recipients Summary
+ + + + + + + + + +
TypeCount
Total Recipients$totalRecipients
1:1 Contacts$contactCount
Groups$groupCount
Distribution Lists$distributionListCount
Self$selfCount
Release Notes$releaseNotesCount
Call Links$callLinkCount
+ """.trimIndent() + } + + private fun formatChatsSummary( + chatInfos: Map, + recipientNames: Map + ): String { + val output = StringBuilder() + + output.append( + """ +
Chats Summary
+

Total Chats: ${chatInfos.size}

+ """.trimIndent() + ) + + if (chatInfos.isNotEmpty()) { + output.append( + """ + + + + + + + + + + """.trimIndent() + ) + + chatInfos.values.sortedByDescending { it.messageCount }.forEach { chatInfo -> + val recipientName = recipientNames[chatInfo.recipientId] ?: "Unknown ${chatInfo.recipientId}" + + output.append( + """ + + + + + + + + + """.trimIndent() + ) + } + + output.append("
ChatRecipientMessagesAttachmentsArchivedPinned
${chatInfo.chatId}$recipientName${chatInfo.messageCount}${chatInfo.attachmentCount}${if (chatInfo.archived) "Yes" else "No"}${chatInfo.pinnedOrder?.let { "Yes (#$it)" } ?: "No"}
") + } + + return output.toString() + } + + private fun formatAttachmentsSummary( + counters: AttachmentCounters + ): String { + val totalWithoutPlaintextHash = counters.withoutPlaintextHashByType.values.sum() + val totalWithoutLocalKey = counters.withoutLocalKeyByType.values.sum() + + // Define the order we want to display the types + val orderedTypes = listOf( + FilePointerType.LINK_PREVIEW to "Link Previews", + FilePointerType.ATTACHMENT to "Attachments", + FilePointerType.LONG_TEXT to "Long Text", + FilePointerType.QUOTE to "Quotes", + FilePointerType.STICKER to "Stickers", + FilePointerType.CONTACT_AVATAR to "Contact Avatars", + FilePointerType.WALLPAPER to "Wallpapers" + ) + + return """ +
Attachments Summary
+ + + + + +
PropertyCount
Total File Pointers${counters.total}
With FilePointer LocatorInfo PlaintextHash${counters.withPlaintextHash}
With LocalKey${counters.withLocalKey}
+ +
File Pointers Missing Fields (by type)
+ + + + + + + ${orderedTypes.joinToString("\n") { (type, label) -> + val withoutHash = counters.withoutPlaintextHashByType.getOrDefault(type, 0) + val withoutKey = counters.withoutLocalKeyByType.getOrDefault(type, 0) + "" + }} + + + + + +
TypeWithout PlaintextHashWithout LocalKey
$label$withoutHash$withoutKey
Total$totalWithoutPlaintextHash$totalWithoutLocalKey
+ """.trimIndent() + } + + private fun extractRecipientName(recipient: org.thoughtcrime.securesms.backup.v2.proto.Recipient?): String { + if (recipient == null) return "Unknown" + + return when { + recipient.contact != null -> { + val contact = recipient.contact + val name = buildString { + if (contact.profileGivenName?.isNotEmpty() == true) { + append(contact.profileGivenName) + if (contact.profileFamilyName?.isNotEmpty() == true) { + append(" ${contact.profileFamilyName}") + } + } else if (contact.nickname?.given?.isNotEmpty() == true) { + append(contact.nickname.given) + if (contact.nickname.family.isNotEmpty()) { + append(" ${contact.nickname.family}") + } + } else if (contact.e164 != null) { + append("+${contact.e164}") + } else { + append("Contact ${recipient.id}") + } + } + name + } + recipient.group != null -> { + val group = recipient.group + group.snapshot?.title?.title ?: "Group ${recipient.id}" + } + recipient.self != null -> "Self" + recipient.releaseNotes != null -> "Release Notes" + recipient.distributionList != null -> "Distribution List ${recipient.id}" + recipient.callLink != null -> "Call Link ${recipient.id}" + else -> "Unknown ${recipient.id}" + } + } + + private fun getRemoteBackup(): String { + return if (SignalStore.backup.hasBackupBeenUploaded) { + """${formatTimestamp(SignalStore.backup.lastBackupTime)} - ${SignalStore.backup.lastBackupProtoSize.bytes.toUnitString()}""" + } else { + "No remote backup" + } + } + + private fun getLocalBackups(): String { + return """ +

Loading local backups...

+

+ + + """.trimIndent() + } + + private data class ChatInfo( + val chatId: Long, + val recipientId: Long, + val archived: Boolean, + val pinnedOrder: Int?, + var messageCount: Int, + var attachmentCount: Int + ) + + private enum class FilePointerType { + LINK_PREVIEW, + ATTACHMENT, + LONG_TEXT, + QUOTE, + STICKER, + CONTACT_AVATAR, + WALLPAPER + } + + private data class AttachmentCounters( + var total: Int = 0, + var withPlaintextHash: Int = 0, + var withLocalKey: Int = 0, + val withoutPlaintextHashByType: MutableMap = mutableMapOf(), + val withoutLocalKeyByType: MutableMap = mutableMapOf() + ) +} diff --git a/app/src/spinner/java/org/thoughtcrime/securesms/PluginCache.kt b/app/src/spinner/java/org/thoughtcrime/securesms/PluginCache.kt new file mode 100644 index 0000000000..06da7d262a --- /dev/null +++ b/app/src/spinner/java/org/thoughtcrime/securesms/PluginCache.kt @@ -0,0 +1,33 @@ +/* + * Copyright 2025 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ + +package org.thoughtcrime.securesms + +import android.net.Uri +import org.thoughtcrime.securesms.backup.v2.local.ArchiveFileSystem +import org.thoughtcrime.securesms.dependencies.AppDependencies +import org.thoughtcrime.securesms.keyvalue.SignalStore + +object PluginCache { + private var archiveFileSystem: ArchiveFileSystem? = null + var localBackups: ApiPlugin.LocalBackups? = null + + fun getArchiveFileSystem(): ArchiveFileSystem? { + if (archiveFileSystem == null) { + val backupDirectoryUri = SignalStore.backup.newLocalBackupsDirectory?.let { Uri.parse(it) } + if (backupDirectoryUri == null || backupDirectoryUri.path == null) { + return null + } + + archiveFileSystem = ArchiveFileSystem.fromUri(AppDependencies.application, backupDirectoryUri) + } + return archiveFileSystem + } + + fun clearBackupCache() { + archiveFileSystem = null + localBackups = null + } +} diff --git a/app/src/spinner/java/org/thoughtcrime/securesms/SpinnerApplicationContext.kt b/app/src/spinner/java/org/thoughtcrime/securesms/SpinnerApplicationContext.kt index 77b0566e3f..520e66304b 100644 --- a/app/src/spinner/java/org/thoughtcrime/securesms/SpinnerApplicationContext.kt +++ b/app/src/spinner/java/org/thoughtcrime/securesms/SpinnerApplicationContext.kt @@ -86,7 +86,9 @@ class SpinnerApplicationContext : ApplicationContext() { ), linkedMapOf( StorageServicePlugin.PATH to StorageServicePlugin(), - AttachmentPlugin.PATH to AttachmentPlugin() + AttachmentPlugin.PATH to AttachmentPlugin(), + BackupPlugin.PATH to BackupPlugin(), + ApiPlugin.PATH to ApiPlugin() ) ) diff --git a/app/src/test/java/org/thoughtcrime/securesms/sms/UploadDependencyGraphTest.kt b/app/src/test/java/org/thoughtcrime/securesms/sms/UploadDependencyGraphTest.kt index 0958513a1f..7286ef2f72 100644 --- a/app/src/test/java/org/thoughtcrime/securesms/sms/UploadDependencyGraphTest.kt +++ b/app/src/test/java/org/thoughtcrime/securesms/sms/UploadDependencyGraphTest.kt @@ -256,7 +256,8 @@ class UploadDependencyGraphTest { thumbnailRestoreState = AttachmentTable.ThumbnailRestoreState.NONE, archiveTransferState = AttachmentTable.ArchiveTransferState.NONE, uuid = null, - quoteTargetContentType = null + quoteTargetContentType = null, + metadata = null ) } diff --git a/app/src/testShared/org/thoughtcrime/securesms/database/FakeMessageRecords.kt b/app/src/testShared/org/thoughtcrime/securesms/database/FakeMessageRecords.kt index 61b1790e8f..5b066493f5 100644 --- a/app/src/testShared/org/thoughtcrime/securesms/database/FakeMessageRecords.kt +++ b/app/src/testShared/org/thoughtcrime/securesms/database/FakeMessageRecords.kt @@ -101,7 +101,8 @@ object FakeMessageRecords { thumbnailRestoreState = thumbnailRestoreState, archiveTransferState = archiveTransferState, uuid = null, - quoteTargetContentType = null + quoteTargetContentType = null, + metadata = null ) } diff --git a/core-models/src/main/java/org/signal/core/models/backup/MediaName.kt b/core-models/src/main/java/org/signal/core/models/backup/MediaName.kt index 7d7da21fb2..ae86a51bae 100644 --- a/core-models/src/main/java/org/signal/core/models/backup/MediaName.kt +++ b/core-models/src/main/java/org/signal/core/models/backup/MediaName.kt @@ -5,6 +5,7 @@ package org.signal.core.models.backup +import org.signal.core.util.CryptoUtil import org.signal.core.util.Hex /** @@ -17,6 +18,7 @@ value class MediaName(val name: String) { fun fromPlaintextHashAndRemoteKey(plaintextHash: ByteArray, remoteKey: ByteArray) = MediaName(Hex.toStringCondensed(plaintextHash + remoteKey)) fun fromPlaintextHashAndRemoteKeyForThumbnail(plaintextHash: ByteArray, remoteKey: ByteArray) = MediaName(Hex.toStringCondensed(plaintextHash + remoteKey) + "_thumbnail") fun forThumbnailFromMediaName(mediaName: String) = MediaName("${mediaName}_thumbnail") + fun forLocalBackupFilename(plaintextHash: ByteArray, localKey: ByteArray) = MediaName(Hex.toStringCondensed(CryptoUtil.sha256(plaintextHash + localKey))) /** * For java, since it struggles with value classes. diff --git a/core-models/src/main/java/org/signal/core/models/backup/MessageBackupKey.kt b/core-models/src/main/java/org/signal/core/models/backup/MessageBackupKey.kt index 7f70a2f302..d4c1c0449a 100644 --- a/core-models/src/main/java/org/signal/core/models/backup/MessageBackupKey.kt +++ b/core-models/src/main/java/org/signal/core/models/backup/MessageBackupKey.kt @@ -54,6 +54,13 @@ class MessageBackupKey(override val value: ByteArray) : BackupKey { ) } + /** + * The AES key used to encrypt the backup id for local file backup metadata header. + */ + fun deriveLocalBackupMetadataKey(): ByteArray { + return LibSignalBackupKey(value).deriveLocalBackupMetadataKey() + } + class BackupKeyMaterial( val id: BackupId, val macKey: ByteArray, diff --git a/spinner/lib/src/main/java/org/signal/spinner/PluginResult.kt b/spinner/lib/src/main/java/org/signal/spinner/PluginResult.kt index 8aeb93bce4..8e8f486c9b 100644 --- a/spinner/lib/src/main/java/org/signal/spinner/PluginResult.kt +++ b/spinner/lib/src/main/java/org/signal/spinner/PluginResult.kt @@ -1,5 +1,6 @@ package org.signal.spinner +import fi.iki.elonen.NanoHTTPD.Response import java.io.InputStream sealed class PluginResult(val type: String) { @@ -22,4 +23,19 @@ sealed class PluginResult(val type: String) { val data: InputStream, val mimeType: String ) : PluginResult("file") + + data class JsonResult( + val json: String + ) : PluginResult("json") + + data class ErrorResult( + val status: Response.Status = Response.Status.INTERNAL_ERROR, + val message: String + ) : PluginResult("error") { + companion object { + fun notFound(message: String): PluginResult.ErrorResult { + return ErrorResult(status = Response.Status.NOT_FOUND, message = message) + } + } + } } diff --git a/spinner/lib/src/main/java/org/signal/spinner/SpinnerServer.kt b/spinner/lib/src/main/java/org/signal/spinner/SpinnerServer.kt index f4366d6946..8883e01b69 100644 --- a/spinner/lib/src/main/java/org/signal/spinner/SpinnerServer.kt +++ b/spinner/lib/src/main/java/org/signal/spinner/SpinnerServer.kt @@ -54,6 +54,8 @@ internal class SpinnerServer( private val dateFormat = SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS zzz", Locale.US) override fun serve(session: IHTTPSession): Response { + Log.v(TAG, "Request: ${session.method} ${session.uri} ${session.parameters}") + if (session.method == Method.POST) { // Needed to populate session.parameters session.parseBody(mutableMapOf()) @@ -280,12 +282,16 @@ internal class SpinnerServer( } private fun getPlugin(dbName: String, plugin: Plugin, parameters: Map>): Response { - val pluginResult = plugin.get(parameters) - - when (pluginResult) { + when (val pluginResult = plugin.get(parameters)) { + is PluginResult.JsonResult -> { + return newFixedLengthResponse(Response.Status.OK, "application/json", pluginResult.json) + } is PluginResult.RawFileResult -> { return newFixedLengthResponse(Response.Status.OK, pluginResult.mimeType, pluginResult.data, pluginResult.length) } + is PluginResult.ErrorResult -> { + return newFixedLengthResponse(pluginResult.status, "text/plain", pluginResult.message) + } else -> { return renderTemplate( "plugin", @@ -296,7 +302,7 @@ internal class SpinnerServer( databases = databases.keys.toList(), plugins = plugins.values.toList(), activePlugin = plugin, - pluginResult = plugin.get(parameters) + pluginResult = pluginResult ) ) }