From 8eb0b2f9601318883ed432dcbaa8ae1d782fe77d Mon Sep 17 00:00:00 2001 From: Cody Henthorne Date: Tue, 13 Aug 2024 17:01:31 -0400 Subject: [PATCH] Add initial local archive export support. --- .gitignore | 2 +- .../provider/DocumentFileHelper.java | 38 -- .../securesms/backup/v2/BackupRepository.kt | 182 +++++++--- .../securesms/backup/v2/BackupV2Event.kt | 14 + .../v2/database/ChatItemExportIterator.kt | 2 +- .../backup/v2/local/ArchiveFileSystem.kt | 334 ++++++++++++++++++ .../backup/v2/local/ArchivedFilesReader.kt | 50 +++ .../backup/v2/local/ArchivedFilesWriter.kt | 28 ++ .../backup/v2/local/LocalArchiver.kt | 177 ++++++++++ .../backup/v2/stream/EncryptedBackupReader.kt | 2 +- .../InternalBackupPlaygroundFragment.kt | 17 + .../InternalBackupPlaygroundViewModel.kt | 28 ++ .../securesms/database/AttachmentTable.kt | 79 ++++- .../securesms/database/SignalDatabase.kt | 6 +- .../securesms/jobs/JobManagerFactories.java | 1 + .../securesms/jobs/LocalArchiveJob.kt | 190 ++++++++++ .../securesms/jobs/LocalBackupJob.java | 10 + .../securesms/jobs/LocalBackupJobApi29.java | 60 ++-- .../securesms/jobs/RequestGroupV2InfoJob.java | 2 +- .../securesms/jobs/RestoreAttachmentJob.kt | 2 +- .../jobs/RestoreAttachmentThumbnailJob.kt | 2 +- .../securesms/util/StorageUtil.java | 30 ++ app/src/main/protowire/LocalArchive.proto | 15 + .../util/stream/NonClosingOutputStream.kt | 19 + core-util/build.gradle.kts | 1 + .../documentfile/provider/DocumentFileHack.kt | 10 + .../core/util/androidx/DocumentFileInfo.kt | 14 + .../core/util/androidx/DocumentFileUtil.kt | 222 ++++++++++++ .../core/util/concurrent/LimitedWorker.kt | 36 ++ dependencies.gradle.kts | 1 + .../crypto/AttachmentCipherInputStream.java | 33 +- 31 files changed, 1474 insertions(+), 133 deletions(-) delete mode 100644 app/src/main/java/androidx/documentfile/provider/DocumentFileHelper.java create mode 100644 app/src/main/java/org/thoughtcrime/securesms/backup/v2/local/ArchiveFileSystem.kt create mode 100644 app/src/main/java/org/thoughtcrime/securesms/backup/v2/local/ArchivedFilesReader.kt create mode 100644 app/src/main/java/org/thoughtcrime/securesms/backup/v2/local/ArchivedFilesWriter.kt create mode 100644 app/src/main/java/org/thoughtcrime/securesms/backup/v2/local/LocalArchiver.kt create mode 100644 app/src/main/java/org/thoughtcrime/securesms/jobs/LocalArchiveJob.kt create mode 100644 app/src/main/protowire/LocalArchive.proto create mode 100644 core-util-jvm/src/main/java/org/signal/core/util/stream/NonClosingOutputStream.kt create mode 100644 core-util/src/main/java/androidx/documentfile/provider/DocumentFileHack.kt create mode 100644 core-util/src/main/java/org/signal/core/util/androidx/DocumentFileInfo.kt create mode 100644 core-util/src/main/java/org/signal/core/util/androidx/DocumentFileUtil.kt create mode 100644 core-util/src/main/java/org/signal/core/util/concurrent/LimitedWorker.kt diff --git a/.gitignore b/.gitignore index dd90f84809..7cb174b45c 100644 --- a/.gitignore +++ b/.gitignore @@ -29,4 +29,4 @@ jni/libspeex/.deps/ pkcs11.password dev.keystore maps.key -local/ +/local/ diff --git a/app/src/main/java/androidx/documentfile/provider/DocumentFileHelper.java b/app/src/main/java/androidx/documentfile/provider/DocumentFileHelper.java deleted file mode 100644 index 806882482a..0000000000 --- a/app/src/main/java/androidx/documentfile/provider/DocumentFileHelper.java +++ /dev/null @@ -1,38 +0,0 @@ -package androidx.documentfile.provider; - -import android.content.Context; -import android.net.Uri; -import android.provider.DocumentsContract; - -import org.signal.core.util.logging.Log; - -/** - * Located in androidx package as {@link TreeDocumentFile} is package protected. - */ -public class DocumentFileHelper { - - private static final String TAG = Log.tag(DocumentFileHelper.class); - - /** - * System implementation swallows the exception and we are having problems with the rename. This inlines the - * same call and logs the exception. Note this implementation does not update the passed in document file like - * the system implementation. Do not use the provided document file after calling this method. - * - * @return true if rename successful - */ - public static boolean renameTo(Context context, DocumentFile documentFile, String displayName) { - if (documentFile instanceof TreeDocumentFile) { - Log.d(TAG, "Renaming document directly"); - try { - final Uri result = DocumentsContract.renameDocument(context.getContentResolver(), documentFile.getUri(), displayName); - return result != null; - } catch (Exception e) { - Log.w(TAG, "Unable to rename document file", e); - return false; - } - } else { - Log.d(TAG, "Letting OS rename document: " + documentFile.getClass().getSimpleName()); - return documentFile.renameTo(displayName); - } - } -} diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/BackupRepository.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/BackupRepository.kt index 61f3565bc7..44814cdb1e 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/BackupRepository.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/BackupRepository.kt @@ -12,9 +12,12 @@ import kotlinx.coroutines.withContext import org.greenrobot.eventbus.EventBus import org.signal.core.util.Base64 import org.signal.core.util.EventTimer +import org.signal.core.util.concurrent.LimitedWorker +import org.signal.core.util.concurrent.SignalExecutors import org.signal.core.util.fullWalCheckpoint import org.signal.core.util.logging.Log import org.signal.core.util.money.FiatMoney +import org.signal.core.util.stream.NonClosingOutputStream import org.signal.core.util.withinTransaction import org.signal.libsignal.messagebackup.MessageBackup import org.signal.libsignal.messagebackup.MessageBackup.ValidationResult @@ -37,6 +40,7 @@ import org.thoughtcrime.securesms.backup.v2.processor.RecipientBackupProcessor import org.thoughtcrime.securesms.backup.v2.processor.StickerBackupProcessor import org.thoughtcrime.securesms.backup.v2.proto.BackupInfo import org.thoughtcrime.securesms.backup.v2.stream.BackupExportWriter +import org.thoughtcrime.securesms.backup.v2.stream.BackupImportReader import org.thoughtcrime.securesms.backup.v2.stream.EncryptedBackupReader import org.thoughtcrime.securesms.backup.v2.stream.EncryptedBackupWriter import org.thoughtcrime.securesms.backup.v2.stream.PlainTextBackupReader @@ -46,6 +50,7 @@ import org.thoughtcrime.securesms.backup.v2.ui.subscription.MessageBackupsTypeFe import org.thoughtcrime.securesms.components.settings.app.subscription.RecurringInAppPaymentRepository import org.thoughtcrime.securesms.crypto.AttachmentSecretProvider import org.thoughtcrime.securesms.crypto.DatabaseSecretProvider +import org.thoughtcrime.securesms.database.AttachmentTable import org.thoughtcrime.securesms.database.KeyValueDatabase import org.thoughtcrime.securesms.database.SignalDatabase import org.thoughtcrime.securesms.database.model.InAppPaymentSubscriberRecord @@ -82,6 +87,7 @@ import java.math.BigDecimal import java.time.ZonedDateTime import java.util.Currency import java.util.Locale +import java.util.concurrent.atomic.AtomicLong import kotlin.time.Duration.Companion.milliseconds object BackupRepository { @@ -90,6 +96,8 @@ object BackupRepository { private const val VERSION = 1L private const val MAIN_DB_SNAPSHOT_NAME = "signal-snapshot.db" private const val KEYVALUE_DB_SNAPSHOT_NAME = "key-value-snapshot.db" + private const val LOCAL_MAIN_DB_SNAPSHOT_NAME = "local-signal-snapshot.db" + private const val LOCAL_KEYVALUE_DB_SNAPSHOT_NAME = "local-key-value-snapshot.db" private val resetInitializedStateErrorAction: StatusCodeErrorAction = { error -> when (error.code) { @@ -126,7 +134,7 @@ object BackupRepository { SignalStore.backup.backupTier = null } - private fun createSignalDatabaseSnapshot(): SignalDatabase { + private fun createSignalDatabaseSnapshot(name: String): SignalDatabase { // Need to do a WAL checkpoint to ensure that the database file we're copying has all pending writes if (!SignalDatabase.rawDatabase.fullWalCheckpoint()) { Log.w(TAG, "Failed to checkpoint WAL for main database! Not guaranteed to be using the most recent data.") @@ -137,7 +145,7 @@ object BackupRepository { val context = AppDependencies.application val existingDbFile = context.getDatabasePath(SignalDatabase.DATABASE_NAME) - val targetFile = File(existingDbFile.parentFile, MAIN_DB_SNAPSHOT_NAME) + val targetFile = File(existingDbFile.parentFile, name) try { existingDbFile.copyTo(targetFile, overwrite = true) @@ -150,12 +158,12 @@ object BackupRepository { context = context, databaseSecret = DatabaseSecretProvider.getOrCreateDatabaseSecret(context), attachmentSecret = AttachmentSecretProvider.getInstance(context).getOrCreateAttachmentSecret(), - name = MAIN_DB_SNAPSHOT_NAME + name = name ) } } - private fun createSignalStoreSnapshot(): SignalStore { + private fun createSignalStoreSnapshot(name: String): SignalStore { val context = AppDependencies.application // Need to do a WAL checkpoint to ensure that the database file we're copying has all pending writes @@ -166,7 +174,7 @@ object BackupRepository { // We make a copy of the database within a transaction to ensure that no writes occur while we're copying the file return KeyValueDatabase.getInstance(context).writableDatabase.withinTransaction { val existingDbFile = context.getDatabasePath(KeyValueDatabase.DATABASE_NAME) - val targetFile = File(existingDbFile.parentFile, KEYVALUE_DB_SNAPSHOT_NAME) + val targetFile = File(existingDbFile.parentFile, name) try { existingDbFile.copyTo(targetFile, overwrite = true) @@ -175,41 +183,98 @@ object BackupRepository { throw IllegalStateException("Failed to copy database file!", e) } - val db = KeyValueDatabase.createWithName(context, KEYVALUE_DB_SNAPSHOT_NAME) + val db = KeyValueDatabase.createWithName(context, name) SignalStore(KeyValueStore(db)) } } - private fun deleteDatabaseSnapshot() { - val targetFile = AppDependencies.application.getDatabasePath(MAIN_DB_SNAPSHOT_NAME) + private fun deleteDatabaseSnapshot(name: String) { + val targetFile = AppDependencies.application.getDatabasePath(name) if (!targetFile.delete()) { Log.w(TAG, "Failed to delete main database snapshot!") } } - private fun deleteSignalStoreSnapshot() { - val targetFile = AppDependencies.application.getDatabasePath(KEYVALUE_DB_SNAPSHOT_NAME) + private fun deleteSignalStoreSnapshot(name: String) { + val targetFile = AppDependencies.application.getDatabasePath(name) if (!targetFile.delete()) { Log.w(TAG, "Failed to delete key value database snapshot!") } } + fun localExport( + main: OutputStream, + localBackupProgressEmitter: ExportProgressListener, + archiveAttachment: (AttachmentTable.LocalArchivableAttachment, () -> InputStream?) -> Unit + ) { + val writer = EncryptedBackupWriter( + key = SignalStore.svr.getOrCreateMasterKey().deriveBackupKey(), + aci = SignalStore.account.aci!!, + outputStream = NonClosingOutputStream(main), + append = { main.write(it) } + ) + + export(currentTime = System.currentTimeMillis(), isLocal = true, writer = writer, progressEmitter = localBackupProgressEmitter) { dbSnapshot -> + val localArchivableAttachments = dbSnapshot + .attachmentTable + .getLocalArchivableAttachments() + .associateBy { MediaName.fromDigest(it.remoteDigest) } + + localBackupProgressEmitter.onAttachment(0, localArchivableAttachments.size.toLong()) + + val progress = AtomicLong(0) + + LimitedWorker.execute(SignalExecutors.BOUNDED_IO, 4, localArchivableAttachments.values) { attachment -> + try { + archiveAttachment(attachment) { dbSnapshot.attachmentTable.getAttachmentStream(attachment) } + } catch (e: IOException) { + Log.w(TAG, "Unable to open attachment, skipping", e) + } + + val currentProgress = progress.incrementAndGet() + localBackupProgressEmitter.onAttachment(currentProgress, localArchivableAttachments.size.toLong()) + } + } + } + fun export(outputStream: OutputStream, append: (ByteArray) -> Unit, plaintext: Boolean = false, currentTime: Long = System.currentTimeMillis()) { + val writer: BackupExportWriter = if (plaintext) { + PlainTextBackupWriter(outputStream) + } else { + EncryptedBackupWriter( + key = SignalStore.svr.getOrCreateMasterKey().deriveBackupKey(), + aci = SignalStore.account.aci!!, + outputStream = outputStream, + append = append + ) + } + + export(currentTime = currentTime, isLocal = false, writer = writer) + } + + /** + * Exports to a blob in memory. Should only be used for testing. + */ + fun debugExport(plaintext: Boolean = false, currentTime: Long = System.currentTimeMillis()): ByteArray { + val outputStream = ByteArrayOutputStream() + export(outputStream = outputStream, append = { mac -> outputStream.write(mac) }, plaintext = plaintext, currentTime = currentTime) + return outputStream.toByteArray() + } + + private fun export( + currentTime: Long, + isLocal: Boolean, + writer: BackupExportWriter, + progressEmitter: ExportProgressListener? = null, + exportExtras: ((SignalDatabase) -> Unit)? = null + ) { val eventTimer = EventTimer() - val dbSnapshot: SignalDatabase = createSignalDatabaseSnapshot() - val signalStoreSnapshot: SignalStore = createSignalStoreSnapshot() + val mainDbName = if (isLocal) LOCAL_MAIN_DB_SNAPSHOT_NAME else MAIN_DB_SNAPSHOT_NAME + val keyValueDbName = if (isLocal) LOCAL_KEYVALUE_DB_SNAPSHOT_NAME else KEYVALUE_DB_SNAPSHOT_NAME try { - val writer: BackupExportWriter = if (plaintext) { - PlainTextBackupWriter(outputStream) - } else { - EncryptedBackupWriter( - key = SignalStore.svr.getOrCreateMasterKey().deriveBackupKey(), - aci = SignalStore.account.aci!!, - outputStream = outputStream, - append = append - ) - } + val dbSnapshot: SignalDatabase = createSignalDatabaseSnapshot(mainDbName) + val signalStoreSnapshot: SignalStore = createSignalStoreSnapshot(keyValueDbName) val exportState = ExportState(backupTime = currentTime, mediaBackupEnabled = SignalStore.backup.backsUpMedia) @@ -223,31 +288,37 @@ object BackupRepository { // We're using a snapshot, so the transaction is more for perf than correctness dbSnapshot.rawWritableDatabase.withinTransaction { + progressEmitter?.onAccount() AccountDataProcessor.export(dbSnapshot, signalStoreSnapshot) { writer.write(it) eventTimer.emit("account") } + progressEmitter?.onRecipient() RecipientBackupProcessor.export(dbSnapshot, signalStoreSnapshot, exportState) { writer.write(it) eventTimer.emit("recipient") } + progressEmitter?.onThread() ChatBackupProcessor.export(dbSnapshot, exportState) { frame -> writer.write(frame) eventTimer.emit("thread") } + progressEmitter?.onCall() AdHocCallBackupProcessor.export(dbSnapshot) { frame -> writer.write(frame) eventTimer.emit("call") } + progressEmitter?.onSticker() StickerBackupProcessor.export(dbSnapshot) { frame -> writer.write(frame) eventTimer.emit("sticker-pack") } + progressEmitter?.onMessage() ChatItemBackupProcessor.export(dbSnapshot, exportState) { frame -> writer.write(frame) eventTimer.emit("message") @@ -255,28 +326,31 @@ object BackupRepository { } } + exportExtras?.invoke(dbSnapshot) + Log.d(TAG, "export() ${eventTimer.stop().summary}") } finally { - deleteDatabaseSnapshot() - deleteSignalStoreSnapshot() + deleteDatabaseSnapshot(mainDbName) + deleteSignalStoreSnapshot(keyValueDbName) } } - /** - * Exports to a blob in memory. Should only be used for testing. - */ - fun debugExport(plaintext: Boolean = false, currentTime: Long = System.currentTimeMillis()): ByteArray { - val outputStream = ByteArrayOutputStream() - export(outputStream = outputStream, append = { mac -> outputStream.write(mac) }, plaintext = plaintext, currentTime = currentTime) - return outputStream.toByteArray() + fun localImport(mainStreamFactory: () -> InputStream, mainStreamLength: Long, selfData: SelfData): ImportResult { + val backupKey = SignalStore.svr.getOrCreateMasterKey().deriveBackupKey() + + val frameReader = EncryptedBackupReader( + key = backupKey, + aci = selfData.aci, + length = mainStreamLength, + dataStream = mainStreamFactory + ) + + return frameReader.use { reader -> + import(backupKey, reader, selfData) + } } - /** - * @return The time the backup was created, or null if the backup could not be read. - */ fun import(length: Long, inputStreamFactory: () -> InputStream, selfData: SelfData, plaintext: Boolean = false): ImportResult { - val eventTimer = EventTimer() - val backupKey = SignalStore.svr.getOrCreateMasterKey().deriveBackupKey() val frameReader = if (plaintext) { @@ -290,6 +364,19 @@ object BackupRepository { ) } + return frameReader.use { reader -> + import(backupKey, reader, selfData) + } + } + + private fun import( + backupKey: BackupKey, + frameReader: BackupImportReader, + selfData: SelfData, + importExtras: ((EventTimer) -> Unit)? = null + ): ImportResult { + val eventTimer = EventTimer() + val header = frameReader.getHeader() if (header == null) { Log.e(TAG, "Backup is missing header!") @@ -364,16 +451,19 @@ object BackupRepository { eventTimer.emit("chatItem") } + importExtras?.invoke(eventTimer) + importState.chatIdToLocalThreadId.values.forEach { SignalDatabase.threads.update(it, unarchive = false, allowDeletion = false) } } - val groups = SignalDatabase.groups.getGroups() - while (groups.hasNext()) { - val group = groups.next() - if (group.id.isV2) { - AppDependencies.jobManager.add(RequestGroupV2InfoJob(group.id as GroupId.V2)) + SignalDatabase.groups.getGroups().use { groups -> + while (groups.hasNext()) { + val group = groups.next() + if (group.id.isV2) { + AppDependencies.jobManager.add(RequestGroupV2InfoJob(group.id as GroupId.V2)) + } } } @@ -949,6 +1039,16 @@ object BackupRepository { iv = Base64.encodeWithPadding(mediaSecrets.iv) ) } + + interface ExportProgressListener { + fun onAccount() + fun onRecipient() + fun onThread() + fun onCall() + fun onSticker() + fun onMessage() + fun onAttachment(currentProgress: Long, totalCount: Long) + } } data class ArchivedMediaObject(val mediaId: String, val cdn: Int) diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/BackupV2Event.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/BackupV2Event.kt index 8899d674ab..1ad508667c 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/BackupV2Event.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/BackupV2Event.kt @@ -12,3 +12,17 @@ class BackupV2Event(val type: Type, val count: Long, val estimatedTotalCount: Lo FINISHED } } + +class LocalBackupV2Event(val type: Type, val count: Long = 0, val estimatedTotalCount: Long = 0) { + enum class Type { + PROGRESS_ACCOUNT, + PROGRESS_RECIPIENT, + PROGRESS_THREAD, + PROGRESS_CALL, + PROGRESS_STICKER, + PROGRESS_MESSAGE, + PROGRESS_ATTACHMENT, + PROGRESS_VERIFYING, + FINISHED + } +} diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/ChatItemExportIterator.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/ChatItemExportIterator.kt index 3f03484b04..da1b53ad97 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/ChatItemExportIterator.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/ChatItemExportIterator.kt @@ -422,7 +422,7 @@ class ChatItemExportIterator(private val cursor: Cursor, private val batchSize: ChatUpdateMessage( groupCall = GroupCall( state = GroupCall.State.GENERIC, - startedCallRecipientId = recipients.getByAci(ACI.from(UuidUtil.parseOrThrow(groupCallUpdateDetails.startedCallUuid))).getOrNull()?.toLong(), + startedCallRecipientId = UuidUtil.parseOrNull(groupCallUpdateDetails.startedCallUuid)?.let { recipients.getByAci(ACI.from(it)).getOrNull()?.toLong() }, startedCallTimestamp = groupCallUpdateDetails.startedCallTimestamp, endedCallTimestamp = groupCallUpdateDetails.endedCallTimestamp ) diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/local/ArchiveFileSystem.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/local/ArchiveFileSystem.kt new file mode 100644 index 0000000000..027531c737 --- /dev/null +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/local/ArchiveFileSystem.kt @@ -0,0 +1,334 @@ +/* + * Copyright 2024 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ + +package org.thoughtcrime.securesms.backup.v2.local + +import android.content.Context +import android.net.Uri +import androidx.documentfile.provider.DocumentFile +import org.signal.core.util.androidx.DocumentFileInfo +import org.signal.core.util.androidx.DocumentFileUtil.delete +import org.signal.core.util.androidx.DocumentFileUtil.hasFile +import org.signal.core.util.androidx.DocumentFileUtil.inputStream +import org.signal.core.util.androidx.DocumentFileUtil.listFiles +import org.signal.core.util.androidx.DocumentFileUtil.mkdirp +import org.signal.core.util.androidx.DocumentFileUtil.newFile +import org.signal.core.util.androidx.DocumentFileUtil.outputStream +import org.signal.core.util.androidx.DocumentFileUtil.renameTo +import org.signal.core.util.logging.Log +import org.whispersystems.signalservice.api.backup.MediaName +import java.io.File +import java.io.IOException +import java.io.InputStream +import java.io.OutputStream +import java.text.SimpleDateFormat +import java.util.Calendar +import java.util.Date +import java.util.Locale + +/** + * Provide a domain-specific interface to the root file system backing a local directory based archive. + */ +@Suppress("JoinDeclarationAndAssignment") +class ArchiveFileSystem private constructor(private val context: Context, root: DocumentFile) { + + companion object { + val TAG = Log.tag(ArchiveFileSystem::class.java) + + const val BACKUP_DIRECTORY_PREFIX: String = "signal-backup" + const val TEMP_BACKUP_DIRECTORY_SUFFIX: String = "tmp" + + /** + * Attempt to create an [ArchiveFileSystem] from a tree [Uri]. + * + * Should likely only be called on API29+ + */ + fun fromUri(context: Context, uri: Uri): ArchiveFileSystem? { + val root = DocumentFile.fromTreeUri(context, uri) + + if (root == null || !root.canWrite()) { + return null + } + + return ArchiveFileSystem(context, root) + } + + /** + * Attempt to create an [ArchiveFileSystem] from a regular [File]. + * + * Should likely only be called on < API29. + */ + fun fromFile(context: Context, backupDirectory: File): ArchiveFileSystem { + return ArchiveFileSystem(context, DocumentFile.fromFile(backupDirectory)) + } + } + + private val signalBackups: DocumentFile + + /** File access to shared super-set of archive related files (e.g., media + attachments) */ + val filesFileSystem: FilesFileSystem + + init { + signalBackups = root.mkdirp("SignalBackups") ?: throw IOException("Unable to create main backups directory") + val filesDirectory = signalBackups.mkdirp("files") ?: throw IOException("Unable to create files directory") + filesFileSystem = FilesFileSystem(context, filesDirectory) + } + + /** + * Delete all folders that match the temp/in-progress backup directory naming convention. Used to clean-up + * previous catastrophic backup failures. + */ + fun deleteOldTemporaryBackups() { + for (file in signalBackups.listFiles()) { + if (file.isDirectory) { + val name = file.name + if (name != null && name.startsWith(BACKUP_DIRECTORY_PREFIX) && name.endsWith(TEMP_BACKUP_DIRECTORY_SUFFIX)) { + if (file.delete()) { + Log.w(TAG, "Deleted old temporary backup folder") + } else { + Log.w(TAG, "Could not delete old temporary backup folder") + } + } + } + } + } + + /** + * Retain up to [limit] most recent backups and delete all others. + */ + fun deleteOldBackups(limit: Int = 2) { + Log.i(TAG, "Deleting old backups") + + listSnapshots() + .drop(limit) + .forEach { it.file.delete() } + } + + /** + * Attempt to create a [SnapshotFileSystem] to represent a single backup snapshot. + */ + fun createSnapshot(): SnapshotFileSystem? { + val timestamp = SimpleDateFormat("yyyy-MM-dd-HH-mm-ss", Locale.US).format(Date()) + val snapshotDirectoryName = "${BACKUP_DIRECTORY_PREFIX}-$timestamp" + + if (signalBackups.hasFile(snapshotDirectoryName)) { + Log.w(TAG, "Backup directory already exists!") + return null + } + + val workingSnapshotDirectoryName = "$snapshotDirectoryName-$TEMP_BACKUP_DIRECTORY_SUFFIX" + val workingSnapshotDirectory = signalBackups.createDirectory(workingSnapshotDirectoryName) ?: return null + + return SnapshotFileSystem(context, snapshotDirectoryName, workingSnapshotDirectoryName, workingSnapshotDirectory) + } + + /** + * Delete an in-progress snapshot folder after a handled backup failure. + * + * @return true if the snapshot was deleted + */ + fun cleanupSnapshot(snapshotFileSystem: SnapshotFileSystem): Boolean { + check(snapshotFileSystem.workingSnapshotDirectoryName.isNotEmpty()) { "Cannot call cleanup on unnamed snapshot" } + return signalBackups.findFile(snapshotFileSystem.workingSnapshotDirectoryName)?.delete() ?: false + } + + /** + * List all snapshots found in this directory sorted by creation timestamp, newest first. + */ + fun listSnapshots(): List { + return signalBackups + .listFiles() + .asSequence() + .filter { it.isDirectory } + .mapNotNull { f -> f.name?.let { it to f } } + .filter { (name, _) -> name.startsWith(BACKUP_DIRECTORY_PREFIX) } + .map { (name, file) -> + val timestamp = name.replace(BACKUP_DIRECTORY_PREFIX, "").toMilliseconds() + SnapshotInfo(timestamp, name, file) + } + .sortedByDescending { it.timestamp } + .toList() + } + + /** + * Clean up unused files in the shared files directory leveraged across all current snapshots. A file + * is unused if it is not referenced directly by any current snapshots. + */ + fun deleteUnusedFiles() { + Log.i(TAG, "Deleting unused files") + + val allFiles: MutableMap = filesFileSystem.allFiles().toMutableMap() + val snapshots: List = listSnapshots() + + snapshots + .mapNotNull { SnapshotFileSystem.filesInputStream(context, it.file) } + .forEach { input -> + ArchivedFilesReader(input).use { reader -> + reader.forEach { f -> f.mediaName?.let { allFiles.remove(it) } } + } + } + + var deleted = 0 + allFiles + .values + .forEach { + if (it.documentFile.delete()) { + deleted++ + } + } + + Log.d(TAG, "Cleanup removed $deleted/${allFiles.size} files") + } + + /** Useful metadata for a given archive snapshot */ + data class SnapshotInfo(val timestamp: Long, val name: String, val file: DocumentFile) +} + +/** + * Domain specific file system for dealing with individual snapshot data. + */ +class SnapshotFileSystem(private val context: Context, private val snapshotDirectoryName: String, val workingSnapshotDirectoryName: String, private val root: DocumentFile) { + companion object { + const val MAIN_NAME = "main" + const val METADATA_NAME = "metadata" + const val FILES_NAME = "files" + + /** + * Get the files metadata file directly for a snapshot. + */ + fun filesInputStream(context: Context, snapshotDirectory: DocumentFile): InputStream? { + return snapshotDirectory.findFile(FILES_NAME)?.inputStream(context) + } + } + + /** + * Creates an unnamed snapshot file system for use in importing. + */ + constructor(context: Context, root: DocumentFile) : this(context, "", "", root) + + fun mainOutputStream(): OutputStream? { + return root.newFile(MAIN_NAME)?.outputStream(context) + } + + fun mainInputStream(): InputStream? { + return root.findFile(MAIN_NAME)?.inputStream(context) + } + + fun mainLength(): Long? { + return root.findFile(MAIN_NAME)?.length() + } + + fun metadataOutputStream(): OutputStream? { + return root.newFile(METADATA_NAME)?.outputStream(context) + } + + fun metadataInputStream(): InputStream? { + return root.findFile(METADATA_NAME)?.inputStream(context) + } + + fun filesOutputStream(): OutputStream? { + return root.newFile(FILES_NAME)?.outputStream(context) + } + + /** + * Rename the snapshot from the working temporary name to final name. + */ + fun finalize(): Boolean { + check(snapshotDirectoryName.isNotEmpty()) { "Cannot call finalize on unnamed snapshot" } + return root.renameTo(context, snapshotDirectoryName) + } +} + +/** + * Domain specific file system access for accessing backup files (e.g., attachments, media, etc.). + */ +class FilesFileSystem(private val context: Context, private val root: DocumentFile) { + + private val subFolders: Map + + init { + val existingFolders = root.listFiles() + .mapNotNull { f -> f.name?.let { name -> name to f } } + .toMap() + + subFolders = (0..255) + .map { i -> i.toString(16).padStart(2, '0') } + .associateWith { name -> + existingFolders[name] ?: root.createDirectory(name)!! + } + } + + /** + * Enumerate all files in the directory. + */ + fun allFiles(): Map { + val allFiles = HashMap() + + for (subfolder in subFolders.values) { + val subFiles = subfolder.listFiles(context) + for (file in subFiles) { + allFiles[file.name] = file + } + } + + return allFiles + } + + /** + * Creates a new file for the given [mediaName] and returns the output stream for writing to it. The caller + * is responsible for determining if the file already exists (see [allFiles]) and deleting it (see [delete]). + * + * Calling this with a pre-existing file will likely create a second file with a modified name, but is generally + * undefined and should be avoided. + */ + fun fileOutputStream(mediaName: MediaName): OutputStream? { + val subFileDirectoryName = mediaName.name.substring(0..1) + val subFileDirectory = subFolders[subFileDirectoryName]!! + val file = subFileDirectory.createFile("application/octet-stream", mediaName.name) + return file?.outputStream(context) + } + + /** + * Given a [file], open and return an [InputStream]. + */ + fun fileInputStream(file: DocumentFileInfo): InputStream? { + return file.documentFile.inputStream(context) + } + + /** + * Delete a file for the given [mediaName] if it exists. + * + * @return true if deleted, false if not, null if not found + */ + fun delete(mediaName: MediaName): Boolean? { + val subFileDirectoryName = mediaName.name.substring(0..1) + val subFileDirectory = subFolders[subFileDirectoryName]!! + + return subFileDirectory.delete(context, mediaName.name) + } +} + +private fun String.toMilliseconds(): Long { + val parts: List = split("-").dropLastWhile { it.isEmpty() } + + if (parts.size == 7) { + try { + val calendar = Calendar.getInstance(Locale.US) + calendar[Calendar.YEAR] = parts[1].toInt() + calendar[Calendar.MONTH] = parts[2].toInt() - 1 + calendar[Calendar.DAY_OF_MONTH] = parts[3].toInt() + calendar[Calendar.HOUR_OF_DAY] = parts[4].toInt() + calendar[Calendar.MINUTE] = parts[5].toInt() + calendar[Calendar.SECOND] = parts[6].toInt() + calendar[Calendar.MILLISECOND] = 0 + + return calendar.timeInMillis + } catch (e: NumberFormatException) { + Log.w(ArchiveFileSystem.TAG, "Unable to parse timestamp from file name", e) + } + } + + return -1 +} diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/local/ArchivedFilesReader.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/local/ArchivedFilesReader.kt new file mode 100644 index 0000000000..ca3f1e8cff --- /dev/null +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/local/ArchivedFilesReader.kt @@ -0,0 +1,50 @@ +/* + * Copyright 2024 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ + +package org.thoughtcrime.securesms.backup.v2.local + +import org.signal.core.util.readNBytesOrThrow +import org.signal.core.util.readVarInt32 +import org.thoughtcrime.securesms.backup.v2.local.proto.FilesFrame +import java.io.EOFException +import java.io.InputStream + +/** + * Reads [FilesFrame] protos encoded with their length. + */ +class ArchivedFilesReader(private val inputStream: InputStream) : Iterator, AutoCloseable { + + private var next: FilesFrame? = null + + init { + next = read() + } + + override fun hasNext(): Boolean { + return next != null + } + + override fun next(): FilesFrame { + next?.let { out -> + next = read() + return out + } ?: throw NoSuchElementException() + } + + private fun read(): FilesFrame? { + try { + val length = inputStream.readVarInt32().also { if (it < 0) return null } + val frameBytes: ByteArray = inputStream.readNBytesOrThrow(length) + + return FilesFrame.ADAPTER.decode(frameBytes) + } catch (e: EOFException) { + return null + } + } + + override fun close() { + inputStream.close() + } +} diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/local/ArchivedFilesWriter.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/local/ArchivedFilesWriter.kt new file mode 100644 index 0000000000..a49c2de612 --- /dev/null +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/local/ArchivedFilesWriter.kt @@ -0,0 +1,28 @@ +/* + * Copyright 2024 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ + +package org.thoughtcrime.securesms.backup.v2.local + +import org.signal.core.util.writeVarInt32 +import org.thoughtcrime.securesms.backup.v2.local.proto.FilesFrame +import java.io.IOException +import java.io.OutputStream + +/** + * Write [FilesFrame] protos encoded with their length. + */ +class ArchivedFilesWriter(private val output: OutputStream) : AutoCloseable { + + @Throws(IOException::class) + fun write(frame: FilesFrame) { + val bytes = frame.encode() + output.writeVarInt32(bytes.size) + output.write(bytes) + } + + override fun close() { + output.close() + } +} diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/local/LocalArchiver.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/local/LocalArchiver.kt new file mode 100644 index 0000000000..f697e715b9 --- /dev/null +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/local/LocalArchiver.kt @@ -0,0 +1,177 @@ +/* + * Copyright 2024 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ + +package org.thoughtcrime.securesms.backup.v2.local + +import org.greenrobot.eventbus.EventBus +import org.signal.core.util.Base64 +import org.signal.core.util.Stopwatch +import org.signal.core.util.StreamUtil +import org.signal.core.util.logging.Log +import org.thoughtcrime.securesms.backup.v2.BackupRepository +import org.thoughtcrime.securesms.backup.v2.LocalBackupV2Event +import org.thoughtcrime.securesms.backup.v2.local.proto.FilesFrame +import org.thoughtcrime.securesms.backup.v2.local.proto.Metadata +import org.thoughtcrime.securesms.database.AttachmentTable +import org.whispersystems.signalservice.api.backup.MediaName +import org.whispersystems.signalservice.api.crypto.AttachmentCipherOutputStream +import org.whispersystems.signalservice.api.crypto.AttachmentCipherStreamUtil +import org.whispersystems.signalservice.internal.crypto.PaddingInputStream +import java.io.IOException +import java.io.InputStream +import java.io.OutputStream +import java.util.Collections +import kotlin.random.Random + +typealias ArchiveResult = org.signal.core.util.Result + +/** + * Handle importing and exporting folder-based archives using backupv2 format. + */ +object LocalArchiver { + + private val TAG = Log.tag(LocalArchiver::class) + private const val VERSION = 1 + + /** + * Export archive to the provided [snapshotFileSystem] and store new files in [filesFileSystem]. + */ + fun export(snapshotFileSystem: SnapshotFileSystem, filesFileSystem: FilesFileSystem, stopwatch: Stopwatch): ArchiveResult { + Log.i(TAG, "Starting export") + + var metadataStream: OutputStream? = null + var mainStream: OutputStream? = null + var filesStream: OutputStream? = null + + try { + metadataStream = snapshotFileSystem.metadataOutputStream() ?: return ArchiveResult.failure(FailureCause.METADATA_STREAM) + metadataStream.use { it.write(Metadata(VERSION).encode()) } + stopwatch.split("metadata") + + mainStream = snapshotFileSystem.mainOutputStream() ?: return ArchiveResult.failure(FailureCause.MAIN_STREAM) + + Log.i(TAG, "Listing all current files") + val allFiles = filesFileSystem.allFiles() + stopwatch.split("files-list") + + val mediaNames: MutableSet = Collections.synchronizedSet(HashSet()) + + Log.i(TAG, "Starting frame export") + BackupRepository.localExport(mainStream, LocalExportProgressListener()) { attachment, source -> + val mediaName = MediaName.fromDigest(attachment.remoteDigest) + + mediaNames.add(mediaName) + + if (allFiles[mediaName.name]?.size != attachment.cipherLength) { + if (allFiles.containsKey(mediaName.name)) { + filesFileSystem.delete(mediaName) + } + + source()?.use { sourceStream -> + val iv = Random.nextBytes(16) // todo [local-backup] but really do an iv from table + val combinedKey = Base64.decode(attachment.remoteKey) + + var destination: OutputStream? = filesFileSystem.fileOutputStream(mediaName) + + if (destination == null) { + Log.w(TAG, "Unable to create output file for attachment") + // todo [local-backup] should we abort here? + } else { + // todo [local-backup] but deal with attachment disappearing/deleted by normal app use + try { + PaddingInputStream(sourceStream, attachment.size).use { input -> + AttachmentCipherOutputStream(combinedKey, iv, destination).use { output -> + StreamUtil.copy(input, output) + } + } + } catch (e: IOException) { + Log.w(TAG, "Unable to save attachment", e) + // todo [local-backup] should we abort here? + } + } + } + } + } + stopwatch.split("frames-and-files") + + filesStream = snapshotFileSystem.filesOutputStream() ?: return ArchiveResult.failure(FailureCause.FILES_STREAM) + ArchivedFilesWriter(filesStream).use { writer -> + mediaNames.forEach { name -> writer.write(FilesFrame(mediaName = name.name)) } + } + stopwatch.split("files-metadata") + } finally { + metadataStream?.close() + mainStream?.close() + filesStream?.close() + } + + return ArchiveResult.success(Unit) + } + + /** + * Import archive data from a folder on the system. Does not restore attachments. + */ + fun import(snapshotFileSystem: SnapshotFileSystem, selfData: BackupRepository.SelfData): ArchiveResult { + var metadataStream: InputStream? = null + + try { + metadataStream = snapshotFileSystem.metadataInputStream() ?: return ArchiveResult.failure(FailureCause.METADATA_STREAM) + + val mainStreamLength = snapshotFileSystem.mainLength() ?: return ArchiveResult.failure(FailureCause.MAIN_STREAM) + + BackupRepository.localImport( + mainStreamFactory = { snapshotFileSystem.mainInputStream()!! }, + mainStreamLength = mainStreamLength, + selfData = selfData + ) + } finally { + metadataStream?.close() + } + + return ArchiveResult.success(Unit) + } + + private val AttachmentTable.LocalArchivableAttachment.cipherLength: Long + get() = AttachmentCipherStreamUtil.getCiphertextLength(PaddingInputStream.getPaddedSize(size)) + + enum class FailureCause { + METADATA_STREAM, MAIN_STREAM, FILES_STREAM + } + + private class LocalExportProgressListener : BackupRepository.ExportProgressListener { + private var lastAttachmentUpdate: Long = 0 + + override fun onAccount() { + EventBus.getDefault().post(LocalBackupV2Event(LocalBackupV2Event.Type.PROGRESS_ACCOUNT)) + } + + override fun onRecipient() { + EventBus.getDefault().post(LocalBackupV2Event(LocalBackupV2Event.Type.PROGRESS_RECIPIENT)) + } + + override fun onThread() { + EventBus.getDefault().post(LocalBackupV2Event(LocalBackupV2Event.Type.PROGRESS_THREAD)) + } + + override fun onCall() { + EventBus.getDefault().post(LocalBackupV2Event(LocalBackupV2Event.Type.PROGRESS_CALL)) + } + + override fun onSticker() { + EventBus.getDefault().post(LocalBackupV2Event(LocalBackupV2Event.Type.PROGRESS_STICKER)) + } + + override fun onMessage() { + EventBus.getDefault().post(LocalBackupV2Event(LocalBackupV2Event.Type.PROGRESS_MESSAGE)) + } + + override fun onAttachment(currentProgress: Long, totalCount: Long) { + if (lastAttachmentUpdate > System.currentTimeMillis() || lastAttachmentUpdate + 1000 < System.currentTimeMillis() || currentProgress >= totalCount) { + EventBus.getDefault().post(LocalBackupV2Event(LocalBackupV2Event.Type.PROGRESS_ATTACHMENT, currentProgress, totalCount)) + lastAttachmentUpdate = System.currentTimeMillis() + } + } + } +} diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/stream/EncryptedBackupReader.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/stream/EncryptedBackupReader.kt index 7e8701d898..1ce0702175 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/stream/EncryptedBackupReader.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/stream/EncryptedBackupReader.kt @@ -45,7 +45,7 @@ class EncryptedBackupReader( init { val keyMaterial = key.deriveBackupSecrets(aci) - validateMac(keyMaterial.macKey, length, dataStream()) + dataStream().use { validateMac(keyMaterial.macKey, length, it) } countingStream = CountingInputStream(dataStream()) val iv = countingStream.readNBytesOrThrow(16) diff --git a/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/backup/InternalBackupPlaygroundFragment.kt b/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/backup/InternalBackupPlaygroundFragment.kt index dfba28c44b..c3fce6bf51 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/backup/InternalBackupPlaygroundFragment.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/backup/InternalBackupPlaygroundFragment.kt @@ -81,6 +81,7 @@ class InternalBackupPlaygroundFragment : ComposeFragment() { private val viewModel: InternalBackupPlaygroundViewModel by viewModels() private lateinit var exportFileLauncher: ActivityResultLauncher private lateinit var importFileLauncher: ActivityResultLauncher + private lateinit var importDirectoryLauncher: ActivityResultLauncher private lateinit var validateFileLauncher: ActivityResultLauncher override fun onCreate(savedInstanceState: Bundle?) { @@ -107,6 +108,12 @@ class InternalBackupPlaygroundFragment : ComposeFragment() { } } + importDirectoryLauncher = registerForActivityResult(ActivityResultContracts.StartActivityForResult()) { result -> + if (result.resultCode == RESULT_OK) { + viewModel.import(result.data!!.data!!) + } + } + validateFileLauncher = registerForActivityResult(ActivityResultContracts.StartActivityForResult()) { result -> if (result.resultCode == RESULT_OK) { result.data?.data?.let { uri -> @@ -144,6 +151,10 @@ class InternalBackupPlaygroundFragment : ComposeFragment() { importFileLauncher.launch(intent) }, + onImportDirectoryClicked = { + val intent = Intent(Intent.ACTION_OPEN_DOCUMENT_TREE) + importDirectoryLauncher.launch(intent) + }, onPlaintextClicked = { viewModel.onPlaintextToggled() }, onSaveToDiskClicked = { val intent = Intent().apply { @@ -251,6 +262,7 @@ fun Screen( onExportClicked: () -> Unit = {}, onImportMemoryClicked: () -> Unit = {}, onImportFileClicked: () -> Unit = {}, + onImportDirectoryClicked: () -> Unit = {}, onPlaintextClicked: () -> Unit = {}, onSaveToDiskClicked: () -> Unit = {}, onValidateFileClicked: () -> Unit = {}, @@ -310,6 +322,11 @@ fun Screen( ) { Text("Import from file") } + Buttons.LargeTonal( + onClick = onImportDirectoryClicked + ) { + Text("Import from directory") + } Buttons.LargeTonal( onClick = onValidateFileClicked diff --git a/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/backup/InternalBackupPlaygroundViewModel.kt b/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/backup/InternalBackupPlaygroundViewModel.kt index 3371f81f42..8b90918f3a 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/backup/InternalBackupPlaygroundViewModel.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/backup/InternalBackupPlaygroundViewModel.kt @@ -5,6 +5,7 @@ package org.thoughtcrime.securesms.components.settings.app.internal.backup +import android.net.Uri import androidx.compose.runtime.MutableState import androidx.compose.runtime.State import androidx.compose.runtime.mutableStateOf @@ -21,6 +22,11 @@ import org.thoughtcrime.securesms.attachments.AttachmentId import org.thoughtcrime.securesms.attachments.DatabaseAttachment import org.thoughtcrime.securesms.backup.v2.BackupMetadata import org.thoughtcrime.securesms.backup.v2.BackupRepository +import org.thoughtcrime.securesms.backup.v2.local.ArchiveFileSystem +import org.thoughtcrime.securesms.backup.v2.local.ArchiveResult +import org.thoughtcrime.securesms.backup.v2.local.LocalArchiver +import org.thoughtcrime.securesms.backup.v2.local.LocalArchiver.FailureCause +import org.thoughtcrime.securesms.backup.v2.local.SnapshotFileSystem import org.thoughtcrime.securesms.database.MessageType import org.thoughtcrime.securesms.database.SignalDatabase import org.thoughtcrime.securesms.dependencies.AppDependencies @@ -111,6 +117,27 @@ class InternalBackupPlaygroundViewModel : ViewModel() { } } + fun import(uri: Uri) { + _state.value = _state.value.copy(backupState = BackupState.IMPORT_IN_PROGRESS) + + val self = Recipient.self() + val selfData = BackupRepository.SelfData(self.aci.get(), self.pni.get(), self.e164.get(), ProfileKey(self.profileKey)) + + disposables += Single.fromCallable { + val archiveFileSystem = ArchiveFileSystem.fromUri(AppDependencies.application, uri)!! + val snapshotInfo = archiveFileSystem.listSnapshots().firstOrNull() ?: return@fromCallable ArchiveResult.failure(FailureCause.MAIN_STREAM) + val snapshotFileSystem = SnapshotFileSystem(AppDependencies.application, snapshotInfo.file) + + LocalArchiver.import(snapshotFileSystem, selfData) + } + .subscribeOn(Schedulers.io()) + .observeOn(AndroidSchedulers.mainThread()) + .subscribeBy { + backupData = null + _state.value = _state.value.copy(backupState = BackupState.NONE) + } + } + fun validate(length: Long, inputStreamFactory: () -> InputStream) { val self = Recipient.self() val selfData = BackupRepository.SelfData(self.aci.get(), self.pni.get(), self.e164.get(), ProfileKey(self.profileKey)) @@ -218,6 +245,7 @@ class InternalBackupPlaygroundViewModel : ViewModel() { reUploadAndArchiveMedia(result.result.mediaIdToAttachmentId(it.mediaId)) } } + else -> _mediaState.set { copy(error = MediaStateError(errorText = "$result")) } } } diff --git a/app/src/main/java/org/thoughtcrime/securesms/database/AttachmentTable.kt b/app/src/main/java/org/thoughtcrime/securesms/database/AttachmentTable.kt index 8b8a163cab..626e0ad9df 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/database/AttachmentTable.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/database/AttachmentTable.kt @@ -293,6 +293,15 @@ class AttachmentTable( } ?: throw IOException("No stream for: $attachmentId") } + @Throws(IOException::class) + fun getAttachmentStream(localArchivableAttachment: LocalArchivableAttachment): InputStream { + return try { + getDataStream(localArchivableAttachment.file, localArchivableAttachment.random, 0) + } catch (e: FileNotFoundException) { + throw IOException("No stream for: ${localArchivableAttachment.file}", e) + } ?: throw IOException("No stream for: ${localArchivableAttachment.file}") + } + @Throws(IOException::class) fun getAttachmentThumbnailStream(attachmentId: AttachmentId, offset: Long): InputStream { return try { @@ -443,6 +452,24 @@ class AttachmentTable( .run() } + fun getLocalArchivableAttachments(): List { + return readableDatabase + .select(*PROJECTION) + .from(TABLE_NAME) + .where("$REMOTE_KEY IS NOT NULL AND $REMOTE_DIGEST IS NOT NULL AND $DATA_FILE IS NOT NULL") + .orderBy("$ID DESC") + .run() + .readToList { + LocalArchivableAttachment( + file = File(it.requireNonNullString(DATA_FILE)), + random = it.requireNonNullBlob(DATA_RANDOM), + size = it.requireLong(DATA_SIZE), + remoteDigest = it.requireBlob(REMOTE_DIGEST)!!, + remoteKey = it.requireBlob(REMOTE_KEY)!! + ) + } + } + fun getRestorableAttachments(batchSize: Int): List { return readableDatabase .select(*PROJECTION) @@ -450,9 +477,29 @@ class AttachmentTable( .where("$TRANSFER_STATE = ?", TRANSFER_NEEDS_RESTORE.toString()) .limit(batchSize) .orderBy("$ID DESC") - .run().readToList { - it.readAttachments() - }.flatten() + .run() + .readToList { + it.readAttachment() + } + } + + fun getLocalRestorableAttachments(batchSize: Int): List { + return readableDatabase + .select(*PROJECTION) + .from(TABLE_NAME) + .where("$REMOTE_KEY IS NOT NULL AND $REMOTE_DIGEST IS NOT NULL AND $TRANSFER_STATE = ?", TRANSFER_NEEDS_RESTORE.toString()) + .limit(batchSize) + .orderBy("$ID DESC") + .run() + .readToList { + LocalRestorableAttachment( + attachmentId = AttachmentId(it.requireLong(ID)), + mmsId = it.requireLong(MESSAGE_ID), + size = it.requireLong(DATA_SIZE), + remoteDigest = it.requireBlob(REMOTE_DIGEST)!!, + remoteKey = it.requireBlob(REMOTE_KEY)!! + ) + } } fun getTotalRestorableAttachmentSize(): Long { @@ -1635,12 +1682,16 @@ class AttachmentTable( @Throws(FileNotFoundException::class) private fun getDataStream(attachmentId: AttachmentId, offset: Long): InputStream? { val dataInfo = getDataFileInfo(attachmentId) ?: return null + return getDataStream(dataInfo.file, dataInfo.random, offset) + } + @Throws(FileNotFoundException::class) + private fun getDataStream(file: File, random: ByteArray, offset: Long): InputStream? { return try { - if (dataInfo.random != null && dataInfo.random.size == 32) { - ModernDecryptingPartInputStream.createFor(attachmentSecret, dataInfo.random, dataInfo.file, offset) + if (random.size == 32) { + ModernDecryptingPartInputStream.createFor(attachmentSecret, random, file, offset) } else { - val stream = ClassicDecryptingPartInputStream.createFor(attachmentSecret, dataInfo.file) + val stream = ClassicDecryptingPartInputStream.createFor(attachmentSecret, file) val skipped = stream.skip(offset) if (skipped != offset) { Log.w(TAG, "Skip failed: $skipped vs $offset") @@ -2353,4 +2404,20 @@ class AttachmentTable( class SyncAttachmentId(val syncMessageId: SyncMessageId, val uuid: UUID?, val digest: ByteArray?, val plaintextHash: String?) class SyncAttachment(val id: AttachmentId, val uuid: UUID?, val digest: ByteArray?, val plaintextHash: String?) + + class LocalArchivableAttachment( + val file: File, + val random: ByteArray, + val size: Long, + val remoteDigest: ByteArray, + val remoteKey: ByteArray + ) + + class LocalRestorableAttachment( + val attachmentId: AttachmentId, + val mmsId: Long, + val size: Long, + val remoteDigest: ByteArray, + val remoteKey: ByteArray + ) } diff --git a/app/src/main/java/org/thoughtcrime/securesms/database/SignalDatabase.kt b/app/src/main/java/org/thoughtcrime/securesms/database/SignalDatabase.kt index 42d538d99a..dea60f8cbb 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/database/SignalDatabase.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/database/SignalDatabase.kt @@ -23,15 +23,15 @@ import org.thoughtcrime.securesms.service.KeyCachingService import org.thoughtcrime.securesms.util.TextSecurePreferences import java.io.File -open class SignalDatabase(private val context: Application, databaseSecret: DatabaseSecret, attachmentSecret: AttachmentSecret, private val name: String = DATABASE_NAME) : +open class SignalDatabase(private val context: Application, databaseSecret: DatabaseSecret, attachmentSecret: AttachmentSecret, name: String = DATABASE_NAME) : SQLiteOpenHelper( context, - DATABASE_NAME, + name, databaseSecret.asString(), null, SignalDatabaseMigrations.DATABASE_VERSION, 0, - SqlCipherErrorHandler(DATABASE_NAME), + SqlCipherErrorHandler(name), SqlCipherDatabaseHook(), true ), diff --git a/app/src/main/java/org/thoughtcrime/securesms/jobs/JobManagerFactories.java b/app/src/main/java/org/thoughtcrime/securesms/jobs/JobManagerFactories.java index 77ed7eda7a..933f950064 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/jobs/JobManagerFactories.java +++ b/app/src/main/java/org/thoughtcrime/securesms/jobs/JobManagerFactories.java @@ -157,6 +157,7 @@ public final class JobManagerFactories { put(LeaveGroupV2Job.KEY, new LeaveGroupV2Job.Factory()); put(LeaveGroupV2WorkerJob.KEY, new LeaveGroupV2WorkerJob.Factory()); put(LinkedDeviceInactiveCheckJob.KEY, new LinkedDeviceInactiveCheckJob.Factory()); + put(LocalArchiveJob.KEY, new LocalArchiveJob.Factory()); put(LocalBackupJob.KEY, new LocalBackupJob.Factory()); put(LocalBackupJobApi29.KEY, new LocalBackupJobApi29.Factory()); put(MarkerJob.KEY, new MarkerJob.Factory()); diff --git a/app/src/main/java/org/thoughtcrime/securesms/jobs/LocalArchiveJob.kt b/app/src/main/java/org/thoughtcrime/securesms/jobs/LocalArchiveJob.kt new file mode 100644 index 0000000000..4e56b0bed8 --- /dev/null +++ b/app/src/main/java/org/thoughtcrime/securesms/jobs/LocalArchiveJob.kt @@ -0,0 +1,190 @@ +package org.thoughtcrime.securesms.jobs + +import org.greenrobot.eventbus.EventBus +import org.greenrobot.eventbus.Subscribe +import org.greenrobot.eventbus.ThreadMode +import org.signal.core.util.Result +import org.signal.core.util.Stopwatch +import org.signal.core.util.logging.Log +import org.thoughtcrime.securesms.R +import org.thoughtcrime.securesms.backup.BackupFileIOError +import org.thoughtcrime.securesms.backup.FullBackupExporter.BackupCanceledException +import org.thoughtcrime.securesms.backup.v2.LocalBackupV2Event +import org.thoughtcrime.securesms.backup.v2.local.ArchiveFileSystem +import org.thoughtcrime.securesms.backup.v2.local.LocalArchiver +import org.thoughtcrime.securesms.backup.v2.local.SnapshotFileSystem +import org.thoughtcrime.securesms.jobmanager.Job +import org.thoughtcrime.securesms.keyvalue.SignalStore +import org.thoughtcrime.securesms.notifications.NotificationChannels +import org.thoughtcrime.securesms.service.GenericForegroundService +import org.thoughtcrime.securesms.service.NotificationController +import org.thoughtcrime.securesms.util.BackupUtil +import org.thoughtcrime.securesms.util.StorageUtil +import java.io.IOException + +/** + * Local backup job for installs using new backupv2 folder format. + * + * @see LocalBackupJob.enqueue + */ +class LocalArchiveJob internal constructor(parameters: Parameters) : Job(parameters) { + + companion object { + const val KEY: String = "LocalArchiveJob" + + private val TAG = Log.tag(LocalArchiveJob::class.java) + } + + override fun serialize(): ByteArray? { + return null + } + + override fun getFactoryKey(): String { + return KEY + } + + override fun run(): Result { + Log.i(TAG, "Executing backup job...") + + BackupFileIOError.clearNotification(context) + + val updater = ProgressUpdater() + + var notification: NotificationController? = null + try { + notification = GenericForegroundService.startForegroundTask( + context, + context.getString(R.string.LocalBackupJob_creating_signal_backup), + NotificationChannels.getInstance().BACKUPS, + R.drawable.ic_signal_backup + ) + } catch (e: UnableToStartException) { + Log.w(TAG, "Unable to start foreground backup service, continuing without service") + } + + try { + updater.notification = notification + EventBus.getDefault().register(updater) + notification?.setIndeterminateProgress() + + val stopwatch = Stopwatch("archive-export") + + val archiveFileSystem = if (BackupUtil.isUserSelectionRequired(context)) { + val backupDirectoryUri = SignalStore.settings.signalBackupDirectory + + if (backupDirectoryUri == null || backupDirectoryUri.path == null) { + throw IOException("Backup Directory has not been selected!") + } + + ArchiveFileSystem.fromUri(context, backupDirectoryUri) + } else { + ArchiveFileSystem.fromFile(context, StorageUtil.getOrCreateBackupV2Directory()) + } + + if (archiveFileSystem == null) { + BackupFileIOError.ACCESS_ERROR.postNotification(context) + Log.w(TAG, "Cannot write to backup directory location.") + return Result.failure() + } + stopwatch.split("create-fs") + + archiveFileSystem.deleteOldTemporaryBackups() + stopwatch.split("delete-old") + + val snapshotFileSystem: SnapshotFileSystem = archiveFileSystem.createSnapshot() ?: return Result.failure() + stopwatch.split("create-snapshot") + + try { + try { + val result = LocalArchiver.export(snapshotFileSystem, archiveFileSystem.filesFileSystem, stopwatch) + Log.i(TAG, "Archive finished with result: $result") + if (result !is org.signal.core.util.Result.Success) { + return Result.failure() + } + } catch (e: Exception) { + Log.w(TAG, "Unable to create local archive", e) + return Result.failure() + } + + stopwatch.split("archive-create") + + // todo [local-backup] verify local backup + EventBus.getDefault().post(LocalBackupV2Event(LocalBackupV2Event.Type.PROGRESS_VERIFYING)) + val valid = true + + stopwatch.split("archive-verify") + + if (valid) { + snapshotFileSystem.finalize() + stopwatch.split("archive-finalize") + } else { + BackupFileIOError.VERIFICATION_FAILED.postNotification(context) + } + + EventBus.getDefault().post(LocalBackupV2Event(LocalBackupV2Event.Type.FINISHED)) + + stopwatch.stop(TAG) + } catch (e: BackupCanceledException) { + EventBus.getDefault().post(LocalBackupV2Event(LocalBackupV2Event.Type.FINISHED)) + Log.w(TAG, "Archive cancelled") + throw e + } catch (e: IOException) { + Log.w(TAG, "Error during archive!", e) + EventBus.getDefault().post(LocalBackupV2Event(LocalBackupV2Event.Type.FINISHED)) + BackupFileIOError.postNotificationForException(context, e) + throw e + } finally { + val cleanUpWasRequired = archiveFileSystem.cleanupSnapshot(snapshotFileSystem) + if (cleanUpWasRequired) { + Log.w(TAG, "Archive failed. Snapshot temp folder needed to be deleted") + } + } + stopwatch.split("new-archive-done") + + archiveFileSystem.deleteOldBackups() + stopwatch.split("delete-old") + + archiveFileSystem.deleteUnusedFiles() + stopwatch.split("delete-unused") + + stopwatch.stop(TAG) + } finally { + notification?.close() + EventBus.getDefault().unregister(updater) + updater.notification = null + } + + return Result.success() + } + + override fun onFailure() { + } + + private class ProgressUpdater { + var notification: NotificationController? = null + + private var previousType: LocalBackupV2Event.Type? = null + + @Subscribe(threadMode = ThreadMode.POSTING) + fun onEvent(event: LocalBackupV2Event) { + val notification = notification ?: return + + if (previousType != event.type) { + notification.replaceTitle(event.type.toString()) // todo [local-backup] use actual strings + previousType = event.type + } + + if (event.estimatedTotalCount == 0L) { + notification.setIndeterminateProgress() + } else { + notification.setProgress(event.estimatedTotalCount, event.count) + } + } + } + + class Factory : Job.Factory { + override fun create(parameters: Parameters, serializedData: ByteArray?): LocalArchiveJob { + return LocalArchiveJob(parameters) + } + } +} diff --git a/app/src/main/java/org/thoughtcrime/securesms/jobs/LocalBackupJob.java b/app/src/main/java/org/thoughtcrime/securesms/jobs/LocalBackupJob.java index b0779186f8..6e64a2a726 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/jobs/LocalBackupJob.java +++ b/app/src/main/java/org/thoughtcrime/securesms/jobs/LocalBackupJob.java @@ -65,6 +65,16 @@ public final class LocalBackupJob extends BaseJob { } } + public static void enqueueArchive() { + JobManager jobManager = AppDependencies.getJobManager(); + Parameters.Builder parameters = new Parameters.Builder() + .setQueue(QUEUE) + .setMaxInstancesForFactory(1) + .setMaxAttempts(3); + + jobManager.add(new LocalArchiveJob(parameters.build())); + } + private LocalBackupJob(@NonNull Job.Parameters parameters) { super(parameters); } diff --git a/app/src/main/java/org/thoughtcrime/securesms/jobs/LocalBackupJobApi29.java b/app/src/main/java/org/thoughtcrime/securesms/jobs/LocalBackupJobApi29.java index a6be26dd6d..2d1a41ed89 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/jobs/LocalBackupJobApi29.java +++ b/app/src/main/java/org/thoughtcrime/securesms/jobs/LocalBackupJobApi29.java @@ -7,19 +7,19 @@ import android.net.Uri; import androidx.annotation.NonNull; import androidx.annotation.Nullable; import androidx.documentfile.provider.DocumentFile; -import androidx.documentfile.provider.DocumentFileHelper; import org.greenrobot.eventbus.EventBus; import org.greenrobot.eventbus.Subscribe; import org.greenrobot.eventbus.ThreadMode; import org.signal.core.util.Stopwatch; -import org.signal.core.util.ThreadUtil; import org.signal.core.util.logging.Log; import org.thoughtcrime.securesms.R; import org.thoughtcrime.securesms.backup.BackupEvent; import org.thoughtcrime.securesms.backup.BackupFileIOError; import org.thoughtcrime.securesms.backup.BackupPassphrase; import org.thoughtcrime.securesms.backup.BackupVerifier; +import org.signal.core.util.androidx.DocumentFileUtil; +import org.signal.core.util.androidx.DocumentFileUtil.OperationResult; import org.thoughtcrime.securesms.backup.FullBackupExporter; import org.thoughtcrime.securesms.crypto.AttachmentSecretProvider; import org.thoughtcrime.securesms.database.SignalDatabase; @@ -36,7 +36,6 @@ import java.text.SimpleDateFormat; import java.util.Date; import java.util.Locale; import java.util.UUID; -import java.util.concurrent.TimeUnit; /** * Backup Job for installs requiring Scoped Storage. @@ -52,16 +51,6 @@ public final class LocalBackupJobApi29 extends BaseJob { public static final String TEMP_BACKUP_FILE_PREFIX = ".backup"; public static final String TEMP_BACKUP_FILE_SUFFIX = ".tmp"; - private static final int MAX_STORAGE_ATTEMPTS = 5; - - private static final long[] WAIT_FOR_SCOPED_STORAGE = new long[] { - TimeUnit.SECONDS.toMillis(0), - TimeUnit.SECONDS.toMillis(2), - TimeUnit.SECONDS.toMillis(10), - TimeUnit.SECONDS.toMillis(20), - TimeUnit.SECONDS.toMillis(30) - }; - LocalBackupJobApi29(@NonNull Parameters parameters) { super(parameters); } @@ -189,43 +178,54 @@ public final class LocalBackupJobApi29 extends BaseJob { } private boolean verifyBackup(String backupPassword, DocumentFile temporaryFile, BackupEvent finishedEvent) throws FullBackupExporter.BackupCanceledException { - Boolean valid = null; - int attempts = 0; + OperationResult result = DocumentFileUtil.retryDocumentFileOperation((attempt, maxAttempts) -> { + Log.i(TAG, "Verify attempt " + (attempt + 1) + "/" + maxAttempts); - while (attempts < MAX_STORAGE_ATTEMPTS && valid == null && !isCanceled()) { - ThreadUtil.sleep(WAIT_FOR_SCOPED_STORAGE[attempts]); + try (InputStream cipherStream = DocumentFileUtil.inputStream(temporaryFile, context)) { + if (cipherStream == null) { + Log.w(TAG, "Found backup file but unable to open input stream"); + return OperationResult.Retry.INSTANCE; + } - try (InputStream cipherStream = context.getContentResolver().openInputStream(temporaryFile.getUri())) { + boolean valid; try { valid = BackupVerifier.verifyFile(cipherStream, backupPassword, finishedEvent.getCount(), this::isCanceled); } catch (IOException e) { Log.w(TAG, "Unable to verify backup", e); valid = false; } + + return new OperationResult.Success(valid); } catch (SecurityException | IOException e) { - attempts++; - Log.w(TAG, "Unable to find backup file, attempt: " + attempts + "/" + MAX_STORAGE_ATTEMPTS, e); + Log.w(TAG, "Unable to find backup file", e); } - } + + if (isCanceled()) { + return new OperationResult.Success(false); + } + + return OperationResult.Retry.INSTANCE; + }); if (isCanceled()) { throw new FullBackupExporter.BackupCanceledException(); } - return valid != null ? valid : false; + return result.isSuccess() && ((OperationResult.Success) result).getValue(); } @SuppressLint("NewApi") private void renameBackup(String fileName, DocumentFile temporaryFile) throws IOException { - int attempts = 0; + OperationResult result = DocumentFileUtil.retryDocumentFileOperation((attempt, maxAttempts) -> { + Log.i(TAG, "Rename attempt " + (attempt + 1) + "/" + maxAttempts); + if (DocumentFileUtil.renameTo(temporaryFile, context, fileName)) { + return new OperationResult.Success(true); + } else { + return OperationResult.Retry.INSTANCE; + } + }); - while (attempts < MAX_STORAGE_ATTEMPTS && !DocumentFileHelper.renameTo(context, temporaryFile, fileName)) { - ThreadUtil.sleep(WAIT_FOR_SCOPED_STORAGE[attempts]); - attempts++; - Log.w(TAG, "Unable to rename backup file, attempt: " + attempts + "/" + MAX_STORAGE_ATTEMPTS); - } - - if (attempts >= MAX_STORAGE_ATTEMPTS) { + if (!result.isSuccess()) { Log.w(TAG, "Failed to rename temp file"); throw new IOException("Renaming temporary backup file failed!"); } diff --git a/app/src/main/java/org/thoughtcrime/securesms/jobs/RequestGroupV2InfoJob.java b/app/src/main/java/org/thoughtcrime/securesms/jobs/RequestGroupV2InfoJob.java index 9f2c9d9825..c272467283 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/jobs/RequestGroupV2InfoJob.java +++ b/app/src/main/java/org/thoughtcrime/securesms/jobs/RequestGroupV2InfoJob.java @@ -30,7 +30,7 @@ public final class RequestGroupV2InfoJob extends BaseJob { /** * Get a particular group state revision for group after message queues are drained. */ - public RequestGroupV2InfoJob(@NonNull GroupId.V2 groupId, int toRevision) { + private RequestGroupV2InfoJob(@NonNull GroupId.V2 groupId, int toRevision) { this(new Parameters.Builder() .setQueue("RequestGroupV2InfoSyncJob") .addConstraint(DecryptionsDrainedConstraint.KEY) diff --git a/app/src/main/java/org/thoughtcrime/securesms/jobs/RestoreAttachmentJob.kt b/app/src/main/java/org/thoughtcrime/securesms/jobs/RestoreAttachmentJob.kt index 875fecbdb3..3a4dfdc1f5 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/jobs/RestoreAttachmentJob.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/jobs/RestoreAttachmentJob.kt @@ -467,7 +467,7 @@ class RestoreAttachmentJob private constructor( pointer, thumbnailFile, maxThumbnailSize, - true, + true, // TODO [backup] don't ignore progressListener ) diff --git a/app/src/main/java/org/thoughtcrime/securesms/jobs/RestoreAttachmentThumbnailJob.kt b/app/src/main/java/org/thoughtcrime/securesms/jobs/RestoreAttachmentThumbnailJob.kt index fb94e63bea..4c67984e7d 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/jobs/RestoreAttachmentThumbnailJob.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/jobs/RestoreAttachmentThumbnailJob.kt @@ -224,7 +224,7 @@ class RestoreAttachmentThumbnailJob private constructor( pointer, thumbnailFile, maxThumbnailSize, - true, + true, // TODO [backup] don't ignore progressListener ) diff --git a/app/src/main/java/org/thoughtcrime/securesms/util/StorageUtil.java b/app/src/main/java/org/thoughtcrime/securesms/util/StorageUtil.java index 1413c6eb5b..5bcdc65322 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/util/StorageUtil.java +++ b/app/src/main/java/org/thoughtcrime/securesms/util/StorageUtil.java @@ -46,6 +46,24 @@ public class StorageUtil { return backups; } + public static File getOrCreateBackupV2Directory() throws NoExternalStorageException { + File storage = Environment.getExternalStorageDirectory(); + + if (!storage.canWrite()) { + throw new NoExternalStorageException(); + } + + File backups = getBackupV2Directory(); + + if (!backups.exists()) { + if (!backups.mkdirs()) { + throw new NoExternalStorageException("Unable to create backup directory..."); + } + } + + return backups; + } + public static File getBackupDirectory() throws NoExternalStorageException { File storage = Environment.getExternalStorageDirectory(); File signal = new File(storage, "Signal"); @@ -59,6 +77,18 @@ public class StorageUtil { return backups; } + public static File getBackupV2Directory() throws NoExternalStorageException { + File storage = Environment.getExternalStorageDirectory(); + File backups = new File(storage, "Signal"); + + //noinspection ConstantConditions + if (BuildConfig.APPLICATION_ID.startsWith(PRODUCTION_PACKAGE_ID + ".")) { + backups = new File(storage, BuildConfig.APPLICATION_ID.substring(PRODUCTION_PACKAGE_ID.length() + 1)); + } + + return backups; + } + @RequiresApi(24) public static @NonNull String getDisplayPath(@NonNull Context context, @NonNull Uri uri) { String lastPathSegment = Objects.requireNonNull(uri.getLastPathSegment()); diff --git a/app/src/main/protowire/LocalArchive.proto b/app/src/main/protowire/LocalArchive.proto new file mode 100644 index 0000000000..2233a1e9c4 --- /dev/null +++ b/app/src/main/protowire/LocalArchive.proto @@ -0,0 +1,15 @@ +syntax = "proto3"; + +package signal.backup.local; + +option java_package = "org.thoughtcrime.securesms.backup.v2.local.proto"; + +message Metadata { + uint32 version = 1; +} + +message FilesFrame { + oneof item { + string mediaName = 1; + } +} diff --git a/core-util-jvm/src/main/java/org/signal/core/util/stream/NonClosingOutputStream.kt b/core-util-jvm/src/main/java/org/signal/core/util/stream/NonClosingOutputStream.kt new file mode 100644 index 0000000000..cb9eb51d97 --- /dev/null +++ b/core-util-jvm/src/main/java/org/signal/core/util/stream/NonClosingOutputStream.kt @@ -0,0 +1,19 @@ +/* + * Copyright 2024 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ + +package org.signal.core.util.stream + +import java.io.FilterOutputStream +import java.io.OutputStream + +/** + * Wraps a provided [OutputStream] but ignores calls to [OutputStream.close] on it but will call [OutputStream.flush] just in case. + * Wrappers must call [OutputStream.close] on the passed in [wrap] stream directly. + */ +class NonClosingOutputStream(wrap: OutputStream) : FilterOutputStream(wrap) { + override fun close() { + flush() + } +} diff --git a/core-util/build.gradle.kts b/core-util/build.gradle.kts index cdb121d35f..4a8011989d 100644 --- a/core-util/build.gradle.kts +++ b/core-util/build.gradle.kts @@ -11,6 +11,7 @@ dependencies { api(project(":core-util-jvm")) implementation(libs.androidx.sqlite) + implementation(libs.androidx.documentfile) testImplementation(testLibs.junit.junit) testImplementation(testLibs.mockito.core) diff --git a/core-util/src/main/java/androidx/documentfile/provider/DocumentFileHack.kt b/core-util/src/main/java/androidx/documentfile/provider/DocumentFileHack.kt new file mode 100644 index 0000000000..75c6de66f3 --- /dev/null +++ b/core-util/src/main/java/androidx/documentfile/provider/DocumentFileHack.kt @@ -0,0 +1,10 @@ +package androidx.documentfile.provider + +/** + * Located in androidx package as [TreeDocumentFile] is package protected. + * + * @return true if can be used like a tree document file (e.g., use content resolver queries) + */ +fun DocumentFile.isTreeDocumentFile(): Boolean { + return this is TreeDocumentFile +} diff --git a/core-util/src/main/java/org/signal/core/util/androidx/DocumentFileInfo.kt b/core-util/src/main/java/org/signal/core/util/androidx/DocumentFileInfo.kt new file mode 100644 index 0000000000..293d4c3a09 --- /dev/null +++ b/core-util/src/main/java/org/signal/core/util/androidx/DocumentFileInfo.kt @@ -0,0 +1,14 @@ +/* + * Copyright 2024 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ + +package org.signal.core.util.androidx + +import androidx.documentfile.provider.DocumentFile + +/** + * Information about a file within the storage. Useful because default [DocumentFile] implementations + * re-query info on each access. + */ +data class DocumentFileInfo(val documentFile: DocumentFile, val name: String, val size: Long) diff --git a/core-util/src/main/java/org/signal/core/util/androidx/DocumentFileUtil.kt b/core-util/src/main/java/org/signal/core/util/androidx/DocumentFileUtil.kt new file mode 100644 index 0000000000..e942592474 --- /dev/null +++ b/core-util/src/main/java/org/signal/core/util/androidx/DocumentFileUtil.kt @@ -0,0 +1,222 @@ +/* + * Copyright 2024 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ + +package org.signal.core.util.androidx + +import android.content.Context +import android.provider.DocumentsContract +import androidx.documentfile.provider.DocumentFile +import androidx.documentfile.provider.isTreeDocumentFile +import org.signal.core.util.ThreadUtil +import org.signal.core.util.logging.Log +import org.signal.core.util.requireLong +import org.signal.core.util.requireNonNullString +import org.signal.core.util.requireString +import java.io.InputStream +import java.io.OutputStream +import kotlin.time.Duration.Companion.seconds + +/** + * Collection of helper and optimizized operations for working with [DocumentFile]s. + */ +object DocumentFileUtil { + + private val TAG = Log.tag(DocumentFileUtil::class) + + private val FILE_PROJECTION = arrayOf(DocumentsContract.Document.COLUMN_DOCUMENT_ID, DocumentsContract.Document.COLUMN_DISPLAY_NAME, DocumentsContract.Document.COLUMN_SIZE) + private const val FILE_SELECTION = "${DocumentsContract.Document.COLUMN_DISPLAY_NAME} = ?" + + private const val LIST_FILES_SELECTION = "${DocumentsContract.Document.COLUMN_MIME_TYPE} != ?" + private val LIST_FILES_SELECTION_ARS = arrayOf(DocumentsContract.Document.MIME_TYPE_DIR) + + private const val MAX_STORAGE_ATTEMPTS: Int = 5 + private val WAIT_FOR_SCOPED_STORAGE: LongArray = longArrayOf(0, 2.seconds.inWholeMilliseconds, 10.seconds.inWholeMilliseconds, 20.seconds.inWholeMilliseconds, 30.seconds.inWholeMilliseconds) + + /** Returns true if the directory represented by the [DocumentFile] has a child with [name]. */ + fun DocumentFile.hasFile(name: String): Boolean { + return findFile(name) != null + } + + /** Returns the [DocumentFile] for a newly created binary file or null if unable or it already exists */ + fun DocumentFile.newFile(name: String): DocumentFile? { + return if (hasFile(name)) { + Log.w(TAG, "Attempt to create new file ($name) but it already exists") + null + } else { + createFile("application/octet-stream", name) + } + } + + /** Returns a [DocumentFile] for directory by [name], creating it if it doesn't already exist */ + fun DocumentFile.mkdirp(name: String): DocumentFile? { + return findFile(name) ?: createDirectory(name) + } + + /** Open an [OutputStream] to the file represented by the [DocumentFile] */ + fun DocumentFile.outputStream(context: Context): OutputStream? { + return context.contentResolver.openOutputStream(uri) + } + + /** Open an [InputStream] to the file represented by the [DocumentFile] */ + @JvmStatic + fun DocumentFile.inputStream(context: Context): InputStream? { + return context.contentResolver.openInputStream(uri) + } + + /** + * Will attempt to find the named [file] in the [root] directory and delete it if found. + * + * @return true if found and deleted, false if the file couldn't be deleted, and null if not found + */ + fun DocumentFile.delete(context: Context, file: String): Boolean? { + return findFile(context, file)?.documentFile?.delete() + } + + /** + * Will attempt to find the name [fileName] in the [root] directory and return useful information if found using + * a single [Context.getContentResolver] query. + * + * Recommend using this over [DocumentFile.findFile] to prevent excess queries for all files and names. + * + * If direct queries fail to find the file, will fallback to using [DocumentFile.findFile]. + */ + fun DocumentFile.findFile(context: Context, fileName: String): DocumentFileInfo? { + val child = if (isTreeDocumentFile()) { + val childrenUri = DocumentsContract.buildChildDocumentsUriUsingTree(uri, DocumentsContract.getDocumentId(uri)) + + try { + context + .contentResolver + .query(childrenUri, FILE_PROJECTION, FILE_SELECTION, arrayOf(fileName), null) + ?.use { cursor -> + if (cursor.count == 1) { + cursor.moveToFirst() + val uri = DocumentsContract.buildDocumentUriUsingTree(uri, cursor.requireString(DocumentsContract.Document.COLUMN_DOCUMENT_ID)) + val displayName = cursor.requireNonNullString(DocumentsContract.Document.COLUMN_DISPLAY_NAME) + val length = cursor.requireLong(DocumentsContract.Document.COLUMN_SIZE) + + DocumentFileInfo(DocumentFile.fromSingleUri(context, uri)!!, displayName, length) + } else { + val message = if (cursor.count > 1) "Multiple files" else "No files" + Log.w(TAG, "$message returned with same name") + null + } + } + } catch (e: Exception) { + Log.d(TAG, "Unable to find file directly on ${javaClass.simpleName}, falling back to OS", e) + null + } + } else { + null + } + + return child ?: this.findFile(fileName)?.let { DocumentFileInfo(it, it.name!!, it.length()) } + } + + /** + * List file names and sizes in the [DocumentFile] by directly querying the content resolver ourselves. The system + * implementation makes a separate query for each name and length method call and gets expensive over 1000's of files. + * + * Will fallback to the provided document file's implementation of [DocumentFile.listFiles] if unable to do it directly. + */ + fun DocumentFile.listFiles(context: Context): List { + if (isTreeDocumentFile()) { + val childrenUri = DocumentsContract.buildChildDocumentsUriUsingTree(uri, DocumentsContract.getDocumentId(uri)) + + try { + val results = context + .contentResolver + .query(childrenUri, FILE_PROJECTION, LIST_FILES_SELECTION, LIST_FILES_SELECTION_ARS, null) + ?.use { cursor -> + val results = ArrayList(cursor.count) + while (cursor.moveToNext()) { + val uri = DocumentsContract.buildDocumentUriUsingTree(uri, cursor.requireString(DocumentsContract.Document.COLUMN_DOCUMENT_ID)) + val displayName = cursor.requireString(DocumentsContract.Document.COLUMN_DISPLAY_NAME) + val length = cursor.requireLong(DocumentsContract.Document.COLUMN_SIZE) + if (displayName != null) { + results.add(DocumentFileInfo(DocumentFile.fromSingleUri(context, uri)!!, displayName, length)) + } + } + + results + } + + if (results != null) { + return results + } else { + Log.w(TAG, "Content provider returned null for query on ${javaClass.simpleName}, falling back to OS") + } + } catch (e: Exception) { + Log.d(TAG, "Unable to query files directly on ${javaClass.simpleName}, falling back to OS", e) + } + } + + return listFiles() + .asSequence() + .filter { it.isFile } + .mapNotNull { file -> file.name?.let { DocumentFileInfo(file, it, file.length()) } } + .toList() + } + + /** + * System implementation swallows the exception and we are having problems with the rename. This inlines the + * same call and logs the exception. Note this implementation does not update the passed in document file like + * the system implementation. Do not use the provided document file after calling this method. + * + * @return true if rename successful + */ + @JvmStatic + fun DocumentFile.renameTo(context: Context, displayName: String): Boolean { + if (isTreeDocumentFile()) { + Log.d(TAG, "Renaming document directly") + try { + val result = DocumentsContract.renameDocument(context.contentResolver, uri, displayName) + return result != null + } catch (e: Exception) { + Log.w(TAG, "Unable to rename document file, falling back to OS", e) + return renameTo(displayName) + } + } else { + return renameTo(displayName) + } + } + + /** + * Historically, we've seen issues with [DocumentFile] operations not working on the first try. This + * retry loop will retry those operations with a varying backoff in attempt to make them work. + */ + @JvmStatic + fun retryDocumentFileOperation(operation: DocumentFileOperation): OperationResult { + var attempts = 0 + + var operationResult = operation.operation(attempts, MAX_STORAGE_ATTEMPTS) + while (attempts < MAX_STORAGE_ATTEMPTS && !operationResult.isSuccess()) { + ThreadUtil.sleep(WAIT_FOR_SCOPED_STORAGE[attempts]) + attempts++ + + operationResult = operation.operation(attempts, MAX_STORAGE_ATTEMPTS) + } + + return operationResult + } + + /** Operation to perform in a retry loop via [retryDocumentFileOperation] that could fail based on timing */ + fun interface DocumentFileOperation { + fun operation(attempt: Int, maxAttempts: Int): OperationResult + } + + /** Result of a single operation in a retry loop via [retryDocumentFileOperation] */ + sealed interface OperationResult { + fun isSuccess(): Boolean { + return this is Success + } + + /** The operation completed successful */ + data class Success(val value: Boolean) : OperationResult + + /** Retry the operation */ + data object Retry : OperationResult + } +} diff --git a/core-util/src/main/java/org/signal/core/util/concurrent/LimitedWorker.kt b/core-util/src/main/java/org/signal/core/util/concurrent/LimitedWorker.kt new file mode 100644 index 0000000000..0dbbc35dd9 --- /dev/null +++ b/core-util/src/main/java/org/signal/core/util/concurrent/LimitedWorker.kt @@ -0,0 +1,36 @@ +/* + * Copyright 2024 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ + +package org.signal.core.util.concurrent + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Semaphore + +object LimitedWorker { + + /** + * Call [worker] on a thread from [executor] for each element in [input] using only up to [maxThreads] concurrently. + * + * This method will block until all work is completed. There is no guarantee that the same threads + * will be used but that only up to [maxThreads] will be actively doing work. + */ + @JvmStatic + fun execute(executor: ExecutorService, maxThreads: Int, input: Collection, worker: (T) -> Unit) { + val doneWorkLatch = CountDownLatch(input.size) + val semaphore = Semaphore(maxThreads) + + for (work in input) { + semaphore.acquire() + executor.execute { + worker(work) + semaphore.release() + doneWorkLatch.countDown() + } + } + + doneWorkLatch.await() + } +} diff --git a/dependencies.gradle.kts b/dependencies.gradle.kts index f2bea311a2..a8325647ba 100644 --- a/dependencies.gradle.kts +++ b/dependencies.gradle.kts @@ -98,6 +98,7 @@ dependencyResolutionManagement { library("androidx-asynclayoutinflater", "androidx.asynclayoutinflater:asynclayoutinflater:1.1.0-alpha01") library("androidx-asynclayoutinflater-appcompat", "androidx.asynclayoutinflater:asynclayoutinflater-appcompat:1.1.0-alpha01") library("androidx-emoji2", "androidx.emoji2:emoji2:1.4.0") + library("androidx-documentfile", "androidx.documentfile:documentfile:1.0.0") // Material library("material-material", "com.google.android.material:material:1.8.0") diff --git a/libsignal-service/src/main/java/org/whispersystems/signalservice/api/crypto/AttachmentCipherInputStream.java b/libsignal-service/src/main/java/org/whispersystems/signalservice/api/crypto/AttachmentCipherInputStream.java index 8ff7501ef9..e81ab35963 100644 --- a/libsignal-service/src/main/java/org/whispersystems/signalservice/api/crypto/AttachmentCipherInputStream.java +++ b/libsignal-service/src/main/java/org/whispersystems/signalservice/api/crypto/AttachmentCipherInputStream.java @@ -25,6 +25,7 @@ import java.security.InvalidKeyException; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.Arrays; +import java.util.function.Supplier; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -71,32 +72,42 @@ public class AttachmentCipherInputStream extends FilterInputStream { */ public static InputStream createForAttachment(File file, long plaintextLength, byte[] combinedKeyMaterial, byte[] digest, byte[] incrementalDigest, int incrementalMacChunkSize, boolean ignoreDigest) throws InvalidMessageException, IOException + { + return createForAttachment(() -> new FileInputStream(file), file.length(), plaintextLength, combinedKeyMaterial, digest, incrementalDigest, incrementalMacChunkSize, ignoreDigest); + } + + /** + * Passing in a null incrementalDigest and/or 0 for the chunk size at the call site disables incremental mac validation. + * + * Passing in true for ignoreDigest DOES NOT VERIFY THE DIGEST + */ + public static InputStream createForAttachment(StreamSupplier streamSupplier, long streamLength, long plaintextLength, byte[] combinedKeyMaterial, byte[] digest, byte[] incrementalDigest, int incrementalMacChunkSize, boolean ignoreDigest) + throws InvalidMessageException, IOException { byte[][] parts = Util.split(combinedKeyMaterial, CIPHER_KEY_SIZE, MAC_KEY_SIZE); Mac mac = initMac(parts[1]); - if (file.length() <= BLOCK_SIZE + mac.getMacLength()) { - throw new InvalidMessageException("Message shorter than crypto overhead!"); + if (streamLength <= BLOCK_SIZE + mac.getMacLength()) { + throw new InvalidMessageException("Message shorter than crypto overhead! length: " + streamLength); } if (!ignoreDigest && digest == null) { throw new InvalidMessageException("Missing digest!"); } - final InputStream wrappedStream; final boolean hasIncrementalMac = incrementalDigest != null && incrementalDigest.length > 0 && incrementalMacChunkSize > 0; if (!hasIncrementalMac) { - try (FileInputStream macVerificationStream = new FileInputStream(file)) { - verifyMac(macVerificationStream, file.length(), mac, digest); + try (InputStream macVerificationStream = streamSupplier.openStream()) { + verifyMac(macVerificationStream, streamLength, mac, digest); } - wrappedStream = new FileInputStream(file); + wrappedStream = streamSupplier.openStream(); } else { wrappedStream = new IncrementalMacInputStream( new IncrementalMacAdditionalValidationsInputStream( - new FileInputStream(file), - file.length(), + streamSupplier.openStream(), + streamLength, mac, digest ), @@ -104,7 +115,7 @@ public class AttachmentCipherInputStream extends FilterInputStream { ChunkSizeChoice.everyNthByte(incrementalMacChunkSize), incrementalDigest); } - InputStream inputStream = new AttachmentCipherInputStream(wrappedStream, parts[0], file.length() - BLOCK_SIZE - mac.getMacLength()); + InputStream inputStream = new AttachmentCipherInputStream(wrappedStream, parts[0], streamLength - BLOCK_SIZE - mac.getMacLength()); if (plaintextLength != 0) { inputStream = new ContentLengthInputStream(inputStream, plaintextLength); @@ -381,4 +392,8 @@ public class AttachmentCipherInputStream extends FilterInputStream { } } } + + public interface StreamSupplier { + @Nonnull InputStream openStream() throws IOException; + } }