diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/BackupRepository.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/BackupRepository.kt index a02c3a2a34..b448223504 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/BackupRepository.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/BackupRepository.kt @@ -12,6 +12,7 @@ import kotlinx.coroutines.withContext import org.greenrobot.eventbus.EventBus import org.signal.core.util.Base64 import org.signal.core.util.EventTimer +import org.signal.core.util.fullWalCheckpoint import org.signal.core.util.logging.Log import org.signal.core.util.money.FiatMoney import org.signal.core.util.withinTransaction @@ -43,6 +44,8 @@ import org.thoughtcrime.securesms.backup.v2.stream.PlainTextBackupWriter import org.thoughtcrime.securesms.backup.v2.ui.subscription.MessageBackupsType import org.thoughtcrime.securesms.backup.v2.ui.subscription.MessageBackupsTypeFeature import org.thoughtcrime.securesms.components.settings.app.subscription.RecurringInAppPaymentRepository +import org.thoughtcrime.securesms.crypto.AttachmentSecretProvider +import org.thoughtcrime.securesms.crypto.DatabaseSecretProvider import org.thoughtcrime.securesms.database.DistributionListTables import org.thoughtcrime.securesms.database.SignalDatabase import org.thoughtcrime.securesms.database.model.InAppPaymentSubscriberRecord @@ -71,6 +74,7 @@ import org.whispersystems.signalservice.internal.push.SubscriptionsConfiguration import org.whispersystems.signalservice.internal.push.http.ResumableUploadSpec import java.io.ByteArrayOutputStream import java.io.File +import java.io.IOException import java.io.InputStream import java.io.OutputStream import java.math.BigDecimal @@ -83,6 +87,7 @@ object BackupRepository { private val TAG = Log.tag(BackupRepository::class.java) private const val VERSION = 1L + private const val DB_SNAPSHOT_NAME = "signal-snapshot.db" private val resetInitializedStateErrorAction: StatusCodeErrorAction = { error -> when (error.code) { @@ -105,64 +110,106 @@ object BackupRepository { SignalStore.backup().backupTier = null } + private fun createSignalDatabaseSnapshot(): SignalDatabase { + // Need to do a WAL checkpoint to ensure that the database file we're copying has all pending writes + if (!SignalDatabase.rawDatabase.fullWalCheckpoint()) { + Log.w(TAG, "Failed to checkpoint WAL! Not guaranteed to be using the most recent data.") + } + + // We make a copy of the database within a transaction to ensure that no writes occur while we're copying the file + return SignalDatabase.rawDatabase.withinTransaction { + val context = AppDependencies.application + + val existingDbFile = context.getDatabasePath(SignalDatabase.DATABASE_NAME) + val targetFile = File(existingDbFile.parentFile, DB_SNAPSHOT_NAME) + + try { + existingDbFile.copyTo(targetFile, overwrite = true) + } catch (e: IOException) { + // TODO [backup] Gracefully handle this error + throw IllegalStateException("Failed to copy database file!", e) + } + + SignalDatabase( + context = context, + databaseSecret = DatabaseSecretProvider.getOrCreateDatabaseSecret(context), + attachmentSecret = AttachmentSecretProvider.getInstance(context).getOrCreateAttachmentSecret(), + name = DB_SNAPSHOT_NAME + ) + } + } + + private fun deleteDatabaseSnapshot() { + val targetFile = AppDependencies.application.getDatabasePath(DB_SNAPSHOT_NAME) + if (!targetFile.delete()) { + Log.w(TAG, "Failed to delete database snapshot!") + } + } + fun export(outputStream: OutputStream, append: (ByteArray) -> Unit, plaintext: Boolean = false) { val eventTimer = EventTimer() - val writer: BackupExportWriter = if (plaintext) { - PlainTextBackupWriter(outputStream) - } else { - EncryptedBackupWriter( - key = SignalStore.svr().getOrCreateMasterKey().deriveBackupKey(), - aci = SignalStore.account().aci!!, - outputStream = outputStream, - append = append - ) - } + val dbSnapshot: SignalDatabase = createSignalDatabaseSnapshot() - val exportState = ExportState(backupTime = System.currentTimeMillis(), allowMediaBackup = SignalStore.backup().backsUpMedia) - - writer.use { - writer.write( - BackupInfo( - version = VERSION, - backupTimeMs = exportState.backupTime + try { + val writer: BackupExportWriter = if (plaintext) { + PlainTextBackupWriter(outputStream) + } else { + EncryptedBackupWriter( + key = SignalStore.svr().getOrCreateMasterKey().deriveBackupKey(), + aci = SignalStore.account().aci!!, + outputStream = outputStream, + append = append ) - ) - // Note: Without a transaction, we may export inconsistent state. But because we have a transaction, - // writes from other threads are blocked. This is something to think more about. - SignalDatabase.rawDatabase.withinTransaction { - AccountDataProcessor.export { - writer.write(it) - eventTimer.emit("account") - } + } - RecipientBackupProcessor.export(exportState) { - writer.write(it) - eventTimer.emit("recipient") - } + val exportState = ExportState(backupTime = System.currentTimeMillis(), allowMediaBackup = SignalStore.backup().backsUpMedia) - ChatBackupProcessor.export(exportState) { frame -> - writer.write(frame) - eventTimer.emit("thread") - } + writer.use { + writer.write( + BackupInfo( + version = VERSION, + backupTimeMs = exportState.backupTime + ) + ) + // Note: Without a transaction, we may export inconsistent state. But because we have a transaction, + // writes from other threads are blocked. This is something to think more about. + dbSnapshot.rawWritableDatabase.withinTransaction { + AccountDataProcessor.export(dbSnapshot) { + writer.write(it) + eventTimer.emit("account") + } - AdHocCallBackupProcessor.export { frame -> - writer.write(frame) - eventTimer.emit("call") - } + RecipientBackupProcessor.export(dbSnapshot, exportState) { + writer.write(it) + eventTimer.emit("recipient") + } - StickerBackupProcessor.export { frame -> - writer.write(frame) - eventTimer.emit("sticker-pack") - } + ChatBackupProcessor.export(dbSnapshot, exportState) { frame -> + writer.write(frame) + eventTimer.emit("thread") + } - ChatItemBackupProcessor.export(exportState) { frame -> - writer.write(frame) - eventTimer.emit("message") + AdHocCallBackupProcessor.export(dbSnapshot) { frame -> + writer.write(frame) + eventTimer.emit("call") + } + + StickerBackupProcessor.export(dbSnapshot) { frame -> + writer.write(frame) + eventTimer.emit("sticker-pack") + } + + ChatItemBackupProcessor.export(dbSnapshot, exportState) { frame -> + writer.write(frame) + eventTimer.emit("message") + } } } - } - Log.d(TAG, "export() ${eventTimer.stop().summary}") + Log.d(TAG, "export() ${eventTimer.stop().summary}") + } finally { + deleteDatabaseSnapshot() + } } fun export(plaintext: Boolean = false): ByteArray { diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/AccountDataProcessor.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/AccountDataProcessor.kt index 2f0eb126a1..2a92523f09 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/AccountDataProcessor.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/AccountDataProcessor.kt @@ -14,7 +14,6 @@ import org.thoughtcrime.securesms.backup.v2.stream.BackupFrameEmitter import org.thoughtcrime.securesms.components.settings.app.subscription.InAppPaymentsRepository import org.thoughtcrime.securesms.components.settings.app.usernamelinks.UsernameQrCodeColorScheme import org.thoughtcrime.securesms.database.SignalDatabase -import org.thoughtcrime.securesms.database.SignalDatabase.Companion.recipients import org.thoughtcrime.securesms.database.model.InAppPaymentSubscriberRecord import org.thoughtcrime.securesms.dependencies.AppDependencies import org.thoughtcrime.securesms.jobs.RetrieveProfileAvatarJob @@ -34,12 +33,14 @@ import kotlin.jvm.optionals.getOrNull object AccountDataProcessor { - fun export(emitter: BackupFrameEmitter) { + fun export(db: SignalDatabase, emitter: BackupFrameEmitter) { val context = AppDependencies.application + // TODO [backup] Need to get it from the db snapshot val self = Recipient.self().fresh() - val record = recipients.getRecordForSync(self.id) + val record = db.recipientTable.getRecordForSync(self.id) + // TODO [backup] Need to get it from the db snapshot val subscriber: InAppPaymentSubscriberRecord? = InAppPaymentsRepository.getSubscriber(InAppPaymentSubscriberRecord.Type.DONATION) emitter.emit( @@ -80,7 +81,7 @@ object AccountDataProcessor { } fun import(accountData: AccountData, selfId: RecipientId) { - recipients.restoreSelfFromBackup(accountData, selfId) + SignalDatabase.recipients.restoreSelfFromBackup(accountData, selfId) SignalStore.account().setRegistered(true) diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/AdHocCallBackupProcessor.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/AdHocCallBackupProcessor.kt index 1d61e5dd08..a4243261cf 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/AdHocCallBackupProcessor.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/AdHocCallBackupProcessor.kt @@ -18,8 +18,8 @@ object AdHocCallBackupProcessor { val TAG = Log.tag(AdHocCallBackupProcessor::class.java) - fun export(emitter: BackupFrameEmitter) { - SignalDatabase.calls.getAdhocCallsForBackup().use { reader -> + fun export(db: SignalDatabase, emitter: BackupFrameEmitter) { + db.callTable.getAdhocCallsForBackup().use { reader -> for (callLog in reader) { if (callLog != null) { emitter.emit(Frame(adHocCall = callLog)) diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/ChatBackupProcessor.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/ChatBackupProcessor.kt index a6e13af11f..88c59a5af5 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/ChatBackupProcessor.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/ChatBackupProcessor.kt @@ -19,8 +19,8 @@ import org.thoughtcrime.securesms.recipients.RecipientId object ChatBackupProcessor { val TAG = Log.tag(ChatBackupProcessor::class.java) - fun export(exportState: ExportState, emitter: BackupFrameEmitter) { - SignalDatabase.threads.getThreadsForBackup().use { reader -> + fun export(db: SignalDatabase, exportState: ExportState, emitter: BackupFrameEmitter) { + db.threadTable.getThreadsForBackup().use { reader -> for (chat in reader) { if (exportState.recipientIds.contains(chat.recipientId)) { exportState.threadIds.add(chat.id) diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/ChatItemBackupProcessor.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/ChatItemBackupProcessor.kt index dc116d135c..3d70c14783 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/ChatItemBackupProcessor.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/ChatItemBackupProcessor.kt @@ -18,8 +18,8 @@ import org.thoughtcrime.securesms.database.SignalDatabase object ChatItemBackupProcessor { val TAG = Log.tag(ChatItemBackupProcessor::class.java) - fun export(exportState: ExportState, emitter: BackupFrameEmitter) { - SignalDatabase.messages.getMessagesForBackup(exportState.backupTime, exportState.allowMediaBackup).use { chatItems -> + fun export(db: SignalDatabase, exportState: ExportState, emitter: BackupFrameEmitter) { + db.messageTable.getMessagesForBackup(exportState.backupTime, exportState.allowMediaBackup).use { chatItems -> while (chatItems.hasNext()) { val chatItem = chatItems.next() if (chatItem != null) { diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/RecipientBackupProcessor.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/RecipientBackupProcessor.kt index 8888f99faa..edc870621e 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/RecipientBackupProcessor.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/RecipientBackupProcessor.kt @@ -30,7 +30,8 @@ object RecipientBackupProcessor { val TAG = Log.tag(RecipientBackupProcessor::class.java) - fun export(state: ExportState, emitter: BackupFrameEmitter) { + fun export(db: SignalDatabase, state: ExportState, emitter: BackupFrameEmitter) { + // TODO [backup] Need to get it from the db snapshot val selfId = Recipient.self().id.toLong() val releaseChannelId = SignalStore.releaseChannelValues().releaseChannelRecipientId if (releaseChannelId != null) { @@ -44,7 +45,7 @@ object RecipientBackupProcessor { ) } - SignalDatabase.recipients.getContactsForBackup(selfId).use { reader -> + db.recipientTable.getContactsForBackup(selfId).use { reader -> for (backupRecipient in reader) { if (backupRecipient != null) { state.recipientIds.add(backupRecipient.id) @@ -53,19 +54,19 @@ object RecipientBackupProcessor { } } - SignalDatabase.recipients.getGroupsForBackup().use { reader -> + db.recipientTable.getGroupsForBackup().use { reader -> for (backupRecipient in reader) { state.recipientIds.add(backupRecipient.id) emitter.emit(Frame(recipient = backupRecipient)) } } - SignalDatabase.distributionLists.getAllForBackup().forEach { + db.distributionListTables.getAllForBackup().forEach { state.recipientIds.add(it.id) emitter.emit(Frame(recipient = it)) } - SignalDatabase.callLinks.getCallLinksForBackup().forEach { + db.callLinkTable.getCallLinksForBackup().forEach { state.recipientIds.add(it.id) emitter.emit(Frame(recipient = it)) } diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/StickerBackupProcessor.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/StickerBackupProcessor.kt index bc5099cf54..5b87a7c88f 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/StickerBackupProcessor.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/StickerBackupProcessor.kt @@ -17,8 +17,8 @@ import org.thoughtcrime.securesms.dependencies.AppDependencies import org.thoughtcrime.securesms.jobs.StickerPackDownloadJob object StickerBackupProcessor { - fun export(emitter: BackupFrameEmitter) { - StickerPackRecordReader(SignalDatabase.stickers.allStickerPacks).use { reader -> + fun export(db: SignalDatabase, emitter: BackupFrameEmitter) { + StickerPackRecordReader(db.stickerTable.allStickerPacks).use { reader -> var record: StickerPackRecord? = reader.next while (record != null) { if (record.isInstalled) { diff --git a/app/src/main/java/org/thoughtcrime/securesms/database/SignalDatabase.kt b/app/src/main/java/org/thoughtcrime/securesms/database/SignalDatabase.kt index 9919b110e9..775c5c509b 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/database/SignalDatabase.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/database/SignalDatabase.kt @@ -23,7 +23,7 @@ import org.thoughtcrime.securesms.service.KeyCachingService import org.thoughtcrime.securesms.util.TextSecurePreferences import java.io.File -open class SignalDatabase(private val context: Application, databaseSecret: DatabaseSecret, attachmentSecret: AttachmentSecret) : +open class SignalDatabase(private val context: Application, databaseSecret: DatabaseSecret, attachmentSecret: AttachmentSecret, private val name: String = DATABASE_NAME) : SQLiteOpenHelper( context, DATABASE_NAME, @@ -219,7 +219,7 @@ open class SignalDatabase(private val context: Application, databaseSecret: Data companion object { private val TAG = Log.tag(SignalDatabase::class.java) - private const val DATABASE_NAME = "signal.db" + const val DATABASE_NAME = "signal.db" @JvmStatic @Volatile diff --git a/core-util/src/main/java/org/signal/core/util/SQLiteDatabaseExtensions.kt b/core-util/src/main/java/org/signal/core/util/SQLiteDatabaseExtensions.kt index 08b47fda22..0e866ef77e 100644 --- a/core-util/src/main/java/org/signal/core/util/SQLiteDatabaseExtensions.kt +++ b/core-util/src/main/java/org/signal/core/util/SQLiteDatabaseExtensions.kt @@ -89,6 +89,33 @@ fun SupportSQLiteDatabase.areForeignKeyConstraintsEnabled(): Boolean { } } +/** + * Does a full WAL checkpoint (TRUNCATE mode, where the log is for sure flushed and the log is zero'd out). + * Will try up to [maxAttempts] times. Can technically fail if the database is too active and the checkpoint + * can't complete in a reasonable amount of time. + * + * See: https://www.sqlite.org/pragma.html#pragma_wal_checkpoint + */ +fun SupportSQLiteDatabase.fullWalCheckpoint(maxAttempts: Int = 3): Boolean { + var attempts = 0 + + while (attempts < maxAttempts) { + if (this.walCheckpoint()) { + return true + } + + attempts++ + } + + return false +} + +private fun SupportSQLiteDatabase.walCheckpoint(): Boolean { + return this.query("PRAGMA wal_checkpoint(TRUNCATE)").use { cursor -> + cursor.moveToFirst() && cursor.getInt(0) == 0 + } +} + fun SupportSQLiteDatabase.getIndexes(): List { return this.query("SELECT name, tbl_name FROM sqlite_master WHERE type='index' ORDER BY name ASC").readToList { cursor -> val indexName = cursor.requireNonNullString("name")