Create backups from copies of the database file.

Still more work here to do with regards to certain tables, like
SignalStore and Recipient.

app/src/main/java/org/thoughtcrime/securesms/backup/v2/BackupRepository.kt

@@ -12,6 +12,7 @@ import kotlinx.coroutines.withContext
 import org.greenrobot.eventbus.EventBus
 import org.signal.core.util.Base64
 import org.signal.core.util.EventTimer
+import org.signal.core.util.fullWalCheckpoint
 import org.signal.core.util.logging.Log
 import org.signal.core.util.money.FiatMoney
 import org.signal.core.util.withinTransaction
@@ -43,6 +44,8 @@ import org.thoughtcrime.securesms.backup.v2.stream.PlainTextBackupWriter
 import org.thoughtcrime.securesms.backup.v2.ui.subscription.MessageBackupsType
 import org.thoughtcrime.securesms.backup.v2.ui.subscription.MessageBackupsTypeFeature
 import org.thoughtcrime.securesms.components.settings.app.subscription.RecurringInAppPaymentRepository
+import org.thoughtcrime.securesms.crypto.AttachmentSecretProvider
+import org.thoughtcrime.securesms.crypto.DatabaseSecretProvider
 import org.thoughtcrime.securesms.database.DistributionListTables
 import org.thoughtcrime.securesms.database.SignalDatabase
 import org.thoughtcrime.securesms.database.model.InAppPaymentSubscriberRecord
@@ -71,6 +74,7 @@ import org.whispersystems.signalservice.internal.push.SubscriptionsConfiguration
 import org.whispersystems.signalservice.internal.push.http.ResumableUploadSpec
 import java.io.ByteArrayOutputStream
 import java.io.File
+import java.io.IOException
 import java.io.InputStream
 import java.io.OutputStream
 import java.math.BigDecimal
@@ -83,6 +87,7 @@ object BackupRepository {
 
   private val TAG = Log.tag(BackupRepository::class.java)
   private const val VERSION = 1L
+  private const val DB_SNAPSHOT_NAME = "signal-snapshot.db"
 
   private val resetInitializedStateErrorAction: StatusCodeErrorAction = { error ->
     when (error.code) {
@@ -105,8 +110,47 @@ object BackupRepository {
     SignalStore.backup().backupTier = null
   }
 
+  private fun createSignalDatabaseSnapshot(): SignalDatabase {
+    // Need to do a WAL checkpoint to ensure that the database file we're copying has all pending writes
+    if (!SignalDatabase.rawDatabase.fullWalCheckpoint()) {
+      Log.w(TAG, "Failed to checkpoint WAL! Not guaranteed to be using the most recent data.")
+    }
+
+    // We make a copy of the database within a transaction to ensure that no writes occur while we're copying the file
+    return SignalDatabase.rawDatabase.withinTransaction {
+      val context = AppDependencies.application
+
+      val existingDbFile = context.getDatabasePath(SignalDatabase.DATABASE_NAME)
+      val targetFile = File(existingDbFile.parentFile, DB_SNAPSHOT_NAME)
+
+      try {
+        existingDbFile.copyTo(targetFile, overwrite = true)
+      } catch (e: IOException) {
+        // TODO [backup] Gracefully handle this error
+        throw IllegalStateException("Failed to copy database file!", e)
+      }
+
+      SignalDatabase(
+        context = context,
+        databaseSecret = DatabaseSecretProvider.getOrCreateDatabaseSecret(context),
+        attachmentSecret = AttachmentSecretProvider.getInstance(context).getOrCreateAttachmentSecret(),
+        name = DB_SNAPSHOT_NAME
+      )
+    }
+  }
+
+  private fun deleteDatabaseSnapshot() {
+    val targetFile = AppDependencies.application.getDatabasePath(DB_SNAPSHOT_NAME)
+    if (!targetFile.delete()) {
+      Log.w(TAG, "Failed to delete database snapshot!")
+    }
+  }
+
   fun export(outputStream: OutputStream, append: (ByteArray) -> Unit, plaintext: Boolean = false) {
     val eventTimer = EventTimer()
+    val dbSnapshot: SignalDatabase = createSignalDatabaseSnapshot()
+
+    try {
       val writer: BackupExportWriter = if (plaintext) {
         PlainTextBackupWriter(outputStream)
       } else {
@@ -129,33 +173,33 @@ object BackupRepository {
         )
         // Note: Without a transaction, we may export inconsistent state. But because we have a transaction,
         // writes from other threads are blocked. This is something to think more about.
-      SignalDatabase.rawDatabase.withinTransaction {
+        dbSnapshot.rawWritableDatabase.withinTransaction {
-        AccountDataProcessor.export {
+          AccountDataProcessor.export(dbSnapshot) {
             writer.write(it)
             eventTimer.emit("account")
           }
 
-        RecipientBackupProcessor.export(exportState) {
+          RecipientBackupProcessor.export(dbSnapshot, exportState) {
             writer.write(it)
             eventTimer.emit("recipient")
           }
 
-        ChatBackupProcessor.export(exportState) { frame ->
+          ChatBackupProcessor.export(dbSnapshot, exportState) { frame ->
             writer.write(frame)
             eventTimer.emit("thread")
           }
 
-        AdHocCallBackupProcessor.export { frame ->
+          AdHocCallBackupProcessor.export(dbSnapshot) { frame ->
             writer.write(frame)
             eventTimer.emit("call")
           }
 
-        StickerBackupProcessor.export { frame ->
+          StickerBackupProcessor.export(dbSnapshot) { frame ->
             writer.write(frame)
             eventTimer.emit("sticker-pack")
           }
 
-        ChatItemBackupProcessor.export(exportState) { frame ->
+          ChatItemBackupProcessor.export(dbSnapshot, exportState) { frame ->
             writer.write(frame)
             eventTimer.emit("message")
           }
@@ -163,6 +207,9 @@ object BackupRepository {
       }
 
       Log.d(TAG, "export() ${eventTimer.stop().summary}")
+    } finally {
+      deleteDatabaseSnapshot()
+    }
   }
 
   fun export(plaintext: Boolean = false): ByteArray {

app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/AccountDataProcessor.kt

@@ -14,7 +14,6 @@ import org.thoughtcrime.securesms.backup.v2.stream.BackupFrameEmitter
 import org.thoughtcrime.securesms.components.settings.app.subscription.InAppPaymentsRepository
 import org.thoughtcrime.securesms.components.settings.app.usernamelinks.UsernameQrCodeColorScheme
 import org.thoughtcrime.securesms.database.SignalDatabase
-import org.thoughtcrime.securesms.database.SignalDatabase.Companion.recipients
 import org.thoughtcrime.securesms.database.model.InAppPaymentSubscriberRecord
 import org.thoughtcrime.securesms.dependencies.AppDependencies
 import org.thoughtcrime.securesms.jobs.RetrieveProfileAvatarJob
@@ -34,12 +33,14 @@ import kotlin.jvm.optionals.getOrNull
 
 object AccountDataProcessor {
 
-  fun export(emitter: BackupFrameEmitter) {
+  fun export(db: SignalDatabase, emitter: BackupFrameEmitter) {
     val context = AppDependencies.application
 
+    // TODO [backup] Need to get it from the db snapshot
     val self = Recipient.self().fresh()
-    val record = recipients.getRecordForSync(self.id)
+    val record = db.recipientTable.getRecordForSync(self.id)
 
+    // TODO [backup] Need to get it from the db snapshot
     val subscriber: InAppPaymentSubscriberRecord? = InAppPaymentsRepository.getSubscriber(InAppPaymentSubscriberRecord.Type.DONATION)
 
     emitter.emit(
@@ -80,7 +81,7 @@ object AccountDataProcessor {
   }
 
   fun import(accountData: AccountData, selfId: RecipientId) {
-    recipients.restoreSelfFromBackup(accountData, selfId)
+    SignalDatabase.recipients.restoreSelfFromBackup(accountData, selfId)
 
     SignalStore.account().setRegistered(true)
 

app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/AdHocCallBackupProcessor.kt

@@ -18,8 +18,8 @@ object AdHocCallBackupProcessor {
 
   val TAG = Log.tag(AdHocCallBackupProcessor::class.java)
 
-  fun export(emitter: BackupFrameEmitter) {
+  fun export(db: SignalDatabase, emitter: BackupFrameEmitter) {
-    SignalDatabase.calls.getAdhocCallsForBackup().use { reader ->
+    db.callTable.getAdhocCallsForBackup().use { reader ->
       for (callLog in reader) {
         if (callLog != null) {
           emitter.emit(Frame(adHocCall = callLog))

app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/ChatBackupProcessor.kt

@@ -19,8 +19,8 @@ import org.thoughtcrime.securesms.recipients.RecipientId
 object ChatBackupProcessor {
   val TAG = Log.tag(ChatBackupProcessor::class.java)
 
-  fun export(exportState: ExportState, emitter: BackupFrameEmitter) {
+  fun export(db: SignalDatabase, exportState: ExportState, emitter: BackupFrameEmitter) {
-    SignalDatabase.threads.getThreadsForBackup().use { reader ->
+    db.threadTable.getThreadsForBackup().use { reader ->
       for (chat in reader) {
         if (exportState.recipientIds.contains(chat.recipientId)) {
           exportState.threadIds.add(chat.id)

app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/ChatItemBackupProcessor.kt

@@ -18,8 +18,8 @@ import org.thoughtcrime.securesms.database.SignalDatabase
 object ChatItemBackupProcessor {
   val TAG = Log.tag(ChatItemBackupProcessor::class.java)
 
-  fun export(exportState: ExportState, emitter: BackupFrameEmitter) {
+  fun export(db: SignalDatabase, exportState: ExportState, emitter: BackupFrameEmitter) {
-    SignalDatabase.messages.getMessagesForBackup(exportState.backupTime, exportState.allowMediaBackup).use { chatItems ->
+    db.messageTable.getMessagesForBackup(exportState.backupTime, exportState.allowMediaBackup).use { chatItems ->
       while (chatItems.hasNext()) {
         val chatItem = chatItems.next()
         if (chatItem != null) {

app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/RecipientBackupProcessor.kt

@@ -30,7 +30,8 @@ object RecipientBackupProcessor {
 
   val TAG = Log.tag(RecipientBackupProcessor::class.java)
 
-  fun export(state: ExportState, emitter: BackupFrameEmitter) {
+  fun export(db: SignalDatabase, state: ExportState, emitter: BackupFrameEmitter) {
+    // TODO [backup] Need to get it from the db snapshot
     val selfId = Recipient.self().id.toLong()
     val releaseChannelId = SignalStore.releaseChannelValues().releaseChannelRecipientId
     if (releaseChannelId != null) {
@@ -44,7 +45,7 @@ object RecipientBackupProcessor {
       )
     }
 
-    SignalDatabase.recipients.getContactsForBackup(selfId).use { reader ->
+    db.recipientTable.getContactsForBackup(selfId).use { reader ->
       for (backupRecipient in reader) {
         if (backupRecipient != null) {
           state.recipientIds.add(backupRecipient.id)
@@ -53,19 +54,19 @@ object RecipientBackupProcessor {
       }
     }
 
-    SignalDatabase.recipients.getGroupsForBackup().use { reader ->
+    db.recipientTable.getGroupsForBackup().use { reader ->
       for (backupRecipient in reader) {
         state.recipientIds.add(backupRecipient.id)
         emitter.emit(Frame(recipient = backupRecipient))
       }
     }
 
-    SignalDatabase.distributionLists.getAllForBackup().forEach {
+    db.distributionListTables.getAllForBackup().forEach {
       state.recipientIds.add(it.id)
       emitter.emit(Frame(recipient = it))
     }
 
-    SignalDatabase.callLinks.getCallLinksForBackup().forEach {
+    db.callLinkTable.getCallLinksForBackup().forEach {
       state.recipientIds.add(it.id)
       emitter.emit(Frame(recipient = it))
     }

app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/StickerBackupProcessor.kt

@@ -17,8 +17,8 @@ import org.thoughtcrime.securesms.dependencies.AppDependencies
 import org.thoughtcrime.securesms.jobs.StickerPackDownloadJob
 
 object StickerBackupProcessor {
-  fun export(emitter: BackupFrameEmitter) {
+  fun export(db: SignalDatabase, emitter: BackupFrameEmitter) {
-    StickerPackRecordReader(SignalDatabase.stickers.allStickerPacks).use { reader ->
+    StickerPackRecordReader(db.stickerTable.allStickerPacks).use { reader ->
       var record: StickerPackRecord? = reader.next
       while (record != null) {
         if (record.isInstalled) {

app/src/main/java/org/thoughtcrime/securesms/database/SignalDatabase.kt

@@ -23,7 +23,7 @@ import org.thoughtcrime.securesms.service.KeyCachingService
 import org.thoughtcrime.securesms.util.TextSecurePreferences
 import java.io.File
 
-open class SignalDatabase(private val context: Application, databaseSecret: DatabaseSecret, attachmentSecret: AttachmentSecret) :
+open class SignalDatabase(private val context: Application, databaseSecret: DatabaseSecret, attachmentSecret: AttachmentSecret, private val name: String = DATABASE_NAME) :
   SQLiteOpenHelper(
     context,
     DATABASE_NAME,
@@ -219,7 +219,7 @@ open class SignalDatabase(private val context: Application, databaseSecret: Data
 
   companion object {
     private val TAG = Log.tag(SignalDatabase::class.java)
-    private const val DATABASE_NAME = "signal.db"
+    const val DATABASE_NAME = "signal.db"
 
     @JvmStatic
     @Volatile

core-util/src/main/java/org/signal/core/util/SQLiteDatabaseExtensions.kt

@@ -89,6 +89,33 @@ fun SupportSQLiteDatabase.areForeignKeyConstraintsEnabled(): Boolean {
   }
 }
 
+/**
+ * Does a full WAL checkpoint (TRUNCATE mode, where the log is for sure flushed and the log is zero'd out).
+ * Will try up to [maxAttempts] times. Can technically fail if the database is too active and the checkpoint
+ * can't complete in a reasonable amount of time.
+ *
+ * See: https://www.sqlite.org/pragma.html#pragma_wal_checkpoint
+ */
+fun SupportSQLiteDatabase.fullWalCheckpoint(maxAttempts: Int = 3): Boolean {
+  var attempts = 0
+
+  while (attempts < maxAttempts) {
+    if (this.walCheckpoint()) {
+      return true
+    }
+
+    attempts++
+  }
+
+  return false
+}
+
+private fun SupportSQLiteDatabase.walCheckpoint(): Boolean {
+  return this.query("PRAGMA wal_checkpoint(TRUNCATE)").use { cursor ->
+    cursor.moveToFirst() && cursor.getInt(0) == 0
+  }
+}
+
 fun SupportSQLiteDatabase.getIndexes(): List<Index> {
   return this.query("SELECT name, tbl_name FROM sqlite_master WHERE type='index' ORDER BY name ASC").readToList { cursor ->
     val indexName = cursor.requireNonNullString("name")