Add initial local archive export support.

app/src/main/java/org/thoughtcrime/securesms/backup/v2/BackupRepository.kt

@@ -12,9 +12,12 @@ import kotlinx.coroutines.withContext
 import org.greenrobot.eventbus.EventBus
 import org.signal.core.util.Base64
 import org.signal.core.util.EventTimer
+import org.signal.core.util.concurrent.LimitedWorker
+import org.signal.core.util.concurrent.SignalExecutors
 import org.signal.core.util.fullWalCheckpoint
 import org.signal.core.util.logging.Log
 import org.signal.core.util.money.FiatMoney
+import org.signal.core.util.stream.NonClosingOutputStream
 import org.signal.core.util.withinTransaction
 import org.signal.libsignal.messagebackup.MessageBackup
 import org.signal.libsignal.messagebackup.MessageBackup.ValidationResult
@@ -37,6 +40,7 @@ import org.thoughtcrime.securesms.backup.v2.processor.RecipientBackupProcessor
 import org.thoughtcrime.securesms.backup.v2.processor.StickerBackupProcessor
 import org.thoughtcrime.securesms.backup.v2.proto.BackupInfo
 import org.thoughtcrime.securesms.backup.v2.stream.BackupExportWriter
+import org.thoughtcrime.securesms.backup.v2.stream.BackupImportReader
 import org.thoughtcrime.securesms.backup.v2.stream.EncryptedBackupReader
 import org.thoughtcrime.securesms.backup.v2.stream.EncryptedBackupWriter
 import org.thoughtcrime.securesms.backup.v2.stream.PlainTextBackupReader
@@ -46,6 +50,7 @@ import org.thoughtcrime.securesms.backup.v2.ui.subscription.MessageBackupsTypeFe
 import org.thoughtcrime.securesms.components.settings.app.subscription.RecurringInAppPaymentRepository
 import org.thoughtcrime.securesms.crypto.AttachmentSecretProvider
 import org.thoughtcrime.securesms.crypto.DatabaseSecretProvider
+import org.thoughtcrime.securesms.database.AttachmentTable
 import org.thoughtcrime.securesms.database.KeyValueDatabase
 import org.thoughtcrime.securesms.database.SignalDatabase
 import org.thoughtcrime.securesms.database.model.InAppPaymentSubscriberRecord
@@ -82,6 +87,7 @@ import java.math.BigDecimal
 import java.time.ZonedDateTime
 import java.util.Currency
 import java.util.Locale
+import java.util.concurrent.atomic.AtomicLong
 import kotlin.time.Duration.Companion.milliseconds
 
 object BackupRepository {
@@ -90,6 +96,8 @@ object BackupRepository {
   private const val VERSION = 1L
   private const val MAIN_DB_SNAPSHOT_NAME = "signal-snapshot.db"
   private const val KEYVALUE_DB_SNAPSHOT_NAME = "key-value-snapshot.db"
+  private const val LOCAL_MAIN_DB_SNAPSHOT_NAME = "local-signal-snapshot.db"
+  private const val LOCAL_KEYVALUE_DB_SNAPSHOT_NAME = "local-key-value-snapshot.db"
 
   private val resetInitializedStateErrorAction: StatusCodeErrorAction = { error ->
     when (error.code) {
@@ -126,7 +134,7 @@ object BackupRepository {
     SignalStore.backup.backupTier = null
   }
 
-  private fun createSignalDatabaseSnapshot(): SignalDatabase {
+  private fun createSignalDatabaseSnapshot(name: String): SignalDatabase {
     // Need to do a WAL checkpoint to ensure that the database file we're copying has all pending writes
     if (!SignalDatabase.rawDatabase.fullWalCheckpoint()) {
       Log.w(TAG, "Failed to checkpoint WAL for main database! Not guaranteed to be using the most recent data.")
@@ -137,7 +145,7 @@ object BackupRepository {
       val context = AppDependencies.application
 
       val existingDbFile = context.getDatabasePath(SignalDatabase.DATABASE_NAME)
-      val targetFile = File(existingDbFile.parentFile, MAIN_DB_SNAPSHOT_NAME)
+      val targetFile = File(existingDbFile.parentFile, name)
 
       try {
         existingDbFile.copyTo(targetFile, overwrite = true)
@@ -150,12 +158,12 @@ object BackupRepository {
         context = context,
         databaseSecret = DatabaseSecretProvider.getOrCreateDatabaseSecret(context),
         attachmentSecret = AttachmentSecretProvider.getInstance(context).getOrCreateAttachmentSecret(),
-        name = MAIN_DB_SNAPSHOT_NAME
+        name = name
       )
     }
   }
 
-  private fun createSignalStoreSnapshot(): SignalStore {
+  private fun createSignalStoreSnapshot(name: String): SignalStore {
     val context = AppDependencies.application
 
     // Need to do a WAL checkpoint to ensure that the database file we're copying has all pending writes
@@ -166,7 +174,7 @@ object BackupRepository {
     // We make a copy of the database within a transaction to ensure that no writes occur while we're copying the file
     return KeyValueDatabase.getInstance(context).writableDatabase.withinTransaction {
       val existingDbFile = context.getDatabasePath(KeyValueDatabase.DATABASE_NAME)
-      val targetFile = File(existingDbFile.parentFile, KEYVALUE_DB_SNAPSHOT_NAME)
+      val targetFile = File(existingDbFile.parentFile, name)
 
       try {
         existingDbFile.copyTo(targetFile, overwrite = true)
@@ -175,41 +183,98 @@ object BackupRepository {
         throw IllegalStateException("Failed to copy database file!", e)
       }
 
-      val db = KeyValueDatabase.createWithName(context, KEYVALUE_DB_SNAPSHOT_NAME)
+      val db = KeyValueDatabase.createWithName(context, name)
       SignalStore(KeyValueStore(db))
     }
   }
 
-  private fun deleteDatabaseSnapshot() {
-    val targetFile = AppDependencies.application.getDatabasePath(MAIN_DB_SNAPSHOT_NAME)
+  private fun deleteDatabaseSnapshot(name: String) {
+    val targetFile = AppDependencies.application.getDatabasePath(name)
     if (!targetFile.delete()) {
       Log.w(TAG, "Failed to delete main database snapshot!")
     }
   }
 
-  private fun deleteSignalStoreSnapshot() {
-    val targetFile = AppDependencies.application.getDatabasePath(KEYVALUE_DB_SNAPSHOT_NAME)
+  private fun deleteSignalStoreSnapshot(name: String) {
+    val targetFile = AppDependencies.application.getDatabasePath(name)
     if (!targetFile.delete()) {
       Log.w(TAG, "Failed to delete key value database snapshot!")
     }
   }
 
+  fun localExport(
+    main: OutputStream,
+    localBackupProgressEmitter: ExportProgressListener,
+    archiveAttachment: (AttachmentTable.LocalArchivableAttachment, () -> InputStream?) -> Unit
+  ) {
+    val writer = EncryptedBackupWriter(
+      key = SignalStore.svr.getOrCreateMasterKey().deriveBackupKey(),
+      aci = SignalStore.account.aci!!,
+      outputStream = NonClosingOutputStream(main),
+      append = { main.write(it) }
+    )
+
+    export(currentTime = System.currentTimeMillis(), isLocal = true, writer = writer, progressEmitter = localBackupProgressEmitter) { dbSnapshot ->
+      val localArchivableAttachments = dbSnapshot
+        .attachmentTable
+        .getLocalArchivableAttachments()
+        .associateBy { MediaName.fromDigest(it.remoteDigest) }
+
+      localBackupProgressEmitter.onAttachment(0, localArchivableAttachments.size.toLong())
+
+      val progress = AtomicLong(0)
+
+      LimitedWorker.execute(SignalExecutors.BOUNDED_IO, 4, localArchivableAttachments.values) { attachment ->
+        try {
+          archiveAttachment(attachment) { dbSnapshot.attachmentTable.getAttachmentStream(attachment) }
+        } catch (e: IOException) {
+          Log.w(TAG, "Unable to open attachment, skipping", e)
+        }
+
+        val currentProgress = progress.incrementAndGet()
+        localBackupProgressEmitter.onAttachment(currentProgress, localArchivableAttachments.size.toLong())
+      }
+    }
+  }
+
   fun export(outputStream: OutputStream, append: (ByteArray) -> Unit, plaintext: Boolean = false, currentTime: Long = System.currentTimeMillis()) {
+    val writer: BackupExportWriter = if (plaintext) {
+      PlainTextBackupWriter(outputStream)
+    } else {
+      EncryptedBackupWriter(
+        key = SignalStore.svr.getOrCreateMasterKey().deriveBackupKey(),
+        aci = SignalStore.account.aci!!,
+        outputStream = outputStream,
+        append = append
+      )
+    }
+
+    export(currentTime = currentTime, isLocal = false, writer = writer)
+  }
+
+  /**
+   * Exports to a blob in memory. Should only be used for testing.
+   */
+  fun debugExport(plaintext: Boolean = false, currentTime: Long = System.currentTimeMillis()): ByteArray {
+    val outputStream = ByteArrayOutputStream()
+    export(outputStream = outputStream, append = { mac -> outputStream.write(mac) }, plaintext = plaintext, currentTime = currentTime)
+    return outputStream.toByteArray()
+  }
+
+  private fun export(
+    currentTime: Long,
+    isLocal: Boolean,
+    writer: BackupExportWriter,
+    progressEmitter: ExportProgressListener? = null,
+    exportExtras: ((SignalDatabase) -> Unit)? = null
+  ) {
     val eventTimer = EventTimer()
-    val dbSnapshot: SignalDatabase = createSignalDatabaseSnapshot()
-    val signalStoreSnapshot: SignalStore = createSignalStoreSnapshot()
+    val mainDbName = if (isLocal) LOCAL_MAIN_DB_SNAPSHOT_NAME else MAIN_DB_SNAPSHOT_NAME
+    val keyValueDbName = if (isLocal) LOCAL_KEYVALUE_DB_SNAPSHOT_NAME else KEYVALUE_DB_SNAPSHOT_NAME
 
     try {
-      val writer: BackupExportWriter = if (plaintext) {
-        PlainTextBackupWriter(outputStream)
-      } else {
-        EncryptedBackupWriter(
-          key = SignalStore.svr.getOrCreateMasterKey().deriveBackupKey(),
-          aci = SignalStore.account.aci!!,
-          outputStream = outputStream,
-          append = append
-        )
-      }
+      val dbSnapshot: SignalDatabase = createSignalDatabaseSnapshot(mainDbName)
+      val signalStoreSnapshot: SignalStore = createSignalStoreSnapshot(keyValueDbName)
 
       val exportState = ExportState(backupTime = currentTime, mediaBackupEnabled = SignalStore.backup.backsUpMedia)
 
@@ -223,31 +288,37 @@ object BackupRepository {
 
         // We're using a snapshot, so the transaction is more for perf than correctness
         dbSnapshot.rawWritableDatabase.withinTransaction {
+          progressEmitter?.onAccount()
           AccountDataProcessor.export(dbSnapshot, signalStoreSnapshot) {
             writer.write(it)
             eventTimer.emit("account")
           }
 
+          progressEmitter?.onRecipient()
           RecipientBackupProcessor.export(dbSnapshot, signalStoreSnapshot, exportState) {
             writer.write(it)
             eventTimer.emit("recipient")
           }
 
+          progressEmitter?.onThread()
           ChatBackupProcessor.export(dbSnapshot, exportState) { frame ->
             writer.write(frame)
             eventTimer.emit("thread")
           }
 
+          progressEmitter?.onCall()
           AdHocCallBackupProcessor.export(dbSnapshot) { frame ->
             writer.write(frame)
             eventTimer.emit("call")
           }
 
+          progressEmitter?.onSticker()
           StickerBackupProcessor.export(dbSnapshot) { frame ->
             writer.write(frame)
             eventTimer.emit("sticker-pack")
           }
 
+          progressEmitter?.onMessage()
           ChatItemBackupProcessor.export(dbSnapshot, exportState) { frame ->
             writer.write(frame)
             eventTimer.emit("message")
@@ -255,28 +326,31 @@ object BackupRepository {
         }
       }
 
+      exportExtras?.invoke(dbSnapshot)
+
       Log.d(TAG, "export() ${eventTimer.stop().summary}")
     } finally {
-      deleteDatabaseSnapshot()
-      deleteSignalStoreSnapshot()
+      deleteDatabaseSnapshot(mainDbName)
+      deleteSignalStoreSnapshot(keyValueDbName)
     }
   }
 
-  /**
-   * Exports to a blob in memory. Should only be used for testing.
-   */
-  fun debugExport(plaintext: Boolean = false, currentTime: Long = System.currentTimeMillis()): ByteArray {
-    val outputStream = ByteArrayOutputStream()
-    export(outputStream = outputStream, append = { mac -> outputStream.write(mac) }, plaintext = plaintext, currentTime = currentTime)
-    return outputStream.toByteArray()
+  fun localImport(mainStreamFactory: () -> InputStream, mainStreamLength: Long, selfData: SelfData): ImportResult {
+    val backupKey = SignalStore.svr.getOrCreateMasterKey().deriveBackupKey()
+
+    val frameReader = EncryptedBackupReader(
+      key = backupKey,
+      aci = selfData.aci,
+      length = mainStreamLength,
+      dataStream = mainStreamFactory
+    )
+
+    return frameReader.use { reader ->
+      import(backupKey, reader, selfData)
+    }
   }
 
-  /**
-   * @return The time the backup was created, or null if the backup could not be read.
-   */
   fun import(length: Long, inputStreamFactory: () -> InputStream, selfData: SelfData, plaintext: Boolean = false): ImportResult {
-    val eventTimer = EventTimer()
-
     val backupKey = SignalStore.svr.getOrCreateMasterKey().deriveBackupKey()
 
     val frameReader = if (plaintext) {
@@ -290,6 +364,19 @@ object BackupRepository {
       )
     }
 
+    return frameReader.use { reader ->
+      import(backupKey, reader, selfData)
+    }
+  }
+
+  private fun import(
+    backupKey: BackupKey,
+    frameReader: BackupImportReader,
+    selfData: SelfData,
+    importExtras: ((EventTimer) -> Unit)? = null
+  ): ImportResult {
+    val eventTimer = EventTimer()
+
     val header = frameReader.getHeader()
     if (header == null) {
       Log.e(TAG, "Backup is missing header!")
@@ -364,16 +451,19 @@ object BackupRepository {
         eventTimer.emit("chatItem")
       }
 
+      importExtras?.invoke(eventTimer)
+
       importState.chatIdToLocalThreadId.values.forEach {
         SignalDatabase.threads.update(it, unarchive = false, allowDeletion = false)
       }
     }
 
-    val groups = SignalDatabase.groups.getGroups()
-    while (groups.hasNext()) {
-      val group = groups.next()
-      if (group.id.isV2) {
-        AppDependencies.jobManager.add(RequestGroupV2InfoJob(group.id as GroupId.V2))
+    SignalDatabase.groups.getGroups().use { groups ->
+      while (groups.hasNext()) {
+        val group = groups.next()
+        if (group.id.isV2) {
+          AppDependencies.jobManager.add(RequestGroupV2InfoJob(group.id as GroupId.V2))
+        }
       }
     }
 
@@ -949,6 +1039,16 @@ object BackupRepository {
       iv = Base64.encodeWithPadding(mediaSecrets.iv)
     )
   }
+
+  interface ExportProgressListener {
+    fun onAccount()
+    fun onRecipient()
+    fun onThread()
+    fun onCall()
+    fun onSticker()
+    fun onMessage()
+    fun onAttachment(currentProgress: Long, totalCount: Long)
+  }
 }
 
 data class ArchivedMediaObject(val mediaId: String, val cdn: Int)

app/src/main/java/org/thoughtcrime/securesms/backup/v2/BackupV2Event.kt

@@ -12,3 +12,17 @@ class BackupV2Event(val type: Type, val count: Long, val estimatedTotalCount: Lo
     FINISHED
   }
 }
+
+class LocalBackupV2Event(val type: Type, val count: Long = 0, val estimatedTotalCount: Long = 0) {
+  enum class Type {
+    PROGRESS_ACCOUNT,
+    PROGRESS_RECIPIENT,
+    PROGRESS_THREAD,
+    PROGRESS_CALL,
+    PROGRESS_STICKER,
+    PROGRESS_MESSAGE,
+    PROGRESS_ATTACHMENT,
+    PROGRESS_VERIFYING,
+    FINISHED
+  }
+}

app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/ChatItemExportIterator.kt

@@ -422,7 +422,7 @@ class ChatItemExportIterator(private val cursor: Cursor, private val batchSize:
             ChatUpdateMessage(
               groupCall = GroupCall(
                 state = GroupCall.State.GENERIC,
-                startedCallRecipientId = recipients.getByAci(ACI.from(UuidUtil.parseOrThrow(groupCallUpdateDetails.startedCallUuid))).getOrNull()?.toLong(),
+                startedCallRecipientId = UuidUtil.parseOrNull(groupCallUpdateDetails.startedCallUuid)?.let { recipients.getByAci(ACI.from(it)).getOrNull()?.toLong() },
                 startedCallTimestamp = groupCallUpdateDetails.startedCallTimestamp,
                 endedCallTimestamp = groupCallUpdateDetails.endedCallTimestamp
               )

app/src/main/java/org/thoughtcrime/securesms/backup/v2/local/ArchiveFileSystem.kt

@@ -0,0 +1,334 @@
+/*
+ * Copyright 2024 Signal Messenger, LLC
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+package org.thoughtcrime.securesms.backup.v2.local
+
+import android.content.Context
+import android.net.Uri
+import androidx.documentfile.provider.DocumentFile
+import org.signal.core.util.androidx.DocumentFileInfo
+import org.signal.core.util.androidx.DocumentFileUtil.delete
+import org.signal.core.util.androidx.DocumentFileUtil.hasFile
+import org.signal.core.util.androidx.DocumentFileUtil.inputStream
+import org.signal.core.util.androidx.DocumentFileUtil.listFiles
+import org.signal.core.util.androidx.DocumentFileUtil.mkdirp
+import org.signal.core.util.androidx.DocumentFileUtil.newFile
+import org.signal.core.util.androidx.DocumentFileUtil.outputStream
+import org.signal.core.util.androidx.DocumentFileUtil.renameTo
+import org.signal.core.util.logging.Log
+import org.whispersystems.signalservice.api.backup.MediaName
+import java.io.File
+import java.io.IOException
+import java.io.InputStream
+import java.io.OutputStream
+import java.text.SimpleDateFormat
+import java.util.Calendar
+import java.util.Date
+import java.util.Locale
+
+/**
+ * Provide a domain-specific interface to the root file system backing a local directory based archive.
+ */
+@Suppress("JoinDeclarationAndAssignment")
+class ArchiveFileSystem private constructor(private val context: Context, root: DocumentFile) {
+
+  companion object {
+    val TAG = Log.tag(ArchiveFileSystem::class.java)
+
+    const val BACKUP_DIRECTORY_PREFIX: String = "signal-backup"
+    const val TEMP_BACKUP_DIRECTORY_SUFFIX: String = "tmp"
+
+    /**
+     * Attempt to create an [ArchiveFileSystem] from a tree [Uri].
+     *
+     * Should likely only be called on API29+
+     */
+    fun fromUri(context: Context, uri: Uri): ArchiveFileSystem? {
+      val root = DocumentFile.fromTreeUri(context, uri)
+
+      if (root == null || !root.canWrite()) {
+        return null
+      }
+
+      return ArchiveFileSystem(context, root)
+    }
+
+    /**
+     * Attempt to create an [ArchiveFileSystem] from a regular [File].
+     *
+     * Should likely only be called on < API29.
+     */
+    fun fromFile(context: Context, backupDirectory: File): ArchiveFileSystem {
+      return ArchiveFileSystem(context, DocumentFile.fromFile(backupDirectory))
+    }
+  }
+
+  private val signalBackups: DocumentFile
+
+  /** File access to shared super-set of archive related files (e.g., media + attachments) */
+  val filesFileSystem: FilesFileSystem
+
+  init {
+    signalBackups = root.mkdirp("SignalBackups") ?: throw IOException("Unable to create main backups directory")
+    val filesDirectory = signalBackups.mkdirp("files") ?: throw IOException("Unable to create files directory")
+    filesFileSystem = FilesFileSystem(context, filesDirectory)
+  }
+
+  /**
+   * Delete all folders that match the temp/in-progress backup directory naming convention. Used to clean-up
+   * previous catastrophic backup failures.
+   */
+  fun deleteOldTemporaryBackups() {
+    for (file in signalBackups.listFiles()) {
+      if (file.isDirectory) {
+        val name = file.name
+        if (name != null && name.startsWith(BACKUP_DIRECTORY_PREFIX) && name.endsWith(TEMP_BACKUP_DIRECTORY_SUFFIX)) {
+          if (file.delete()) {
+            Log.w(TAG, "Deleted old temporary backup folder")
+          } else {
+            Log.w(TAG, "Could not delete old temporary backup folder")
+          }
+        }
+      }
+    }
+  }
+
+  /**
+   * Retain up to [limit] most recent backups and delete all others.
+   */
+  fun deleteOldBackups(limit: Int = 2) {
+    Log.i(TAG, "Deleting old backups")
+
+    listSnapshots()
+      .drop(limit)
+      .forEach { it.file.delete() }
+  }
+
+  /**
+   * Attempt to create a [SnapshotFileSystem] to represent a single backup snapshot.
+   */
+  fun createSnapshot(): SnapshotFileSystem? {
+    val timestamp = SimpleDateFormat("yyyy-MM-dd-HH-mm-ss", Locale.US).format(Date())
+    val snapshotDirectoryName = "${BACKUP_DIRECTORY_PREFIX}-$timestamp"
+
+    if (signalBackups.hasFile(snapshotDirectoryName)) {
+      Log.w(TAG, "Backup directory already exists!")
+      return null
+    }
+
+    val workingSnapshotDirectoryName = "$snapshotDirectoryName-$TEMP_BACKUP_DIRECTORY_SUFFIX"
+    val workingSnapshotDirectory = signalBackups.createDirectory(workingSnapshotDirectoryName) ?: return null
+
+    return SnapshotFileSystem(context, snapshotDirectoryName, workingSnapshotDirectoryName, workingSnapshotDirectory)
+  }
+
+  /**
+   * Delete an in-progress snapshot folder after a handled backup failure.
+   *
+   * @return true if the snapshot was deleted
+   */
+  fun cleanupSnapshot(snapshotFileSystem: SnapshotFileSystem): Boolean {
+    check(snapshotFileSystem.workingSnapshotDirectoryName.isNotEmpty()) { "Cannot call cleanup on unnamed snapshot" }
+    return signalBackups.findFile(snapshotFileSystem.workingSnapshotDirectoryName)?.delete() ?: false
+  }
+
+  /**
+   * List all snapshots found in this directory sorted by creation timestamp, newest first.
+   */
+  fun listSnapshots(): List<SnapshotInfo> {
+    return signalBackups
+      .listFiles()
+      .asSequence()
+      .filter { it.isDirectory }
+      .mapNotNull { f -> f.name?.let { it to f } }
+      .filter { (name, _) -> name.startsWith(BACKUP_DIRECTORY_PREFIX) }
+      .map { (name, file) ->
+        val timestamp = name.replace(BACKUP_DIRECTORY_PREFIX, "").toMilliseconds()
+        SnapshotInfo(timestamp, name, file)
+      }
+      .sortedByDescending { it.timestamp }
+      .toList()
+  }
+
+  /**
+   * Clean up unused files in the shared files directory leveraged across all current snapshots. A file
+   * is unused if it is not referenced directly by any current snapshots.
+   */
+  fun deleteUnusedFiles() {
+    Log.i(TAG, "Deleting unused files")
+
+    val allFiles: MutableMap<String, DocumentFileInfo> = filesFileSystem.allFiles().toMutableMap()
+    val snapshots: List<SnapshotInfo> = listSnapshots()
+
+    snapshots
+      .mapNotNull { SnapshotFileSystem.filesInputStream(context, it.file) }
+      .forEach { input ->
+        ArchivedFilesReader(input).use { reader ->
+          reader.forEach { f -> f.mediaName?.let { allFiles.remove(it) } }
+        }
+      }
+
+    var deleted = 0
+    allFiles
+      .values
+      .forEach {
+        if (it.documentFile.delete()) {
+          deleted++
+        }
+      }
+
+    Log.d(TAG, "Cleanup removed $deleted/${allFiles.size} files")
+  }
+
+  /** Useful metadata for a given archive snapshot */
+  data class SnapshotInfo(val timestamp: Long, val name: String, val file: DocumentFile)
+}
+
+/**
+ * Domain specific file system for dealing with individual snapshot data.
+ */
+class SnapshotFileSystem(private val context: Context, private val snapshotDirectoryName: String, val workingSnapshotDirectoryName: String, private val root: DocumentFile) {
+  companion object {
+    const val MAIN_NAME = "main"
+    const val METADATA_NAME = "metadata"
+    const val FILES_NAME = "files"
+
+    /**
+     * Get the files metadata file directly for a snapshot.
+     */
+    fun filesInputStream(context: Context, snapshotDirectory: DocumentFile): InputStream? {
+      return snapshotDirectory.findFile(FILES_NAME)?.inputStream(context)
+    }
+  }
+
+  /**
+   * Creates an unnamed snapshot file system for use in importing.
+   */
+  constructor(context: Context, root: DocumentFile) : this(context, "", "", root)
+
+  fun mainOutputStream(): OutputStream? {
+    return root.newFile(MAIN_NAME)?.outputStream(context)
+  }
+
+  fun mainInputStream(): InputStream? {
+    return root.findFile(MAIN_NAME)?.inputStream(context)
+  }
+
+  fun mainLength(): Long? {
+    return root.findFile(MAIN_NAME)?.length()
+  }
+
+  fun metadataOutputStream(): OutputStream? {
+    return root.newFile(METADATA_NAME)?.outputStream(context)
+  }
+
+  fun metadataInputStream(): InputStream? {
+    return root.findFile(METADATA_NAME)?.inputStream(context)
+  }
+
+  fun filesOutputStream(): OutputStream? {
+    return root.newFile(FILES_NAME)?.outputStream(context)
+  }
+
+  /**
+   * Rename the snapshot from the working temporary name to final name.
+   */
+  fun finalize(): Boolean {
+    check(snapshotDirectoryName.isNotEmpty()) { "Cannot call finalize on unnamed snapshot" }
+    return root.renameTo(context, snapshotDirectoryName)
+  }
+}
+
+/**
+ * Domain specific file system access for accessing backup files (e.g., attachments, media, etc.).
+ */
+class FilesFileSystem(private val context: Context, private val root: DocumentFile) {
+
+  private val subFolders: Map<String, DocumentFile>
+
+  init {
+    val existingFolders = root.listFiles()
+      .mapNotNull { f -> f.name?.let { name -> name to f } }
+      .toMap()
+
+    subFolders = (0..255)
+      .map { i -> i.toString(16).padStart(2, '0') }
+      .associateWith { name ->
+        existingFolders[name] ?: root.createDirectory(name)!!
+      }
+  }
+
+  /**
+   * Enumerate all files in the directory.
+   */
+  fun allFiles(): Map<String, DocumentFileInfo> {
+    val allFiles = HashMap<String, DocumentFileInfo>()
+
+    for (subfolder in subFolders.values) {
+      val subFiles = subfolder.listFiles(context)
+      for (file in subFiles) {
+        allFiles[file.name] = file
+      }
+    }
+
+    return allFiles
+  }
+
+  /**
+   * Creates a new file for the given [mediaName] and returns the output stream for writing to it. The caller
+   * is responsible for determining if the file already exists (see [allFiles]) and deleting it (see [delete]).
+   *
+   * Calling this with a pre-existing file will likely create a second file with a modified name, but is generally
+   * undefined and should be avoided.
+   */
+  fun fileOutputStream(mediaName: MediaName): OutputStream? {
+    val subFileDirectoryName = mediaName.name.substring(0..1)
+    val subFileDirectory = subFolders[subFileDirectoryName]!!
+    val file = subFileDirectory.createFile("application/octet-stream", mediaName.name)
+    return file?.outputStream(context)
+  }
+
+  /**
+   * Given a [file], open and return an [InputStream].
+   */
+  fun fileInputStream(file: DocumentFileInfo): InputStream? {
+    return file.documentFile.inputStream(context)
+  }
+
+  /**
+   * Delete a file for the given [mediaName] if it exists.
+   *
+   * @return true if deleted, false if not, null if not found
+   */
+  fun delete(mediaName: MediaName): Boolean? {
+    val subFileDirectoryName = mediaName.name.substring(0..1)
+    val subFileDirectory = subFolders[subFileDirectoryName]!!
+
+    return subFileDirectory.delete(context, mediaName.name)
+  }
+}
+
+private fun String.toMilliseconds(): Long {
+  val parts: List<String> = split("-").dropLastWhile { it.isEmpty() }
+
+  if (parts.size == 7) {
+    try {
+      val calendar = Calendar.getInstance(Locale.US)
+      calendar[Calendar.YEAR] = parts[1].toInt()
+      calendar[Calendar.MONTH] = parts[2].toInt() - 1
+      calendar[Calendar.DAY_OF_MONTH] = parts[3].toInt()
+      calendar[Calendar.HOUR_OF_DAY] = parts[4].toInt()
+      calendar[Calendar.MINUTE] = parts[5].toInt()
+      calendar[Calendar.SECOND] = parts[6].toInt()
+      calendar[Calendar.MILLISECOND] = 0
+
+      return calendar.timeInMillis
+    } catch (e: NumberFormatException) {
+      Log.w(ArchiveFileSystem.TAG, "Unable to parse timestamp from file name", e)
+    }
+  }
+
+  return -1
+}

app/src/main/java/org/thoughtcrime/securesms/backup/v2/local/ArchivedFilesReader.kt

@@ -0,0 +1,50 @@
+/*
+ * Copyright 2024 Signal Messenger, LLC
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+package org.thoughtcrime.securesms.backup.v2.local
+
+import org.signal.core.util.readNBytesOrThrow
+import org.signal.core.util.readVarInt32
+import org.thoughtcrime.securesms.backup.v2.local.proto.FilesFrame
+import java.io.EOFException
+import java.io.InputStream
+
+/**
+ * Reads [FilesFrame] protos encoded with their length.
+ */
+class ArchivedFilesReader(private val inputStream: InputStream) : Iterator<FilesFrame>, AutoCloseable {
+
+  private var next: FilesFrame? = null
+
+  init {
+    next = read()
+  }
+
+  override fun hasNext(): Boolean {
+    return next != null
+  }
+
+  override fun next(): FilesFrame {
+    next?.let { out ->
+      next = read()
+      return out
+    } ?: throw NoSuchElementException()
+  }
+
+  private fun read(): FilesFrame? {
+    try {
+      val length = inputStream.readVarInt32().also { if (it < 0) return null }
+      val frameBytes: ByteArray = inputStream.readNBytesOrThrow(length)
+
+      return FilesFrame.ADAPTER.decode(frameBytes)
+    } catch (e: EOFException) {
+      return null
+    }
+  }
+
+  override fun close() {
+    inputStream.close()
+  }
+}

app/src/main/java/org/thoughtcrime/securesms/backup/v2/local/ArchivedFilesWriter.kt

@@ -0,0 +1,28 @@
+/*
+ * Copyright 2024 Signal Messenger, LLC
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+package org.thoughtcrime.securesms.backup.v2.local
+
+import org.signal.core.util.writeVarInt32
+import org.thoughtcrime.securesms.backup.v2.local.proto.FilesFrame
+import java.io.IOException
+import java.io.OutputStream
+
+/**
+ * Write [FilesFrame] protos encoded with their length.
+ */
+class ArchivedFilesWriter(private val output: OutputStream) : AutoCloseable {
+
+  @Throws(IOException::class)
+  fun write(frame: FilesFrame) {
+    val bytes = frame.encode()
+    output.writeVarInt32(bytes.size)
+    output.write(bytes)
+  }
+
+  override fun close() {
+    output.close()
+  }
+}

app/src/main/java/org/thoughtcrime/securesms/backup/v2/local/LocalArchiver.kt

@@ -0,0 +1,177 @@
+/*
+ * Copyright 2024 Signal Messenger, LLC
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+package org.thoughtcrime.securesms.backup.v2.local
+
+import org.greenrobot.eventbus.EventBus
+import org.signal.core.util.Base64
+import org.signal.core.util.Stopwatch
+import org.signal.core.util.StreamUtil
+import org.signal.core.util.logging.Log
+import org.thoughtcrime.securesms.backup.v2.BackupRepository
+import org.thoughtcrime.securesms.backup.v2.LocalBackupV2Event
+import org.thoughtcrime.securesms.backup.v2.local.proto.FilesFrame
+import org.thoughtcrime.securesms.backup.v2.local.proto.Metadata
+import org.thoughtcrime.securesms.database.AttachmentTable
+import org.whispersystems.signalservice.api.backup.MediaName
+import org.whispersystems.signalservice.api.crypto.AttachmentCipherOutputStream
+import org.whispersystems.signalservice.api.crypto.AttachmentCipherStreamUtil
+import org.whispersystems.signalservice.internal.crypto.PaddingInputStream
+import java.io.IOException
+import java.io.InputStream
+import java.io.OutputStream
+import java.util.Collections
+import kotlin.random.Random
+
+typealias ArchiveResult = org.signal.core.util.Result<Unit, LocalArchiver.FailureCause>
+
+/**
+ * Handle importing and exporting folder-based archives using backupv2 format.
+ */
+object LocalArchiver {
+
+  private val TAG = Log.tag(LocalArchiver::class)
+  private const val VERSION = 1
+
+  /**
+   * Export archive to the provided [snapshotFileSystem] and store new files in [filesFileSystem].
+   */
+  fun export(snapshotFileSystem: SnapshotFileSystem, filesFileSystem: FilesFileSystem, stopwatch: Stopwatch): ArchiveResult {
+    Log.i(TAG, "Starting export")
+
+    var metadataStream: OutputStream? = null
+    var mainStream: OutputStream? = null
+    var filesStream: OutputStream? = null
+
+    try {
+      metadataStream = snapshotFileSystem.metadataOutputStream() ?: return ArchiveResult.failure(FailureCause.METADATA_STREAM)
+      metadataStream.use { it.write(Metadata(VERSION).encode()) }
+      stopwatch.split("metadata")
+
+      mainStream = snapshotFileSystem.mainOutputStream() ?: return ArchiveResult.failure(FailureCause.MAIN_STREAM)
+
+      Log.i(TAG, "Listing all current files")
+      val allFiles = filesFileSystem.allFiles()
+      stopwatch.split("files-list")
+
+      val mediaNames: MutableSet<MediaName> = Collections.synchronizedSet(HashSet())
+
+      Log.i(TAG, "Starting frame export")
+      BackupRepository.localExport(mainStream, LocalExportProgressListener()) { attachment, source ->
+        val mediaName = MediaName.fromDigest(attachment.remoteDigest)
+
+        mediaNames.add(mediaName)
+
+        if (allFiles[mediaName.name]?.size != attachment.cipherLength) {
+          if (allFiles.containsKey(mediaName.name)) {
+            filesFileSystem.delete(mediaName)
+          }
+
+          source()?.use { sourceStream ->
+            val iv = Random.nextBytes(16) // todo [local-backup] but really do an iv from table
+            val combinedKey = Base64.decode(attachment.remoteKey)
+
+            var destination: OutputStream? = filesFileSystem.fileOutputStream(mediaName)
+
+            if (destination == null) {
+              Log.w(TAG, "Unable to create output file for attachment")
+              // todo [local-backup] should we abort here?
+            } else {
+              // todo [local-backup] but deal with attachment disappearing/deleted by normal app use
+              try {
+                PaddingInputStream(sourceStream, attachment.size).use { input ->
+                  AttachmentCipherOutputStream(combinedKey, iv, destination).use { output ->
+                    StreamUtil.copy(input, output)
+                  }
+                }
+              } catch (e: IOException) {
+                Log.w(TAG, "Unable to save attachment", e)
+                // todo [local-backup] should we abort here?
+              }
+            }
+          }
+        }
+      }
+      stopwatch.split("frames-and-files")
+
+      filesStream = snapshotFileSystem.filesOutputStream() ?: return ArchiveResult.failure(FailureCause.FILES_STREAM)
+      ArchivedFilesWriter(filesStream).use { writer ->
+        mediaNames.forEach { name -> writer.write(FilesFrame(mediaName = name.name)) }
+      }
+      stopwatch.split("files-metadata")
+    } finally {
+      metadataStream?.close()
+      mainStream?.close()
+      filesStream?.close()
+    }
+
+    return ArchiveResult.success(Unit)
+  }
+
+  /**
+   * Import archive data from a folder on the system. Does not restore attachments.
+   */
+  fun import(snapshotFileSystem: SnapshotFileSystem, selfData: BackupRepository.SelfData): ArchiveResult {
+    var metadataStream: InputStream? = null
+
+    try {
+      metadataStream = snapshotFileSystem.metadataInputStream() ?: return ArchiveResult.failure(FailureCause.METADATA_STREAM)
+
+      val mainStreamLength = snapshotFileSystem.mainLength() ?: return ArchiveResult.failure(FailureCause.MAIN_STREAM)
+
+      BackupRepository.localImport(
+        mainStreamFactory = { snapshotFileSystem.mainInputStream()!! },
+        mainStreamLength = mainStreamLength,
+        selfData = selfData
+      )
+    } finally {
+      metadataStream?.close()
+    }
+
+    return ArchiveResult.success(Unit)
+  }
+
+  private val AttachmentTable.LocalArchivableAttachment.cipherLength: Long
+    get() = AttachmentCipherStreamUtil.getCiphertextLength(PaddingInputStream.getPaddedSize(size))
+
+  enum class FailureCause {
+    METADATA_STREAM, MAIN_STREAM, FILES_STREAM
+  }
+
+  private class LocalExportProgressListener : BackupRepository.ExportProgressListener {
+    private var lastAttachmentUpdate: Long = 0
+
+    override fun onAccount() {
+      EventBus.getDefault().post(LocalBackupV2Event(LocalBackupV2Event.Type.PROGRESS_ACCOUNT))
+    }
+
+    override fun onRecipient() {
+      EventBus.getDefault().post(LocalBackupV2Event(LocalBackupV2Event.Type.PROGRESS_RECIPIENT))
+    }
+
+    override fun onThread() {
+      EventBus.getDefault().post(LocalBackupV2Event(LocalBackupV2Event.Type.PROGRESS_THREAD))
+    }
+
+    override fun onCall() {
+      EventBus.getDefault().post(LocalBackupV2Event(LocalBackupV2Event.Type.PROGRESS_CALL))
+    }
+
+    override fun onSticker() {
+      EventBus.getDefault().post(LocalBackupV2Event(LocalBackupV2Event.Type.PROGRESS_STICKER))
+    }
+
+    override fun onMessage() {
+      EventBus.getDefault().post(LocalBackupV2Event(LocalBackupV2Event.Type.PROGRESS_MESSAGE))
+    }
+
+    override fun onAttachment(currentProgress: Long, totalCount: Long) {
+      if (lastAttachmentUpdate > System.currentTimeMillis() || lastAttachmentUpdate + 1000 < System.currentTimeMillis() || currentProgress >= totalCount) {
+        EventBus.getDefault().post(LocalBackupV2Event(LocalBackupV2Event.Type.PROGRESS_ATTACHMENT, currentProgress, totalCount))
+        lastAttachmentUpdate = System.currentTimeMillis()
+      }
+    }
+  }
+}

app/src/main/java/org/thoughtcrime/securesms/backup/v2/stream/EncryptedBackupReader.kt

@@ -45,7 +45,7 @@ class EncryptedBackupReader(
   init {
     val keyMaterial = key.deriveBackupSecrets(aci)
 
-    validateMac(keyMaterial.macKey, length, dataStream())
+    dataStream().use { validateMac(keyMaterial.macKey, length, it) }
 
     countingStream = CountingInputStream(dataStream())
     val iv = countingStream.readNBytesOrThrow(16)