Add underpinnings to allow for local plaintext export.

Co-authored-by: Cody Henthorne <cody@signal.org>
This commit is contained in:
Alex Hart
2026-03-25 16:44:26 -03:00
committed by Cody Henthorne
parent b605148ac4
commit f2e4881026
15 changed files with 859 additions and 42 deletions

View File

@@ -814,6 +814,34 @@ object BackupRepository {
}
}
@WorkerThread
fun exportForLocalPlaintextArchive(
outputStream: OutputStream,
progressEmitter: ExportProgressListener?,
cancellationSignal: () -> Boolean,
includeMedia: Boolean
): List<AttachmentTable.LocalArchivableAttachment> {
val writer = LibSignalJsonBackupWriter(NonClosingOutputStream(outputStream))
val collectedAttachments = mutableListOf<AttachmentTable.LocalArchivableAttachment>()
export(
currentTime = System.currentTimeMillis(),
isLocal = true,
writer = writer,
backupMode = BackupMode.PLAINTEXT_EXPORT,
progressEmitter = progressEmitter,
cancellationSignal = cancellationSignal,
extraFrameOperation = null,
messageInclusionCutoffTime = 0
) { dbSnapshot ->
if (includeMedia) {
collectedAttachments.addAll(dbSnapshot.attachmentTable.getLocalArchivableAttachmentsForPlaintextExport())
}
}
return collectedAttachments
}
/**
* Export a backup that will be uploaded to the archive CDN.
*/
@@ -2536,7 +2564,8 @@ sealed interface RestoreTimestampResult {
enum class BackupMode {
REMOTE,
LINK_SYNC,
LOCAL;
LOCAL,
PLAINTEXT_EXPORT;
val isLinkAndSync: Boolean
get() = this == LINK_SYNC

View File

@@ -0,0 +1,62 @@
/*
* Copyright 2026 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.thoughtcrime.securesms.backup.v2
import org.signal.archive.proto.BackupInfo
import org.signal.archive.proto.Frame
import org.signal.archive.stream.BackupExportWriter
import org.signal.core.util.logging.Log
import org.signal.core.util.writeVarInt32
import org.signal.libsignal.messagebackup.BackupJsonExporter
import java.io.ByteArrayOutputStream
import java.io.OutputStream
/**
* A [BackupExportWriter] that serializes frames to newline-delimited JSON (JSONL) using
* libsignal's [BackupJsonExporter], which applies sanitization (strips disappearing messages
* and view-once attachments) and optional validation.
*/
class LibSignalJsonBackupWriter(private val outputStream: OutputStream) : BackupExportWriter {
private val TAG = Log.tag(LibSignalJsonBackupWriter::class)
private var exporter: BackupJsonExporter? = null
override fun write(header: BackupInfo) {
val (newExporter, initialChunk) = BackupJsonExporter.start(header.encode())
exporter = newExporter
outputStream.write(initialChunk.toByteArray())
outputStream.write("\n".toByteArray())
}
override fun write(frame: Frame) {
val frameBytes = frame.encode()
val buf = ByteArrayOutputStream(frameBytes.size + 5)
buf.writeVarInt32(frameBytes.size)
buf.write(frameBytes)
val results = exporter!!.exportFrames(buf.toByteArray())
for (result in results) {
result.line?.let {
outputStream.write(it.toByteArray())
outputStream.write("\n".toByteArray())
}
result.errorMessage?.let {
Log.w(TAG, "Frame validation warning: $it")
}
}
}
override fun close() {
exporter?.use {
val error = it.finishExport()
if (error != null) {
Log.w(TAG, "Backup export validation error: $error")
}
}
outputStream.close()
}
}

View File

@@ -164,7 +164,13 @@ fun MessageTable.getMessagesForBackup(db: SignalDatabase, backupTime: Long, self
MessageTable.DELETED_BY
)
.from("${MessageTable.TABLE_NAME} INDEXED BY $dateReceivedIndex")
.where("$STORY_TYPE = 0 AND $PARENT_STORY_ID <= 0 AND $SCHEDULED_DATE = -1 AND ($EXPIRES_IN == 0 OR $EXPIRES_IN > ${1.days.inWholeMilliseconds}) AND $DATE_RECEIVED >= $lastSeenReceivedTime $cutoffQuery")
.where(
buildString {
append("$STORY_TYPE = 0 AND $PARENT_STORY_ID <= 0 AND $SCHEDULED_DATE = -1 AND ")
append("($EXPIRES_IN == 0 OR $EXPIRES_IN > ${1.days.inWholeMilliseconds})")
append(" AND $DATE_RECEIVED >= $lastSeenReceivedTime $cutoffQuery")
}
)
.limit(count)
.orderBy("$DATE_RECEIVED ASC")
.run()

View File

@@ -5,6 +5,7 @@
package org.thoughtcrime.securesms.backup.v2.local
import android.webkit.MimeTypeMap
import okio.ByteString.Companion.toByteString
import org.signal.archive.local.ArchivedFilesWriter
import org.signal.archive.local.proto.FilesFrame
@@ -18,9 +19,11 @@ import org.signal.core.util.StreamUtil
import org.signal.core.util.Util
import org.signal.core.util.logging.Log
import org.signal.core.util.readFully
import org.signal.core.util.toJson
import org.thoughtcrime.securesms.attachments.AttachmentId
import org.thoughtcrime.securesms.backup.v2.BackupRepository
import org.thoughtcrime.securesms.database.AttachmentTable
import org.thoughtcrime.securesms.database.SignalDatabase
import org.thoughtcrime.securesms.keyvalue.SignalStore
import org.thoughtcrime.securesms.keyvalue.protos.LocalBackupCreationProgress
import org.whispersystems.signalservice.api.crypto.AttachmentCipherOutputStream
@@ -30,6 +33,8 @@ import java.io.IOException
import java.io.InputStream
import java.io.OutputStream
import java.util.Collections
import java.util.zip.ZipEntry
import java.util.zip.ZipOutputStream
import javax.crypto.Cipher
import javax.crypto.spec.IvParameterSpec
import javax.crypto.spec.SecretKeySpec
@@ -140,6 +145,73 @@ object LocalArchiver {
}
}
/**
* Export a plaintext archive to the provided [zipOutputStream].
*/
fun exportPlaintext(
zipOutputStream: ZipOutputStream,
includeMedia: Boolean,
stopwatch: Stopwatch,
cancellationSignal: () -> Boolean = { false }
): ArchiveResult {
try {
zipOutputStream.putNextEntry(ZipEntry("metadata.json"))
zipOutputStream.write(Metadata(version = VERSION, backupId = getEncryptedBackupId()).toJson().toByteArray())
zipOutputStream.closeEntry()
stopwatch.split("metadata")
zipOutputStream.putNextEntry(ZipEntry("main.jsonl"))
val progressListener = LocalPlaintextExportProgressListener()
val attachments = BackupRepository.exportForLocalPlaintextArchive(
outputStream = zipOutputStream,
progressEmitter = progressListener,
cancellationSignal = cancellationSignal,
includeMedia = includeMedia
)
zipOutputStream.closeEntry()
stopwatch.split("frames")
if (includeMedia) {
val total = attachments.size.toLong()
var completed = 0L
progressListener.onAttachment(0, total)
val writtenEntries = HashSet<String>()
for (attachment in attachments) {
if (cancellationSignal()) break
val mediaName = MediaName.forLocalBackupFilename(attachment.plaintextHash, attachment.localBackupKey.key)
try {
val ext = attachment.contentType
?.let { MimeTypeMap.getSingleton().getExtensionFromMimeType(it) }
?.let { ".$it" }
?: ""
val prefix = mediaName.name.substring(0..1)
val entryName = "files/$prefix/${mediaName.name}$ext"
if (!writtenEntries.add(entryName)) continue
zipOutputStream.putNextEntry(ZipEntry(entryName))
SignalDatabase.attachments.getAttachmentStream(attachment).use { input ->
StreamUtil.copy(input, zipOutputStream, false, false)
}
zipOutputStream.closeEntry()
} catch (e: IOException) {
Log.w(TAG, "Unable to export attachment ${attachment.attachmentId}, skipping", e)
}
progressListener.onAttachment(++completed, total)
}
stopwatch.split("media")
}
} catch (e: Exception) {
Log.w(TAG, "Unable to create plaintext archive", e)
return ArchiveResult.failure(ArchiveFailure.MainStream)
}
if (cancellationSignal()) {
return ArchiveResult.failure(ArchiveFailure.Cancelled)
}
return ArchiveResult.success(ArchiveSuccess.FullSuccess)
}
private fun getEncryptedBackupId(): Metadata.EncryptedBackupId {
val metadataKey = SignalStore.backup.messageBackupKey.deriveLocalBackupMetadataKey()
val iv = Util.getSecretBytes(12)
@@ -318,4 +390,59 @@ object LocalArchiver {
SignalStore.backup.newLocalBackupProgress = progress
}
}
private class LocalPlaintextExportProgressListener : BackupRepository.ExportProgressListener {
private var lastVerboseUpdate: Long = 0
override fun onAccount() {
post(LocalBackupCreationProgress(exporting = LocalBackupCreationProgress.Exporting(phase = LocalBackupCreationProgress.ExportPhase.ACCOUNT)))
}
override fun onRecipient() {
post(LocalBackupCreationProgress(exporting = LocalBackupCreationProgress.Exporting(phase = LocalBackupCreationProgress.ExportPhase.RECIPIENT)))
}
override fun onThread() {
post(LocalBackupCreationProgress(exporting = LocalBackupCreationProgress.Exporting(phase = LocalBackupCreationProgress.ExportPhase.THREAD)))
}
override fun onCall() {
post(LocalBackupCreationProgress(exporting = LocalBackupCreationProgress.Exporting(phase = LocalBackupCreationProgress.ExportPhase.CALL)))
}
override fun onSticker() {
post(LocalBackupCreationProgress(exporting = LocalBackupCreationProgress.Exporting(phase = LocalBackupCreationProgress.ExportPhase.STICKER)))
}
override fun onNotificationProfile() {
post(LocalBackupCreationProgress(exporting = LocalBackupCreationProgress.Exporting(phase = LocalBackupCreationProgress.ExportPhase.NOTIFICATION_PROFILE)))
}
override fun onChatFolder() {
post(LocalBackupCreationProgress(exporting = LocalBackupCreationProgress.Exporting(phase = LocalBackupCreationProgress.ExportPhase.CHAT_FOLDER)))
}
override fun onMessage(currentProgress: Long, approximateCount: Long) {
if (shouldThrottle(currentProgress >= approximateCount)) return
post(LocalBackupCreationProgress(exporting = LocalBackupCreationProgress.Exporting(phase = LocalBackupCreationProgress.ExportPhase.MESSAGE, frameExportCount = currentProgress, frameTotalCount = approximateCount)))
}
override fun onAttachment(currentProgress: Long, totalCount: Long) {
if (shouldThrottle(currentProgress >= totalCount)) return
post(LocalBackupCreationProgress(transferring = LocalBackupCreationProgress.Transferring(completed = currentProgress, total = totalCount, mediaPhase = true)))
}
private fun shouldThrottle(forceUpdate: Boolean): Boolean {
val now = System.currentTimeMillis()
if (forceUpdate || lastVerboseUpdate > now || lastVerboseUpdate + 1000 < now) {
lastVerboseUpdate = now
return false
}
return true
}
private fun post(progress: LocalBackupCreationProgress) {
SignalStore.backup.newLocalPlaintextBackupProgress = progress
}
}
}

View File

@@ -21,6 +21,7 @@ import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.rememberScrollState
import androidx.compose.foundation.text.KeyboardOptions
import androidx.compose.foundation.verticalScroll
import androidx.compose.material3.AlertDialog
import androidx.compose.material3.ExperimentalMaterial3Api
import androidx.compose.material3.Icon
import androidx.compose.material3.IconButton
@@ -71,9 +72,11 @@ import org.signal.core.util.Hex
import org.signal.core.util.Util
import org.signal.core.util.getLength
import org.thoughtcrime.securesms.MainActivity
import org.thoughtcrime.securesms.backup.isIdle
import org.thoughtcrime.securesms.backup.v2.BackupRepository
import org.thoughtcrime.securesms.backup.v2.ui.BackupAlert
import org.thoughtcrime.securesms.backup.v2.ui.BackupAlertBottomSheet
import org.thoughtcrime.securesms.backup.v2.ui.status.BackupCreationProgressRow
import org.thoughtcrime.securesms.components.settings.app.internal.backup.InternalBackupPlaygroundViewModel.DialogState
import org.thoughtcrime.securesms.components.settings.app.internal.backup.InternalBackupPlaygroundViewModel.ScreenState
import org.thoughtcrime.securesms.dependencies.AppDependencies
@@ -111,10 +114,7 @@ class InternalBackupPlaygroundFragment : ComposeFragment() {
savePlaintextBackupToDiskLauncher = registerForActivityResult(ActivityResultContracts.StartActivityForResult()) { result ->
if (result.resultCode == RESULT_OK) {
result.data?.data?.let { uri ->
viewModel.exportPlaintext(
openStream = { requireContext().contentResolver.openOutputStream(uri)!! },
appendStream = { requireContext().contentResolver.openOutputStream(uri, "wa")!! }
)
viewModel.exportPlaintextZip(uri)
} ?: Toast.makeText(requireContext(), "No URI selected", Toast.LENGTH_SHORT).show()
}
}
@@ -173,15 +173,21 @@ class InternalBackupPlaygroundFragment : ComposeFragment() {
saveEncryptedBackupToDiskLauncher.launch(intent)
},
onSavePlaintextBackupToDiskClicked = {
val intent = Intent().apply {
action = Intent.ACTION_CREATE_DOCUMENT
type = "application/octet-stream"
addCategory(Intent.CATEGORY_OPENABLE)
putExtra(Intent.EXTRA_TITLE, "backup-plaintext-${System.currentTimeMillis()}.bin")
}
viewModel.showPlaintextExportDialog()
},
onPlaintextExportWithMedia = {
viewModel.onPlaintextExportMediaChoiceSelected(includeMedia = true)
val intent = Intent(Intent.ACTION_OPEN_DOCUMENT_TREE)
savePlaintextBackupToDiskLauncher.launch(intent)
},
onPlaintextExportWithoutMedia = {
viewModel.onPlaintextExportMediaChoiceSelected(includeMedia = false)
val intent = Intent(Intent.ACTION_OPEN_DOCUMENT_TREE)
savePlaintextBackupToDiskLauncher.launch(intent)
},
onPlaintextExportDismissed = {
viewModel.dismissPlaintextExportDialog()
},
onSavePlaintextCopyOfRemoteBackupClicked = {
val intent = Intent().apply {
action = Intent.ACTION_CREATE_DOCUMENT
@@ -352,6 +358,9 @@ fun Screen(
onValidateBackupClicked: () -> Unit = {},
onSaveEncryptedBackupToDiskClicked: () -> Unit = {},
onSavePlaintextBackupToDiskClicked: () -> Unit = {},
onPlaintextExportWithMedia: () -> Unit = {},
onPlaintextExportWithoutMedia: () -> Unit = {},
onPlaintextExportDismissed: () -> Unit = {},
onImportEncryptedBackupFromDiskClicked: () -> Unit = {},
onImportEncryptedBackupFromDiskDismissed: () -> Unit = {},
onImportEncryptedBackupFromDiskConfirmed: (aci: String, backupKey: String) -> Unit = { _, _ -> },
@@ -370,6 +379,19 @@ fun Screen(
onDismissed = onImportEncryptedBackupFromDiskDismissed
)
}
DialogState.PlaintextExportMediaChoice -> {
AlertDialog(
onDismissRequest = onPlaintextExportDismissed,
title = { Text("Plaintext backup") },
text = { Text("Include media in the backup?") },
confirmButton = {
TextButton(onClick = onPlaintextExportWithMedia) { Text("With media") }
},
dismissButton = {
TextButton(onClick = onPlaintextExportWithoutMedia) { Text("Without media") }
}
)
}
}
Surface {
@@ -446,11 +468,18 @@ fun Screen(
onClick = onSaveEncryptedBackupToDiskClicked
)
Rows.TextRow(
text = "Save plaintext backup to disk",
label = "Generates a plaintext, uncompressed backup and saves it to your local disk.",
onClick = onSavePlaintextBackupToDiskClicked
)
if (state.plaintextProgress.isIdle) {
Rows.TextRow(
text = "Save plaintext backup to disk",
label = "Generates a plaintext backup as a zip file in the selected directory.",
onClick = onSavePlaintextBackupToDiskClicked
)
} else {
BackupCreationProgressRow(
progress = state.plaintextProgress,
isRemote = false
)
}
Rows.TextRow(
text = "Save plaintext copy of remote backup",
@@ -602,7 +631,7 @@ private fun ImportCredentialsDialog(onSubmit: (aci: String, backupKey: String) -
keyboardType = KeyboardType.Ascii,
imeAction = ImeAction.Next
)
androidx.compose.material3.AlertDialog(
AlertDialog(
onDismissRequest = onDismissed,
title = { Text(text = "Are you sure?") },
text = {

View File

@@ -5,6 +5,7 @@
package org.thoughtcrime.securesms.components.settings.app.internal.backup
import android.net.Uri
import androidx.compose.runtime.MutableState
import androidx.compose.runtime.State
import androidx.compose.runtime.mutableStateOf
@@ -24,7 +25,7 @@ import kotlinx.coroutines.launch
import kotlinx.coroutines.withContext
import org.signal.archive.stream.EncryptedBackupReader
import org.signal.archive.stream.EncryptedBackupReader.Companion.MAC_SIZE
import org.signal.core.models.ServiceId.ACI
import org.signal.core.models.ServiceId
import org.signal.core.models.backup.MessageBackupKey
import org.signal.core.util.Hex
import org.signal.core.util.ThreadUtil
@@ -49,7 +50,9 @@ import org.thoughtcrime.securesms.database.AttachmentTable.DebugAttachmentStats
import org.thoughtcrime.securesms.database.SignalDatabase
import org.thoughtcrime.securesms.dependencies.AppDependencies
import org.thoughtcrime.securesms.jobs.BackupMessagesJob
import org.thoughtcrime.securesms.jobs.LocalBackupJob
import org.thoughtcrime.securesms.keyvalue.SignalStore
import org.thoughtcrime.securesms.keyvalue.protos.LocalBackupCreationProgress
import org.thoughtcrime.securesms.net.SignalNetwork
import org.thoughtcrime.securesms.providers.BlobProvider
import org.thoughtcrime.securesms.recipients.Recipient
@@ -87,9 +90,12 @@ class InternalBackupPlaygroundViewModel : ViewModel() {
val statsState: MutableStateFlow<StatsState> = MutableStateFlow(StatsState())
enum class DialogState {
None,
ImportCredentials
init {
viewModelScope.launch {
SignalStore.backup.newLocalPlaintextBackupProgressFlow.collect { progress ->
_state.value = _state.value.copy(plaintextProgress = progress)
}
}
}
fun exportEncrypted(openStream: () -> OutputStream, appendStream: () -> OutputStream) {
@@ -108,21 +114,23 @@ class InternalBackupPlaygroundViewModel : ViewModel() {
}
}
fun exportPlaintext(openStream: () -> OutputStream, appendStream: () -> OutputStream) {
_state.value = _state.value.copy(statusMessage = "Exporting plaintext backup to disk...")
disposables += Single
.fromCallable {
BackupRepository.exportForDebugging(
outputStream = openStream(),
append = { bytes -> appendStream().use { it.write(bytes) } },
plaintext = true
)
}
.subscribeOn(Schedulers.io())
.observeOn(AndroidSchedulers.mainThread())
.subscribe { data ->
_state.value = _state.value.copy(statusMessage = "Plaintext backup complete!")
}
private var plaintextIncludeMedia: Boolean = false
fun showPlaintextExportDialog() {
_state.value = _state.value.copy(dialog = DialogState.PlaintextExportMediaChoice)
}
fun dismissPlaintextExportDialog() {
_state.value = _state.value.copy(dialog = DialogState.None)
}
fun onPlaintextExportMediaChoiceSelected(includeMedia: Boolean) {
plaintextIncludeMedia = includeMedia
_state.value = _state.value.copy(dialog = DialogState.None)
}
fun exportPlaintextZip(directoryUri: Uri) {
LocalBackupJob.enqueuePlaintextArchive(directoryUri.toString(), plaintextIncludeMedia)
}
fun validateBackup() {
@@ -289,7 +297,7 @@ class InternalBackupPlaygroundViewModel : ViewModel() {
/** True if data is valid, else false */
fun onImportConfirmed(aci: String, backupKey: String): Boolean {
val parsedAci: ACI? = ACI.parseOrNull(aci)
val parsedAci: ServiceId.ACI? = ServiceId.ACI.parseOrNull(aci)
if (aci.isNotBlank() && parsedAci == null) {
_state.value = _state.value.copy(statusMessage = "Invalid ACI! Cannot import.")
@@ -332,6 +340,7 @@ class InternalBackupPlaygroundViewModel : ViewModel() {
_state.value = _state.value.copy(statusMessage = "Import complete!")
ThreadUtil.runOnMain { afterDbRestoreCallback() }
}
RemoteRestoreResult.Canceled,
RemoteRestoreResult.Failure,
RemoteRestoreResult.PermanentSvrBFailure,
@@ -382,6 +391,7 @@ class InternalBackupPlaygroundViewModel : ViewModel() {
SignalStore.backup.cachedMediaCdnPath = null
return@withContext true
}
else -> Log.w(TAG, "Unable to delete remote data", result.getCause())
}
@@ -400,6 +410,7 @@ class InternalBackupPlaygroundViewModel : ViewModel() {
val canReadWriteBackupDirectory: Boolean = false,
val backupTier: MessageBackupTier? = null,
val statusMessage: String? = null,
val plaintextProgress: LocalBackupCreationProgress = LocalBackupCreationProgress(),
val customBackupCredentials: ImportCredentials? = null,
val dialog: DialogState = DialogState.None
)
@@ -470,7 +481,7 @@ class InternalBackupPlaygroundViewModel : ViewModel() {
data class ImportCredentials(
val messageBackupKey: MessageBackupKey,
val aci: ACI
val aci: ServiceId.ACI
)
data class StatsState(
@@ -481,4 +492,10 @@ class InternalBackupPlaygroundViewModel : ViewModel() {
) {
val valid = attachmentStats != null
}
enum class DialogState {
None,
ImportCredentials,
PlaintextExportMediaChoice
}
}

View File

@@ -606,7 +606,32 @@ class AttachmentTable(
random = it.requireNonNullBlob(DATA_RANDOM),
size = it.requireLong(DATA_SIZE),
localBackupKey = AttachmentMetadataTable.getMetadata(it)!!.localBackupKey!!,
plaintextHash = Base64.decode(it.requireNonNullString(DATA_HASH_END))
plaintextHash = Base64.decode(it.requireNonNullString(DATA_HASH_END)),
contentType = it.requireString(CONTENT_TYPE)
)
}
}
fun getLocalArchivableAttachmentsForPlaintextExport(): List<LocalArchivableAttachment> {
return readableDatabase
.select(*PROJECTION_WITH_METADATA)
.from("$TABLE_NAME_WITH_METADTA INNER JOIN ${MessageTable.TABLE_NAME} ON $TABLE_NAME.${MESSAGE_ID} = ${MessageTable.TABLE_NAME}.${MessageTable.ID}")
.where(
"$DATA_HASH_END IS NOT NULL AND $DATA_FILE IS NOT NULL AND ${AttachmentMetadataTable.TABLE_NAME}.${AttachmentMetadataTable.LOCAL_BACKUP_KEY} IS NOT NULL" +
" AND ${MessageTable.TABLE_NAME}.${MessageTable.VIEW_ONCE} = 0" +
" AND ${MessageTable.TABLE_NAME}.${MessageTable.EXPIRES_IN} = 0"
)
.orderBy("$TABLE_NAME.$ID DESC")
.run()
.readToList {
LocalArchivableAttachment(
attachmentId = AttachmentId(it.requireLong(ID)),
file = File(it.requireNonNullString(DATA_FILE)),
random = it.requireNonNullBlob(DATA_RANDOM),
size = it.requireLong(DATA_SIZE),
localBackupKey = AttachmentMetadataTable.getMetadata(it)!!.localBackupKey!!,
plaintextHash = Base64.decode(it.requireNonNullString(DATA_HASH_END)),
contentType = it.requireString(CONTENT_TYPE)
)
}
}
@@ -3853,7 +3878,8 @@ class AttachmentTable(
val random: ByteArray,
val size: Long,
val plaintextHash: ByteArray,
val localBackupKey: LocalBackupKey
val localBackupKey: LocalBackupKey,
val contentType: String? = null
)
data class RestorableAttachment(

View File

@@ -202,6 +202,7 @@ public final class JobManagerFactories {
put(LinkedDeviceInactiveCheckJob.KEY, new LinkedDeviceInactiveCheckJob.Factory());
put(LocalArchiveJob.KEY, new LocalArchiveJob.Factory());
put(LocalBackupJob.KEY, new LocalBackupJob.Factory());
put(LocalPlaintextArchiveJob.KEY, new LocalPlaintextArchiveJob.Factory());
put(LocalBackupJobApi29.KEY, new LocalBackupJobApi29.Factory());
put(MarkerJob.KEY, new MarkerJob.Factory());
put(MultiDeviceAttachmentBackfillMissingJob.KEY, new MultiDeviceAttachmentBackfillMissingJob.Factory());

View File

@@ -77,6 +77,16 @@ public final class LocalBackupJob extends BaseJob {
jobManager.add(new LocalArchiveJob(parameters.build()));
}
public static void enqueuePlaintextArchive(String destinationUri, boolean includeMedia) {
JobManager jobManager = AppDependencies.getJobManager();
Parameters.Builder parameters = new Parameters.Builder()
.setQueue(QUEUE)
.setMaxInstancesForFactory(1)
.setMaxAttempts(3);
jobManager.add(new LocalPlaintextArchiveJob(destinationUri, includeMedia, parameters.build()));
}
private LocalBackupJob(@NonNull Job.Parameters parameters) {
super(parameters);
}

View File

@@ -0,0 +1,188 @@
package org.thoughtcrime.securesms.jobs
import android.net.Uri
import androidx.documentfile.provider.DocumentFile
import org.signal.core.util.Stopwatch
import org.signal.core.util.logging.Log
import org.thoughtcrime.securesms.R
import org.thoughtcrime.securesms.backup.v2.local.LocalArchiver
import org.thoughtcrime.securesms.database.SignalDatabase
import org.thoughtcrime.securesms.dependencies.AppDependencies
import org.thoughtcrime.securesms.jobmanager.Job
import org.thoughtcrime.securesms.jobmanager.JsonJobData
import org.thoughtcrime.securesms.keyvalue.SignalStore
import org.thoughtcrime.securesms.keyvalue.protos.LocalBackupCreationProgress
import org.thoughtcrime.securesms.notifications.NotificationChannels
import org.thoughtcrime.securesms.service.GenericForegroundService
import org.thoughtcrime.securesms.service.NotificationController
import java.io.IOException
import java.text.SimpleDateFormat
import java.util.Date
import java.util.Locale
import java.util.zip.ZipOutputStream
class LocalPlaintextArchiveJob internal constructor(
private val destinationUri: String,
private val includeMedia: Boolean,
parameters: Parameters
) : Job(parameters) {
companion object {
const val KEY: String = "LocalPlaintextArchiveJob"
private val TAG = Log.tag(LocalPlaintextArchiveJob::class.java)
private const val KEY_DESTINATION_URI = "destination_uri"
private const val KEY_INCLUDE_MEDIA = "include_media"
}
override fun serialize(): ByteArray? {
return JsonJobData.Builder()
.putString(KEY_DESTINATION_URI, destinationUri)
.putBoolean(KEY_INCLUDE_MEDIA, includeMedia)
.serialize()
}
override fun getFactoryKey(): String {
return KEY
}
override fun run(): Result {
Log.i(TAG, "Executing plaintext archive job...")
var notification: NotificationController? = null
try {
notification = GenericForegroundService.startForegroundTask(
context,
context.getString(R.string.LocalBackupJob_creating_signal_backup),
NotificationChannels.getInstance().BACKUPS,
R.drawable.ic_signal_backup
)
} catch (e: UnableToStartException) {
Log.w(TAG, "Unable to start foreground service, continuing without service")
}
try {
notification?.setIndeterminateProgress()
setProgress(LocalBackupCreationProgress(exporting = LocalBackupCreationProgress.Exporting(phase = LocalBackupCreationProgress.ExportPhase.INITIALIZING)), notification)
val stopwatch = Stopwatch("plaintext-archive-export")
val root = DocumentFile.fromTreeUri(context, Uri.parse(destinationUri))
if (root == null || !root.canWrite()) {
Log.w(TAG, "Cannot write to destination directory.")
return Result.failure()
}
val timestamp = SimpleDateFormat("yyyy-MM-dd-HH-mm-ss", Locale.US).format(Date())
val fileName = "signal-export-$timestamp"
val zipFile = root.createFile("application/zip", fileName)
if (zipFile == null) {
Log.w(TAG, "Unable to create zip file")
return Result.failure()
}
stopwatch.split("create-file")
try {
SignalDatabase.attachmentMetadata.insertNewKeysForExistingAttachments()
val outputStream = context.contentResolver.openOutputStream(zipFile.uri)
if (outputStream == null) {
Log.w(TAG, "Unable to open output stream for zip file")
zipFile.delete()
return Result.failure()
}
ZipOutputStream(outputStream).use { zipOutputStream ->
val result = LocalArchiver.exportPlaintext(zipOutputStream, includeMedia, stopwatch, cancellationSignal = { isCanceled })
Log.i(TAG, "Plaintext archive finished with result: $result")
if (result !is org.signal.core.util.Result.Success) {
zipFile.delete()
return Result.failure()
}
}
stopwatch.split("archive-create")
setProgress(LocalBackupCreationProgress(idle = LocalBackupCreationProgress.Idle()), notification)
} catch (e: IOException) {
Log.w(TAG, "Error during plaintext archive!", e)
setProgress(LocalBackupCreationProgress(idle = LocalBackupCreationProgress.Idle()), notification)
zipFile.delete()
throw e
}
stopwatch.stop(TAG)
} finally {
notification?.close()
}
return Result.success()
}
override fun onFailure() {
SignalStore.backup.newLocalPlaintextBackupProgress = LocalBackupCreationProgress(idle = LocalBackupCreationProgress.Idle())
}
private fun setProgress(progress: LocalBackupCreationProgress, notification: NotificationController?) {
SignalStore.backup.newLocalPlaintextBackupProgress = progress
updateNotification(progress, notification)
}
private var previousPhase: NotificationPhase? = null
private fun updateNotification(progress: LocalBackupCreationProgress, notification: NotificationController?) {
if (notification == null) return
val exporting = progress.exporting
val transferring = progress.transferring
when {
exporting != null -> {
val phase = NotificationPhase.Export(exporting.phase)
if (previousPhase != phase) {
notification.replaceTitle(exporting.phase.toString())
previousPhase = phase
}
if (exporting.frameTotalCount == 0L) {
notification.setIndeterminateProgress()
} else {
notification.setProgress(exporting.frameTotalCount, exporting.frameExportCount)
}
}
transferring != null -> {
if (previousPhase !is NotificationPhase.Transfer) {
notification.replaceTitle(AppDependencies.application.getString(R.string.LocalArchiveJob__exporting_media))
previousPhase = NotificationPhase.Transfer
}
if (transferring.total == 0L) {
notification.setIndeterminateProgress()
} else {
notification.setProgress(transferring.total, transferring.completed)
}
}
else -> {
notification.setIndeterminateProgress()
}
}
}
private sealed interface NotificationPhase {
data class Export(val phase: LocalBackupCreationProgress.ExportPhase) : NotificationPhase
data object Transfer : NotificationPhase
}
class Factory : Job.Factory<LocalPlaintextArchiveJob?> {
override fun create(parameters: Parameters, serializedData: ByteArray?): LocalPlaintextArchiveJob {
val data = JsonJobData.deserialize(serializedData)
return LocalPlaintextArchiveJob(
destinationUri = data.getString(KEY_DESTINATION_URI),
includeMedia = data.getBoolean(KEY_INCLUDE_MEDIA),
parameters = parameters
)
}
}
}

View File

@@ -106,6 +106,7 @@ class BackupValues(store: KeyValueStore) : SignalStoreValues(store) {
private const val KEY_NEW_LOCAL_BACKUPS_LAST_BACKUP_TIME = "backup.new_local_backups_last_backup_time"
private const val KEY_NEW_LOCAL_BACKUPS_SELECTED_SNAPSHOT_TIMESTAMP = "backup.new_local_backups_selected_snapshot_timestamp"
private const val KEY_NEW_LOCAL_BACKUPS_CREATION_PROGRESS = "backup.new_local_backups_creation_progress"
private const val KEY_NEW_LOCAL_PLAINTEXT_BACKUPS_CREATION_PROGRESS = "backup.new_local_plaintext_backups_creation_progress"
private const val KEY_LOCAL_RESTORE_ACCOUNT_ENTROPY_POOL = "backup.local_restore_account_entropy_pool"
@@ -484,6 +485,13 @@ class BackupValues(store: KeyValueStore) : SignalStoreValues(store) {
var newLocalBackupProgress: LocalBackupCreationProgress by newLocalBackupProgressValue
val newLocalBackupProgressFlow: Flow<LocalBackupCreationProgress> by lazy { newLocalBackupProgressValue.toFlow() }
/**
* Progress values for local plaintext backup progress.
*/
private val newLocalPlaintextBackupProgressValue = protoValue(KEY_NEW_LOCAL_PLAINTEXT_BACKUPS_CREATION_PROGRESS, LocalBackupCreationProgress(), LocalBackupCreationProgress.ADAPTER)
var newLocalPlaintextBackupProgress: LocalBackupCreationProgress by newLocalPlaintextBackupProgressValue
val newLocalPlaintextBackupProgressFlow: Flow<LocalBackupCreationProgress> by lazy { newLocalPlaintextBackupProgressValue.toFlow() }
/**IT
* The directory URI path selected for new local backups.
*/

View File

@@ -0,0 +1,52 @@
/*
* Copyright 2024 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.signal.core.util
import com.squareup.wire.Message
import com.squareup.wire.WireEnum
import com.squareup.wire.WireField
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonArray
import kotlinx.serialization.json.JsonElement
import kotlinx.serialization.json.JsonNull
import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.JsonPrimitive
import okio.ByteString
fun Message<*, *>.toJson(): String {
return Json.encodeToString(JsonElement.serializer(), toJsonElement())
}
private fun Message<*, *>.toJsonElement(): JsonObject {
val map = mutableMapOf<String, JsonElement>()
for (field in this.javaClass.declaredFields) {
field.getAnnotation(WireField::class.java) ?: continue
field.isAccessible = true
val value = field.get(this)
map[field.name] = valueToJsonElement(value)
}
return JsonObject(map)
}
private fun valueToJsonElement(value: Any?): JsonElement {
return when (value) {
null -> JsonNull
is Message<*, *> -> value.toJsonElement()
is WireEnum -> JsonPrimitive((value as Enum<*>).name)
is ByteString -> JsonPrimitive(value.base64())
is String -> JsonPrimitive(value)
is Boolean -> JsonPrimitive(value)
is Int -> JsonPrimitive(value)
is Long -> JsonPrimitive(value)
is Float -> JsonPrimitive(value)
is Double -> JsonPrimitive(value)
is List<*> -> JsonArray(value.map { valueToJsonElement(it) })
else -> JsonPrimitive(value.toString())
}
}

View File

@@ -0,0 +1,109 @@
/*
* Copyright 2024 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.signal.core.util
import com.squareup.wire.EnumAdapter
import com.squareup.wire.FieldEncoding
import com.squareup.wire.Message
import com.squareup.wire.ProtoAdapter
import com.squareup.wire.ProtoReader
import com.squareup.wire.ProtoWriter
import com.squareup.wire.ReverseProtoWriter
import com.squareup.wire.Syntax
import com.squareup.wire.WireEnum
import com.squareup.wire.WireField
import okio.ByteString
import kotlin.jvm.JvmField
class TestMessage(
@field:WireField(tag = 1, adapter = "com.squareup.wire.ProtoAdapter#STRING")
@JvmField
val name: String = "",
@field:WireField(tag = 2, adapter = "com.squareup.wire.ProtoAdapter#INT64")
@JvmField
val id: Long = 0L,
@field:WireField(tag = 3, adapter = "com.squareup.wire.ProtoAdapter#BYTES")
@JvmField
val data: ByteString = ByteString.EMPTY,
@field:WireField(tag = 4, adapter = "org.signal.core.util.TestMessage${'$'}TestEnum#ADAPTER")
@JvmField
val status: TestEnum? = null,
@field:WireField(tag = 5, adapter = "org.signal.core.util.TestMessage${'$'}Nested#ADAPTER")
@JvmField
val nested: Nested? = null,
@field:WireField(tag = 6, adapter = "com.squareup.wire.ProtoAdapter#STRING", label = WireField.Label.REPEATED)
@JvmField
val tags: List<String> = emptyList(),
unknownFields: ByteString = ByteString.EMPTY
) : Message<TestMessage, Nothing>(ADAPTER, unknownFields) {
override fun newBuilder(): Nothing = throw UnsupportedOperationException()
enum class TestEnum(override val value: Int) : WireEnum {
FIRST(0),
SECOND(1);
companion object {
@JvmField
val ADAPTER: ProtoAdapter<TestEnum> = object : EnumAdapter<TestEnum>(TestEnum::class, Syntax.PROTO_3, FIRST) {
override fun fromValue(value: Int): TestEnum? = entries.find { it.value == value }
}
}
}
class Nested(
@field:WireField(tag = 1, adapter = "com.squareup.wire.ProtoAdapter#STRING")
@JvmField
val label: String = "",
unknownFields: ByteString = ByteString.EMPTY
) : Message<Nested, Nothing>(ADAPTER, unknownFields) {
override fun newBuilder(): Nothing = throw UnsupportedOperationException()
companion object {
@JvmField
val ADAPTER: ProtoAdapter<Nested> = object : ProtoAdapter<Nested>(
FieldEncoding.LENGTH_DELIMITED,
Nested::class,
null,
Syntax.PROTO_3,
null,
null
) {
override fun encodedSize(value: Nested): Int = 0
override fun encode(writer: ProtoWriter, value: Nested) = Unit
override fun encode(writer: ReverseProtoWriter, value: Nested) = Unit
override fun decode(reader: ProtoReader): Nested = Nested()
override fun redact(value: Nested): Nested = value
}
}
}
companion object {
@JvmField
val ADAPTER: ProtoAdapter<TestMessage> = object : ProtoAdapter<TestMessage>(
FieldEncoding.LENGTH_DELIMITED,
TestMessage::class,
null,
Syntax.PROTO_3,
null,
null
) {
override fun encodedSize(value: TestMessage): Int = 0
override fun encode(writer: ProtoWriter, value: TestMessage) = Unit
override fun encode(writer: ReverseProtoWriter, value: TestMessage) = Unit
override fun decode(reader: ProtoReader): TestMessage = TestMessage()
override fun redact(value: TestMessage): TestMessage = value
}
}
}

View File

@@ -0,0 +1,121 @@
/*
* Copyright 2024 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.signal.core.util
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonNull
import kotlinx.serialization.json.jsonArray
import kotlinx.serialization.json.jsonObject
import kotlinx.serialization.json.jsonPrimitive
import kotlinx.serialization.json.long
import okio.ByteString.Companion.encodeUtf8
import org.junit.Assert.assertEquals
import org.junit.Assert.assertFalse
import org.junit.Assert.assertTrue
import org.junit.Test
class WireProtoJsonTest {
@Test
fun `basic string and int64 fields serialize correctly`() {
val message = TestMessage(name = "alice", id = 42L)
val json = message.toJson()
val obj = Json.parseToJsonElement(json).jsonObject
assertEquals("alice", obj["name"]!!.jsonPrimitive.content)
assertEquals(42L, obj["id"]!!.jsonPrimitive.long)
}
@Test
fun `ByteString field serializes as base64`() {
val bytes = "hello".encodeUtf8()
val message = TestMessage(data = bytes)
val json = message.toJson()
val obj = Json.parseToJsonElement(json).jsonObject
assertEquals(bytes.base64(), obj["data"]!!.jsonPrimitive.content)
}
@Test
fun `enum field serializes as name`() {
val message = TestMessage(status = TestMessage.TestEnum.SECOND)
val json = message.toJson()
val obj = Json.parseToJsonElement(json).jsonObject
assertEquals("SECOND", obj["status"]!!.jsonPrimitive.content)
}
@Test
fun `nested message serializes as JSON object`() {
val message = TestMessage(nested = TestMessage.Nested(label = "inner"))
val json = message.toJson()
val obj = Json.parseToJsonElement(json).jsonObject
val nested = obj["nested"]!!.jsonObject
assertEquals("inner", nested["label"]!!.jsonPrimitive.content)
}
@Test
fun `repeated field serializes as JSON array`() {
val message = TestMessage(tags = listOf("a", "b", "c"))
val json = message.toJson()
val obj = Json.parseToJsonElement(json).jsonObject
val tags = obj["tags"]!!.jsonArray
assertEquals(3, tags.size)
assertEquals("a", tags[0].jsonPrimitive.content)
assertEquals("b", tags[1].jsonPrimitive.content)
assertEquals("c", tags[2].jsonPrimitive.content)
}
@Test
fun `default values produce sensible output`() {
val message = TestMessage()
val json = message.toJson()
val obj = Json.parseToJsonElement(json).jsonObject
assertEquals("", obj["name"]!!.jsonPrimitive.content)
assertEquals(0L, obj["id"]!!.jsonPrimitive.long)
}
@Test
fun `null optional fields serialize as null`() {
val message = TestMessage()
val json = message.toJson()
val obj = Json.parseToJsonElement(json).jsonObject
assertTrue(obj["nested"] is JsonNull)
assertTrue(obj["status"] is JsonNull)
}
@Test
fun `round trip produces valid parseable JSON`() {
val message = TestMessage(
name = "test",
id = 100L,
data = "bytes".encodeUtf8(),
status = TestMessage.TestEnum.FIRST,
nested = TestMessage.Nested(label = "deep"),
tags = listOf("x", "y")
)
val json = message.toJson()
val obj = Json.parseToJsonElement(json).jsonObject
assertEquals("test", obj["name"]!!.jsonPrimitive.content)
assertEquals(100L, obj["id"]!!.jsonPrimitive.long)
assertEquals("deep", obj["nested"]!!.jsonObject["label"]!!.jsonPrimitive.content)
assertEquals(2, obj["tags"]!!.jsonArray.size)
}
@Test
fun `unknownFields and adapter are excluded from output`() {
val message = TestMessage(name = "test")
val json = message.toJson()
assertFalse(json.contains("unknownFields"))
assertFalse(json.contains("\"adapter\""))
}
}

View File

@@ -0,0 +1,32 @@
/*
* Copyright 2024 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.signal.archive.stream
import org.signal.archive.proto.BackupInfo
import org.signal.archive.proto.Frame
import org.signal.core.util.toJson
import java.io.IOException
import java.io.OutputStream
/**
* Writes backup frames to the wrapped stream as newline-delimited JSON (JSONL).
*/
class JsonBackupWriter(private val outputStream: OutputStream) : BackupExportWriter {
@Throws(IOException::class)
override fun write(header: BackupInfo) {
outputStream.write((header.toJson() + "\n").toByteArray())
}
@Throws(IOException::class)
override fun write(frame: Frame) {
outputStream.write((frame.toJson() + "\n").toByteArray())
}
override fun close() {
outputStream.close()
}
}