Signal/Android
1
0
Fork 0
mirror of https://github.com/signalapp/Signal-Android.git synced 2026-04-29 13:16:01 +01:00
Code Issues Packages Projects Releases Wiki Activity

Add initial support for backup and restore of message and media to staging.

Browse Source 
Co-authored-by: Cody Henthorne <cody@signal.org>
This commit is contained in:
Clark
2024-04-12 11:57:34 -04:00
committed by Greyson Parrelli
parent 8617a074ad
commit 689eacd618
71 changed files with 3198 additions and 744 deletions
Download Patch File Download Diff File Expand all files Collapse all files

88
app/src/main/java/org/thoughtcrime/securesms/attachments/ArchivedAttachment.kt Normal file
View File

@@ -0,0 +1,88 @@
/*
* Copyright 2024 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.thoughtcrime.securesms.attachments
import android.net.Uri
import android.os.Parcel
import org.signal.core.util.Base64
import org.thoughtcrime.securesms.blurhash.BlurHash
import org.thoughtcrime.securesms.database.AttachmentTable
class ArchivedAttachment : Attachment {
@JvmField
val archiveCdn: Int
@JvmField
val archiveMediaName: String
@JvmField
val archiveMediaId: String
constructor(
contentType: String?,
size: Long,
cdn: Cdn,
cdnKey: ByteArray,
archiveMediaName: String,
archiveMediaId: String,
digest: ByteArray,
incrementalMac: ByteArray?,
incrementalMacChunkSize: Int?,
width: Int?,
height: Int?,
caption: String?,
blurHash: String?,
voiceNote: Boolean,
borderless: Boolean,
gif: Boolean,
quote: Boolean
) : super(
contentType = contentType ?: "",
quote = quote,
transferState = AttachmentTable.TRANSFER_NEEDS_RESTORE,
size = size,
fileName = null,
cdn = cdn,
remoteLocation = null,
remoteKey = Base64.encodeWithoutPadding(cdnKey),
remoteDigest = digest,
incrementalDigest = incrementalMac,
fastPreflightId = null,
voiceNote = voiceNote,
borderless = borderless,
videoGif = gif,
width = width ?: 0,
height = height ?: 0,
incrementalMacChunkSize = incrementalMacChunkSize ?: 0,
uploadTimestamp = 0,
caption = caption,
stickerLocator = null,
blurHash = BlurHash.parseOrNull(blurHash),
audioHash = null,
transformProperties = null
) {
this.archiveCdn = cdn.cdnNumber
this.archiveMediaName = archiveMediaName
this.archiveMediaId = archiveMediaId
}
constructor(parcel: Parcel) : super(parcel) {
archiveCdn = parcel.readInt()
archiveMediaName = parcel.readString()!!
archiveMediaId = parcel.readString()!!
}
override fun writeToParcel(dest: Parcel, flags: Int) {
super.writeToParcel(dest, flags)
dest.writeInt(archiveCdn)
dest.writeString(archiveMediaName)
dest.writeString(archiveMediaId)
}
override val uri: Uri? = null
override val publicUri: Uri? = null
}

6
app/src/main/java/org/thoughtcrime/securesms/attachments/Attachment.kt
View File

@@ -29,7 +29,7 @@ abstract class Attachment(
@JvmField
val fileName: String?,
@JvmField
val cdnNumber: Int,
val cdn: Cdn,
@JvmField
val remoteLocation: String?,
@JvmField
@@ -76,7 +76,7 @@ abstract class Attachment(
transferState = parcel.readInt(),
size = parcel.readLong(),
fileName = parcel.readString(),
cdnNumber = parcel.readInt(),
cdn = Cdn.deserialize(parcel.readInt()),
remoteLocation = parcel.readString(),
remoteKey = parcel.readString(),
remoteDigest = ParcelUtil.readByteArray(parcel),
@@ -103,7 +103,7 @@ abstract class Attachment(
dest.writeInt(transferState)
dest.writeLong(size)
dest.writeString(fileName)
dest.writeInt(cdnNumber)
dest.writeInt(cdn.serialize())
dest.writeString(remoteLocation)
dest.writeString(remoteKey)
ParcelUtil.writeByteArray(dest, remoteDigest)

4
app/src/main/java/org/thoughtcrime/securesms/attachments/AttachmentCreator.kt
View File

@@ -17,7 +17,8 @@ object AttachmentCreator : Parcelable.Creator<Attachment> {
DATABASE(DatabaseAttachment::class.java, "database"),
POINTER(PointerAttachment::class.java, "pointer"),
TOMBSTONE(TombstoneAttachment::class.java, "tombstone"),
URI(UriAttachment::class.java, "uri")
URI(UriAttachment::class.java, "uri"),
ARCHIVED(ArchivedAttachment::class.java, "archived")
}
@JvmStatic
@@ -34,6 +35,7 @@ object AttachmentCreator : Parcelable.Creator<Attachment> {
Subclass.POINTER -> PointerAttachment(source)
Subclass.TOMBSTONE -> TombstoneAttachment(source)
Subclass.URI -> UriAttachment(source)
Subclass.ARCHIVED -> ArchivedAttachment(source)
}
}

53
app/src/main/java/org/thoughtcrime/securesms/attachments/Cdn.kt Normal file
View File

@@ -0,0 +1,53 @@
/*
* Copyright 2024 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.thoughtcrime.securesms.attachments
import org.signal.core.util.IntSerializer
/**
* Attachments/media can come from and go to multiple CDN locations depending on when and where
* they were uploaded. This class represents the CDNs where attachments/media can live.
*/
enum class Cdn(private val value: Int) {
S3(-1),
CDN_0(0),
CDN_2(2),
CDN_3(3);
val cdnNumber: Int
get() {
return when (this) {
S3 -> -1
CDN_0 -> 0
CDN_2 -> 2
CDN_3 -> 3
}
}
fun serialize(): Int {
return Serializer.serialize(this)
}
companion object Serializer : IntSerializer<Cdn> {
override fun serialize(data: Cdn): Int {
return data.value
}
override fun deserialize(data: Int): Cdn {
return values().first { it.value == data }
}
fun fromCdnNumber(cdnNumber: Int): Cdn {
return when (cdnNumber) {
-1 -> S3
0 -> CDN_0
2 -> CDN_2
3 -> CDN_3
else -> throw UnsupportedOperationException()
}
}
}
}

27
app/src/main/java/org/thoughtcrime/securesms/attachments/DatabaseAttachment.kt
View File

@@ -25,6 +25,15 @@ class DatabaseAttachment : Attachment {
@JvmField
val dataHash: String?
@JvmField
val archiveCdn: Int
@JvmField
val archiveMediaName: String?
@JvmField
val archiveMediaId: String?
private val hasThumbnail: Boolean
val displayOrder: Int
@@ -37,7 +46,7 @@ class DatabaseAttachment : Attachment {
transferProgress: Int,
size: Long,
fileName: String?,
cdnNumber: Int,
cdn: Cdn,
location: String?,
key: String?,
digest: ByteArray?,
@@ -57,13 +66,16 @@ class DatabaseAttachment : Attachment {
transformProperties: TransformProperties?,
displayOrder: Int,
uploadTimestamp: Long,
dataHash: String?
dataHash: String?,
archiveCdn: Int,
archiveMediaName: String?,
archiveMediaId: String?
) : super(
contentType = contentType!!,
transferState = transferProgress,
size = size,
fileName = fileName,
cdnNumber = cdnNumber,
cdn = cdn,
remoteLocation = location,
remoteKey = key,
remoteDigest = digest,
@@ -88,6 +100,9 @@ class DatabaseAttachment : Attachment {
this.dataHash = dataHash
this.hasThumbnail = hasThumbnail
this.displayOrder = displayOrder
this.archiveCdn = archiveCdn
this.archiveMediaName = archiveMediaName
this.archiveMediaId = archiveMediaId
}
constructor(parcel: Parcel) : super(parcel) {
@@ -97,6 +112,9 @@ class DatabaseAttachment : Attachment {
hasThumbnail = ParcelUtil.readBoolean(parcel)
mmsId = parcel.readLong()
displayOrder = parcel.readInt()
archiveCdn = parcel.readInt()
archiveMediaName = parcel.readString()
archiveMediaId = parcel.readString()
}
override fun writeToParcel(dest: Parcel, flags: Int) {
@@ -107,6 +125,9 @@ class DatabaseAttachment : Attachment {
ParcelUtil.writeBoolean(dest, hasThumbnail)
dest.writeLong(mmsId)
dest.writeInt(displayOrder)
dest.writeInt(archiveCdn)
dest.writeString(archiveMediaName)
dest.writeString(archiveMediaId)
}
override val uri: Uri?

42
app/src/main/java/org/thoughtcrime/securesms/attachments/PointerAttachment.kt
View File

@@ -9,7 +9,6 @@ import org.thoughtcrime.securesms.database.AttachmentTable
import org.thoughtcrime.securesms.stickers.StickerLocator
import org.whispersystems.signalservice.api.InvalidMessageStructureException
import org.whispersystems.signalservice.api.messages.SignalServiceAttachment
import org.whispersystems.signalservice.api.messages.SignalServiceDataMessage
import org.whispersystems.signalservice.api.util.AttachmentPointerUtil
import org.whispersystems.signalservice.internal.push.DataMessage
import java.util.Optional
@@ -21,7 +20,7 @@ class PointerAttachment : Attachment {
transferState: Int,
size: Long,
fileName: String?,
cdnNumber: Int,
cdn: Cdn,
location: String,
key: String?,
digest: ByteArray?,
@@ -42,7 +41,7 @@ class PointerAttachment : Attachment {
transferState = transferState,
size = size,
fileName = fileName,
cdnNumber = cdnNumber,
cdn = cdn,
remoteLocation = location,
remoteKey = key,
remoteDigest = digest,
@@ -83,7 +82,7 @@ class PointerAttachment : Attachment {
@JvmStatic
@JvmOverloads
fun forPointer(pointer: Optional<SignalServiceAttachment>, stickerLocator: StickerLocator? = null, fastPreflightId: String? = null): Optional<Attachment> {
fun forPointer(pointer: Optional<SignalServiceAttachment>, stickerLocator: StickerLocator? = null, fastPreflightId: String? = null, transferState: Int = AttachmentTable.TRANSFER_PROGRESS_PENDING): Optional<Attachment> {
if (!pointer.isPresent || !pointer.get().isPointer) {
return Optional.empty()
}
@@ -97,10 +96,10 @@ class PointerAttachment : Attachment {
return Optional.of(
PointerAttachment(
contentType = pointer.get().contentType,
transferState = AttachmentTable.TRANSFER_PROGRESS_PENDING,
transferState = transferState,
size = pointer.get().asPointer().size.orElse(0).toLong(),
fileName = pointer.get().asPointer().fileName.orElse(null),
cdnNumber = pointer.get().asPointer().cdnNumber,
cdn = Cdn.fromCdnNumber(pointer.get().asPointer().cdnNumber),
location = pointer.get().asPointer().remoteId.toString(),
key = encodedKey,
digest = pointer.get().asPointer().digest.orElse(null),
@@ -120,35 +119,6 @@ class PointerAttachment : Attachment {
)
}
fun forPointer(pointer: SignalServiceDataMessage.Quote.QuotedAttachment): Optional<Attachment> {
val thumbnail = pointer.thumbnail
return Optional.of(
PointerAttachment(
contentType = pointer.contentType,
transferState = AttachmentTable.TRANSFER_PROGRESS_PENDING,
size = (if (thumbnail != null) thumbnail.asPointer().size.orElse(0) else 0).toLong(),
fileName = pointer.fileName,
cdnNumber = thumbnail?.asPointer()?.cdnNumber ?: 0,
location = thumbnail?.asPointer()?.remoteId?.toString() ?: "0",
key = if (thumbnail != null && thumbnail.asPointer().key != null) encodeWithPadding(thumbnail.asPointer().key) else null,
digest = thumbnail?.asPointer()?.digest?.orElse(null),
incrementalDigest = thumbnail?.asPointer()?.incrementalDigest?.orElse(null),
incrementalMacChunkSize = thumbnail?.asPointer()?.incrementalMacChunkSize ?: 0,
fastPreflightId = null,
voiceNote = false,
borderless = false,
videoGif = false,
width = thumbnail?.asPointer()?.width ?: 0,
height = thumbnail?.asPointer()?.height ?: 0,
uploadTimestamp = thumbnail?.asPointer()?.uploadTimestamp ?: 0,
caption = thumbnail?.asPointer()?.caption?.orElse(null),
stickerLocator = null,
blurHash = null
)
)
}
fun forPointer(quotedAttachment: DataMessage.Quote.QuotedAttachment): Optional<Attachment> {
val thumbnail: SignalServiceAttachment? = try {
if (quotedAttachment.thumbnail != null) {
@@ -166,7 +136,7 @@ class PointerAttachment : Attachment {
transferState = AttachmentTable.TRANSFER_PROGRESS_PENDING,
size = (if (thumbnail != null) thumbnail.asPointer().size.orElse(0) else 0).toLong(),
fileName = quotedAttachment.fileName,
cdnNumber = thumbnail?.asPointer()?.cdnNumber ?: 0,
cdn = Cdn.fromCdnNumber(thumbnail?.asPointer()?.cdnNumber ?: 0),
location = thumbnail?.asPointer()?.remoteId?.toString() ?: "0",
key = if (thumbnail != null && thumbnail.asPointer().key != null) encodeWithPadding(thumbnail.asPointer().key) else null,
digest = thumbnail?.asPointer()?.digest?.orElse(null),

41
app/src/main/java/org/thoughtcrime/securesms/attachments/TombstoneAttachment.kt
View File

@@ -2,6 +2,7 @@ package org.thoughtcrime.securesms.attachments
import android.net.Uri
import android.os.Parcel
import org.thoughtcrime.securesms.blurhash.BlurHash
import org.thoughtcrime.securesms.database.AttachmentTable
/**
@@ -17,7 +18,7 @@ class TombstoneAttachment : Attachment {
transferState = AttachmentTable.TRANSFER_PROGRESS_DONE,
size = 0,
fileName = null,
cdnNumber = 0,
cdn = Cdn.CDN_0,
remoteLocation = null,
remoteKey = null,
remoteDigest = null,
@@ -37,6 +38,44 @@ class TombstoneAttachment : Attachment {
transformProperties = null
)
constructor(
contentType: String?,
incrementalMac: ByteArray?,
incrementalMacChunkSize: Int?,
width: Int?,
height: Int?,
caption: String?,
blurHash: String?,
voiceNote: Boolean = false,
borderless: Boolean = false,
gif: Boolean = false,
quote: Boolean
) : super(
contentType = contentType ?: "",
quote = quote,
transferState = AttachmentTable.TRANSFER_PROGRESS_PERMANENT_FAILURE,
size = 0,
fileName = null,
cdn = Cdn.CDN_0,
remoteLocation = null,
remoteKey = null,
remoteDigest = null,
incrementalDigest = incrementalMac,
fastPreflightId = null,
voiceNote = voiceNote,
borderless = borderless,
videoGif = gif,
width = width ?: 0,
height = height ?: 0,
incrementalMacChunkSize = incrementalMacChunkSize ?: 0,
uploadTimestamp = 0,
caption = caption,
stickerLocator = null,
blurHash = BlurHash.parseOrNull(blurHash),
audioHash = null,
transformProperties = null
)
constructor(parcel: Parcel) : super(parcel)
override val uri: Uri? = null

2
app/src/main/java/org/thoughtcrime/securesms/attachments/UriAttachment.kt
View File

@@ -69,7 +69,7 @@ class UriAttachment : Attachment {
transferState = transferState,
size = size,
fileName = fileName,
cdnNumber = 0,
cdn = Cdn.CDN_0,
remoteLocation = null,
remoteKey = null,
remoteDigest = null,

36
app/src/main/java/org/thoughtcrime/securesms/backup/RestoreState.kt Normal file
View File

@@ -0,0 +1,36 @@
/*
* Copyright 2024 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.thoughtcrime.securesms.backup
import org.signal.core.util.LongSerializer
enum class RestoreState(val id: Int, val inProgress: Boolean) {
FAILED(-1, false),
NONE(0, false),
PENDING(1, true),
RESTORING_DB(2, true),
RESTORING_MEDIA(3, true);
companion object {
val serializer: LongSerializer<RestoreState> = Serializer()
}
class Serializer : LongSerializer<RestoreState> {
override fun serialize(data: RestoreState): Long {
return data.id.toLong()
}
override fun deserialize(data: Long): RestoreState {
return when (data.toInt()) {
FAILED.id -> FAILED
PENDING.id -> PENDING
RESTORING_DB.id -> RESTORING_DB
RESTORING_MEDIA.id -> RESTORING_MEDIA
else -> NONE
}
}
}
}

237
app/src/main/java/org/thoughtcrime/securesms/backup/v2/BackupRepository.kt
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2023 Signal Messenger, LLC
* Copyright 2024 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
@@ -14,6 +14,7 @@ import org.signal.libsignal.messagebackup.MessageBackup.ValidationResult
import org.signal.libsignal.messagebackup.MessageBackupKey
import org.signal.libsignal.protocol.ServiceId.Aci
import org.signal.libsignal.zkgroup.profiles.ProfileKey
import org.thoughtcrime.securesms.attachments.AttachmentId
import org.thoughtcrime.securesms.attachments.DatabaseAttachment
import org.thoughtcrime.securesms.backup.v2.database.ChatItemImportInserter
import org.thoughtcrime.securesms.backup.v2.database.clearAllDataForBackupRestore
@@ -37,17 +38,20 @@ import org.thoughtcrime.securesms.recipients.RecipientId
import org.whispersystems.signalservice.api.NetworkResult
import org.whispersystems.signalservice.api.archive.ArchiveGetMediaItemsResponse
import org.whispersystems.signalservice.api.archive.ArchiveMediaRequest
import org.whispersystems.signalservice.api.archive.ArchiveMediaResponse
import org.whispersystems.signalservice.api.archive.ArchiveServiceCredential
import org.whispersystems.signalservice.api.archive.BatchArchiveMediaResponse
import org.whispersystems.signalservice.api.archive.DeleteArchivedMediaRequest
import org.whispersystems.signalservice.api.archive.GetArchiveCdnCredentialsResponse
import org.whispersystems.signalservice.api.backup.BackupKey
import org.whispersystems.signalservice.api.backup.MediaName
import org.whispersystems.signalservice.api.crypto.AttachmentCipherStreamUtil
import org.whispersystems.signalservice.api.messages.SignalServiceAttachment.ProgressListener
import org.whispersystems.signalservice.api.push.ServiceId.ACI
import org.whispersystems.signalservice.api.push.ServiceId.PNI
import org.whispersystems.signalservice.internal.crypto.PaddingInputStream
import java.io.ByteArrayOutputStream
import java.io.File
import java.io.InputStream
import java.io.OutputStream
import kotlin.time.Duration.Companion.milliseconds
object BackupRepository {
@@ -55,10 +59,8 @@ object BackupRepository {
private val TAG = Log.tag(BackupRepository::class.java)
private const val VERSION = 1L
fun export(plaintext: Boolean = false): ByteArray {
fun export(outputStream: OutputStream, append: (ByteArray) -> Unit, plaintext: Boolean = false) {
val eventTimer = EventTimer()
val outputStream = ByteArrayOutputStream()
val writer: BackupExportWriter = if (plaintext) {
PlainTextBackupWriter(outputStream)
} else {
@@ -66,11 +68,11 @@ object BackupRepository {
key = SignalStore.svr().getOrCreateMasterKey().deriveBackupKey(),
aci = SignalStore.account().aci!!,
outputStream = outputStream,
append = { mac -> outputStream.write(mac) }
append = append
)
}
val exportState = ExportState(System.currentTimeMillis())
val exportState = ExportState(backupTime = System.currentTimeMillis(), allowMediaBackup = true)
writer.use {
writer.write(
@@ -110,7 +112,11 @@ object BackupRepository {
}
Log.d(TAG, "export() ${eventTimer.stop().summary}")
}
fun export(plaintext: Boolean = false): ByteArray {
val outputStream = ByteArrayOutputStream()
export(outputStream = outputStream, append = { mac -> outputStream.write(mac) }, plaintext = plaintext)
return outputStream.toByteArray()
}
@@ -124,11 +130,13 @@ object BackupRepository {
fun import(length: Long, inputStreamFactory: () -> InputStream, selfData: SelfData, plaintext: Boolean = false) {
val eventTimer = EventTimer()
val backupKey = SignalStore.svr().getOrCreateMasterKey().deriveBackupKey()
val frameReader = if (plaintext) {
PlainTextBackupReader(inputStreamFactory())
} else {
EncryptedBackupReader(
key = SignalStore.svr().getOrCreateMasterKey().deriveBackupKey(),
key = backupKey,
aci = selfData.aci,
streamLength = length,
dataStream = inputStreamFactory
@@ -160,7 +168,7 @@ object BackupRepository {
SignalDatabase.recipients.setProfileSharing(selfId, true)
eventTimer.emit("setup")
val backupState = BackupState()
val backupState = BackupState(backupKey)
val chatItemInserter: ChatItemImportInserter = ChatItemBackupProcessor.beginImport(backupState)
for (frame in frameReader) {
@@ -281,6 +289,24 @@ object BackupRepository {
.also { Log.i(TAG, "OverallResult: $it") } is NetworkResult.Success
}
fun downloadBackupFile(destination: File, listener: ProgressListener? = null): Boolean {
val api = ApplicationDependencies.getSignalServiceAccountManager().archiveApi
val backupKey = SignalStore.svr().getOrCreateMasterKey().deriveBackupKey()
return api
.triggerBackupIdReservation(backupKey)
.then { getAuthCredential() }
.then { credential ->
api.getBackupInfo(backupKey, credential)
}
.then { info -> getCdnReadCredentials().map { it.headers to info } }
.map { pair ->
val (cdnCredentials, info) = pair
val messageReceiver = ApplicationDependencies.getSignalServiceMessageReceiver()
messageReceiver.retrieveBackup(info.cdn!!, cdnCredentials, "backups/${info.backupDir}/${info.backupName}", destination, listener)
} is NetworkResult.Success
}
/**
* Returns an object with details about the remote backup state.
*/
@@ -296,7 +322,7 @@ object BackupRepository {
}
}
fun archiveMedia(attachment: DatabaseAttachment): NetworkResult<ArchiveMediaResponse> {
fun archiveMedia(attachment: DatabaseAttachment): NetworkResult<Unit> {
val api = ApplicationDependencies.getSignalServiceAccountManager().archiveApi
val backupKey = SignalStore.svr().getOrCreateMasterKey().deriveBackupKey()
@@ -304,16 +330,23 @@ object BackupRepository {
.triggerBackupIdReservation(backupKey)
.then { getAuthCredential() }
.then { credential ->
api.archiveAttachmentMedia(
backupKey = backupKey,
serviceCredential = credential,
item = attachment.toArchiveMediaRequest(backupKey)
)
val mediaName = attachment.getMediaName()
val request = attachment.toArchiveMediaRequest(mediaName, backupKey)
api
.archiveAttachmentMedia(
backupKey = backupKey,
serviceCredential = credential,
item = request
)
.map { Triple(mediaName, request.mediaId, it) }
}
.also { Log.i(TAG, "backupMediaResult: $it") }
.map { (mediaName, mediaId, response) ->
SignalDatabase.attachments.setArchiveData(attachmentId = attachment.attachmentId, archiveCdn = response.cdn, archiveMediaName = mediaName.name, archiveMediaId = mediaId)
}
.also { Log.i(TAG, "archiveMediaResult: $it") }
}
fun archiveMedia(attachments: List<DatabaseAttachment>): NetworkResult<BatchArchiveMediaResponse> {
fun archiveMedia(databaseAttachments: List<DatabaseAttachment>): NetworkResult<BatchArchiveMediaResult> {
val api = ApplicationDependencies.getSignalServiceAccountManager().archiveApi
val backupKey = SignalStore.svr().getOrCreateMasterKey().deriveBackupKey()
@@ -321,24 +354,55 @@ object BackupRepository {
.triggerBackupIdReservation(backupKey)
.then { getAuthCredential() }
.then { credential ->
api.archiveAttachmentMedia(
backupKey = backupKey,
serviceCredential = credential,
items = attachments.map { it.toArchiveMediaRequest(backupKey) }
)
val requests = mutableListOf<ArchiveMediaRequest>()
val mediaIdToAttachmentId = mutableMapOf<String, AttachmentId>()
val attachmentIdToMediaName = mutableMapOf<AttachmentId, String>()
databaseAttachments.forEach {
val mediaName = it.getMediaName()
val request = it.toArchiveMediaRequest(mediaName, backupKey)
requests += request
mediaIdToAttachmentId[request.mediaId] = it.attachmentId
attachmentIdToMediaName[it.attachmentId] = mediaName.name
}
api
.archiveAttachmentMedia(
backupKey = backupKey,
serviceCredential = credential,
items = requests
)
.map { BatchArchiveMediaResult(it, mediaIdToAttachmentId, attachmentIdToMediaName) }
}
.also { Log.i(TAG, "backupMediaResult: $it") }
.map { result ->
result
.successfulResponses
.forEach {
val attachmentId = result.mediaIdToAttachmentId(it.mediaId)
val mediaName = result.attachmentIdToMediaName(attachmentId)
SignalDatabase.attachments.setArchiveData(attachmentId = attachmentId, archiveCdn = it.cdn!!, archiveMediaName = mediaName, archiveMediaId = it.mediaId)
}
result
}
.also { Log.i(TAG, "archiveMediaResult: $it") }
}
fun deleteArchivedMedia(attachments: List<DatabaseAttachment>): NetworkResult<Unit> {
val api = ApplicationDependencies.getSignalServiceAccountManager().archiveApi
val backupKey = SignalStore.svr().getOrCreateMasterKey().deriveBackupKey()
val mediaToDelete = attachments.map {
DeleteArchivedMediaRequest.ArchivedMediaObject(
cdn = 3, // TODO [cody] store and reuse backup cdn returned from copy/move call
mediaId = backupKey.deriveMediaId(Base64.decode(it.dataHash!!)).toString()
)
val mediaToDelete = attachments
.filter { it.archiveMediaId != null }
.map {
DeleteArchivedMediaRequest.ArchivedMediaObject(
cdn = it.archiveCdn,
mediaId = it.archiveMediaId!!
)
}
if (mediaToDelete.isEmpty()) {
Log.i(TAG, "No media to delete, quick success")
return NetworkResult.Success(Unit)
}
return getAuthCredential()
@@ -349,7 +413,101 @@ object BackupRepository {
mediaToDelete = mediaToDelete
)
}
.also { Log.i(TAG, "deleteBackupMediaResult: $it") }
.map {
SignalDatabase.attachments.clearArchiveData(attachments.map { it.attachmentId })
}
.also { Log.i(TAG, "deleteArchivedMediaResult: $it") }
}
fun debugDeleteAllArchivedMedia(): NetworkResult<Unit> {
val api = ApplicationDependencies.getSignalServiceAccountManager().archiveApi
val backupKey = SignalStore.svr().getOrCreateMasterKey().deriveBackupKey()
return debugGetArchivedMediaState()
.then { archivedMedia ->
val mediaToDelete = archivedMedia
.map {
DeleteArchivedMediaRequest.ArchivedMediaObject(
cdn = it.cdn,
mediaId = it.mediaId
)
}
if (mediaToDelete.isEmpty()) {
Log.i(TAG, "No media to delete, quick success")
NetworkResult.Success(Unit)
} else {
getAuthCredential()
.then { credential ->
api.deleteArchivedMedia(
backupKey = backupKey,
serviceCredential = credential,
mediaToDelete = mediaToDelete
)
}
}
}
.map {
SignalDatabase.attachments.clearAllArchiveData()
}
.also { Log.i(TAG, "debugDeleteAllArchivedMediaResult: $it") }
}
/**
* Retrieve credentials for reading from the backup cdn.
*/
fun getCdnReadCredentials(): NetworkResult<GetArchiveCdnCredentialsResponse> {
val cached = SignalStore.backup().cdnReadCredentials
if (cached != null) {
return NetworkResult.Success(cached)
}
val api = ApplicationDependencies.getSignalServiceAccountManager().archiveApi
val backupKey = SignalStore.svr().getOrCreateMasterKey().deriveBackupKey()
return getAuthCredential()
.then { credential ->
api.getCdnReadCredentials(
backupKey = backupKey,
serviceCredential = credential
)
}
.also {
if (it is NetworkResult.Success) {
SignalStore.backup().cdnReadCredentials = it.result
}
}
.also { Log.i(TAG, "getCdnReadCredentialsResult: $it") }
}
/**
* Retrieves backupDir and mediaDir, preferring cached value if available.
*
* These will only ever change if the backup expires.
*/
fun getCdnBackupDirectories(): NetworkResult<BackupDirectories> {
val cachedBackupDirectory = SignalStore.backup().cachedBackupDirectory
val cachedBackupMediaDirectory = SignalStore.backup().cachedBackupMediaDirectory
if (cachedBackupDirectory != null && cachedBackupMediaDirectory != null) {
return NetworkResult.Success(BackupDirectories(cachedBackupDirectory, cachedBackupMediaDirectory))
}
val api = ApplicationDependencies.getSignalServiceAccountManager().archiveApi
val backupKey = SignalStore.svr().getOrCreateMasterKey().deriveBackupKey()
return getAuthCredential()
.then { credential ->
api.getBackupInfo(backupKey, credential).map {
BackupDirectories(it.backupDir!!, it.mediaDir!!)
}
}
.also {
if (it is NetworkResult.Success) {
SignalStore.backup().cachedBackupDirectory = it.result.backupDir
SignalStore.backup().cachedBackupMediaDirectory = it.result.mediaDir
}
}
}
/**
@@ -380,15 +538,20 @@ object BackupRepository {
val profileKey: ProfileKey
)
private fun DatabaseAttachment.toArchiveMediaRequest(backupKey: BackupKey): ArchiveMediaRequest {
val mediaSecrets = backupKey.deriveMediaSecrets(Base64.decode(dataHash!!))
fun DatabaseAttachment.getMediaName(): MediaName {
return MediaName.fromDigest(remoteDigest!!)
}
private fun DatabaseAttachment.toArchiveMediaRequest(mediaName: MediaName, backupKey: BackupKey): ArchiveMediaRequest {
val mediaSecrets = backupKey.deriveMediaSecrets(mediaName)
return ArchiveMediaRequest(
sourceAttachment = ArchiveMediaRequest.SourceAttachment(
cdn = cdnNumber,
cdn = cdn.cdnNumber,
key = remoteLocation!!
),
objectLength = AttachmentCipherStreamUtil.getCiphertextLength(PaddingInputStream.getPaddedSize(size)).toInt(),
mediaId = mediaSecrets.id.toString(),
mediaId = mediaSecrets.id.encode(),
hmacKey = Base64.encodeWithPadding(mediaSecrets.macKey),
encryptionKey = Base64.encodeWithPadding(mediaSecrets.cipherKey),
iv = Base64.encodeWithPadding(mediaSecrets.iv)
@@ -396,12 +559,14 @@ object BackupRepository {
}
}
class ExportState(val backupTime: Long) {
data class BackupDirectories(val backupDir: String, val mediaDir: String)
class ExportState(val backupTime: Long, val allowMediaBackup: Boolean) {
val recipientIds = HashSet<Long>()
val threadIds = HashSet<Long>()
}
class BackupState {
class BackupState(val backupKey: BackupKey) {
val backupToLocalRecipientId = HashMap<Long, RecipientId>()
val chatIdToLocalThreadId = HashMap<Long, Long>()
val chatIdToLocalRecipientId = HashMap<Long, RecipientId>()

46
app/src/main/java/org/thoughtcrime/securesms/backup/v2/BackupRestoreManager.kt Normal file
View File

@@ -0,0 +1,46 @@
/*
* Copyright 2024 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.thoughtcrime.securesms.backup.v2
import org.signal.core.util.concurrent.SignalExecutors
import org.thoughtcrime.securesms.attachments.AttachmentId
import org.thoughtcrime.securesms.attachments.DatabaseAttachment
import org.thoughtcrime.securesms.database.AttachmentTable
import org.thoughtcrime.securesms.database.model.MessageRecord
import org.thoughtcrime.securesms.database.model.MmsMessageRecord
import org.thoughtcrime.securesms.jobs.RestoreAttachmentJob
/**
* Responsible for managing logic around restore prioritization
*/
object BackupRestoreManager {
private val reprioritizedAttachments: HashSet<AttachmentId> = HashSet()
/**
* Raise priority of all attachments for the included message records.
*
* This is so we can make certain attachments get downloaded more quickly
*/
fun prioritizeAttachmentsIfNeeded(messageRecords: List<MessageRecord>) {
SignalExecutors.BOUNDED.execute {
synchronized(this) {
val restoringAttachments: List<AttachmentId> = messageRecords
.mapNotNull { (it as? MmsMessageRecord?)?.slideDeck?.slides }
.flatten()
.mapNotNull { it.asAttachment() as? DatabaseAttachment }
.filter { it.transferState == AttachmentTable.TRANSFER_RESTORE_IN_PROGRESS && !reprioritizedAttachments.contains(it.attachmentId) }
.map { it.attachmentId }
reprioritizedAttachments += restoringAttachments
if (restoringAttachments.isNotEmpty()) {
RestoreAttachmentJob.modifyPriorities(restoringAttachments.toSet(), 1)
}
}
}
}
}

39
app/src/main/java/org/thoughtcrime/securesms/backup/v2/BatchArchiveMediaResult.kt Normal file
View File

@@ -0,0 +1,39 @@
/*
* Copyright 2024 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.thoughtcrime.securesms.backup.v2
import org.thoughtcrime.securesms.attachments.AttachmentId
import org.whispersystems.signalservice.api.archive.BatchArchiveMediaResponse
/**
* Result of attempting to batch copy multiple attachments at once with helpers for
* processing the collection of mini-responses.
*/
data class BatchArchiveMediaResult(
private val response: BatchArchiveMediaResponse,
private val mediaIdToAttachmentId: Map<String, AttachmentId>,
private val attachmentIdToMediaName: Map<AttachmentId, String>
) {
val successfulResponses: Sequence<BatchArchiveMediaResponse.BatchArchiveMediaItemResponse>
get() = response
.responses
.asSequence()
.filter { it.status == 200 }
val sourceNotFoundResponses: Sequence<BatchArchiveMediaResponse.BatchArchiveMediaItemResponse>
get() = response
.responses
.asSequence()
.filter { it.status == 410 }
fun mediaIdToAttachmentId(mediaId: String): AttachmentId {
return mediaIdToAttachmentId[mediaId]!!
}
fun attachmentIdToMediaName(attachmentId: AttachmentId): String {
return attachmentIdToMediaName[attachmentId]!!
}
}

62
app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/ChatItemExportIterator.kt
View File

@@ -17,7 +17,9 @@ import org.signal.core.util.requireBoolean
import org.signal.core.util.requireInt
import org.signal.core.util.requireLong
import org.signal.core.util.requireString
import org.thoughtcrime.securesms.attachments.Cdn
import org.thoughtcrime.securesms.attachments.DatabaseAttachment
import org.thoughtcrime.securesms.backup.v2.BackupRepository.getMediaName
import org.thoughtcrime.securesms.backup.v2.proto.CallChatUpdate
import org.thoughtcrime.securesms.backup.v2.proto.ChatItem
import org.thoughtcrime.securesms.backup.v2.proto.ChatUpdateMessage
@@ -36,6 +38,7 @@ import org.thoughtcrime.securesms.backup.v2.proto.SimpleChatUpdate
import org.thoughtcrime.securesms.backup.v2.proto.StandardMessage
import org.thoughtcrime.securesms.backup.v2.proto.Text
import org.thoughtcrime.securesms.backup.v2.proto.ThreadMergeChatUpdate
import org.thoughtcrime.securesms.database.AttachmentTable
import org.thoughtcrime.securesms.database.GroupReceiptTable
import org.thoughtcrime.securesms.database.MessageTable
import org.thoughtcrime.securesms.database.MessageTypes
@@ -73,7 +76,7 @@ import org.thoughtcrime.securesms.backup.v2.proto.BodyRange as BackupBodyRange
*
* All of this complexity is hidden from the user -- they just get a normal iterator interface.
*/
class ChatItemExportIterator(private val cursor: Cursor, private val batchSize: Int) : Iterator<ChatItem>, Closeable {
class ChatItemExportIterator(private val cursor: Cursor, private val batchSize: Int, private val archiveMedia: Boolean) : Iterator<ChatItem>, Closeable {
companion object {
private val TAG = Log.tag(ChatItemExportIterator::class.java)
@@ -139,6 +142,7 @@ class ChatItemExportIterator(private val cursor: Cursor, private val batchSize:
builder.expiresInMs = null
}
MessageTypes.isProfileChange(record.type) -> {
if (record.body == null) continue
builder.updateMessage = ChatUpdateMessage(
profileChange = try {
val decoded: ByteArray = Base64.decode(record.body!!)
@@ -354,24 +358,46 @@ class ChatItemExportIterator(private val cursor: Cursor, private val batchSize:
}
private fun DatabaseAttachment.toBackupAttachment(): MessageAttachment {
val builder = FilePointer.Builder()
builder.contentType = contentType
builder.incrementalMac = incrementalDigest?.toByteString()
builder.incrementalMacChunkSize = incrementalMacChunkSize
builder.fileName = fileName
builder.width = width
builder.height = height
builder.caption = caption
builder.blurHash = blurHash?.hash
if (remoteKey.isNullOrBlank() || remoteDigest == null || size == 0L) {
builder.invalidAttachmentLocator = FilePointer.InvalidAttachmentLocator()
} else {
if (archiveMedia) {
builder.backupLocator = FilePointer.BackupLocator(
mediaName = archiveMediaName ?: this.getMediaName().toString(),
cdnNumber = if (archiveMediaName != null) archiveCdn else Cdn.CDN_3.cdnNumber, // TODO (clark): Update when new proto with optional cdn is landed
key = decode(remoteKey).toByteString(),
size = this.size.toInt(),
digest = remoteDigest.toByteString()
)
} else {
if (remoteLocation.isNullOrBlank()) {
builder.invalidAttachmentLocator = FilePointer.InvalidAttachmentLocator()
} else {
builder.attachmentLocator = FilePointer.AttachmentLocator(
cdnKey = this.remoteLocation,
cdnNumber = this.cdn.cdnNumber,
uploadTimestamp = this.uploadTimestamp,
key = decode(remoteKey).toByteString(),
size = this.size.toInt(),
digest = remoteDigest.toByteString()
)
}
}
}
return MessageAttachment(
pointer = FilePointer(
attachmentLocator = FilePointer.AttachmentLocator(
cdnKey = this.remoteLocation ?: "",
cdnNumber = this.cdnNumber,
uploadTimestamp = this.uploadTimestamp
),
key = if (remoteKey != null) decode(remoteKey).toByteString() else null,
contentType = this.contentType,
size = this.size.toInt(),
incrementalMac = this.incrementalDigest?.toByteString(),
incrementalMacChunkSize = this.incrementalMacChunkSize,
fileName = this.fileName,
width = this.width,
height = this.height,
caption = this.caption,
blurHash = this.blurHash?.hash
)
pointer = builder.build(),
wasDownloaded = this.transferState == AttachmentTable.TRANSFER_PROGRESS_DONE || this.transferState == AttachmentTable.TRANSFER_NEEDS_RESTORE,
flag = if (voiceNote) MessageAttachment.Flag.VOICE_MESSAGE else if (videoGif) MessageAttachment.Flag.GIF else if (borderless) MessageAttachment.Flag.BORDERLESS else MessageAttachment.Flag.NONE
)
}

54
app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/ChatItemImportInserter.kt
View File

@@ -13,8 +13,11 @@ import org.signal.core.util.logging.Log
import org.signal.core.util.orNull
import org.signal.core.util.requireLong
import org.signal.core.util.toInt
import org.thoughtcrime.securesms.attachments.ArchivedAttachment
import org.thoughtcrime.securesms.attachments.Attachment
import org.thoughtcrime.securesms.attachments.Cdn
import org.thoughtcrime.securesms.attachments.PointerAttachment
import org.thoughtcrime.securesms.attachments.TombstoneAttachment
import org.thoughtcrime.securesms.backup.v2.BackupState
import org.thoughtcrime.securesms.backup.v2.proto.BodyRange
import org.thoughtcrime.securesms.backup.v2.proto.ChatItem
@@ -26,6 +29,7 @@ import org.thoughtcrime.securesms.backup.v2.proto.Reaction
import org.thoughtcrime.securesms.backup.v2.proto.SendStatus
import org.thoughtcrime.securesms.backup.v2.proto.SimpleChatUpdate
import org.thoughtcrime.securesms.backup.v2.proto.StandardMessage
import org.thoughtcrime.securesms.database.AttachmentTable
import org.thoughtcrime.securesms.database.CallTable
import org.thoughtcrime.securesms.database.GroupReceiptTable
import org.thoughtcrime.securesms.database.MessageTable
@@ -48,11 +52,12 @@ import org.thoughtcrime.securesms.mms.QuoteModel
import org.thoughtcrime.securesms.recipients.Recipient
import org.thoughtcrime.securesms.recipients.RecipientId
import org.thoughtcrime.securesms.util.JsonUtils
import org.whispersystems.signalservice.api.backup.MediaName
import org.whispersystems.signalservice.api.messages.SignalServiceAttachmentPointer
import org.whispersystems.signalservice.api.messages.SignalServiceAttachmentRemoteId
import org.whispersystems.signalservice.api.messages.SignalServiceDataMessage
import org.whispersystems.signalservice.api.push.ServiceId
import org.whispersystems.signalservice.api.util.UuidUtil
import org.whispersystems.signalservice.internal.push.DataMessage
import java.util.Optional
/**
@@ -570,12 +575,12 @@ class ChatItemImportInserter(
pointer.attachmentLocator.cdnNumber,
SignalServiceAttachmentRemoteId.from(pointer.attachmentLocator.cdnKey),
contentType,
pointer.key?.toByteArray(),
Optional.ofNullable(pointer.size),
pointer.attachmentLocator.key.toByteArray(),
Optional.ofNullable(pointer.attachmentLocator.size),
Optional.empty(),
pointer.width ?: 0,
pointer.height ?: 0,
Optional.empty(),
Optional.ofNullable(pointer.attachmentLocator.digest.toByteArray()),
Optional.ofNullable(pointer.incrementalMac?.toByteArray()),
pointer.incrementalMacChunkSize ?: 0,
Optional.ofNullable(fileName),
@@ -586,14 +591,51 @@ class ChatItemImportInserter(
Optional.ofNullable(pointer.blurHash),
pointer.attachmentLocator.uploadTimestamp
)
return PointerAttachment.forPointer(Optional.of(signalAttachmentPointer)).orNull()
return PointerAttachment.forPointer(
pointer = Optional.of(signalAttachmentPointer),
transferState = if (wasDownloaded) AttachmentTable.TRANSFER_NEEDS_RESTORE else AttachmentTable.TRANSFER_PROGRESS_PENDING
).orNull()
} else if (pointer.invalidAttachmentLocator != null) {
return TombstoneAttachment(
contentType = contentType,
incrementalMac = pointer.incrementalMac?.toByteArray(),
incrementalMacChunkSize = pointer.incrementalMacChunkSize,
width = pointer.width,
height = pointer.height,
caption = pointer.caption,
blurHash = pointer.blurHash,
voiceNote = flag == MessageAttachment.Flag.VOICE_MESSAGE,
borderless = flag == MessageAttachment.Flag.BORDERLESS,
gif = flag == MessageAttachment.Flag.GIF,
quote = false
)
} else if (pointer.backupLocator != null) {
return ArchivedAttachment(
contentType = contentType,
size = pointer.backupLocator.size.toLong(),
cdn = Cdn.fromCdnNumber(pointer.backupLocator.cdnNumber),
cdnKey = pointer.backupLocator.key.toByteArray(),
archiveMediaName = pointer.backupLocator.mediaName,
archiveMediaId = backupState.backupKey.deriveMediaId(MediaName(pointer.backupLocator.mediaName)).encode(),
digest = pointer.backupLocator.digest.toByteArray(),
incrementalMac = pointer.incrementalMac?.toByteArray(),
incrementalMacChunkSize = pointer.incrementalMacChunkSize,
width = pointer.width,
height = pointer.height,
caption = pointer.caption,
blurHash = pointer.blurHash,
voiceNote = flag == MessageAttachment.Flag.VOICE_MESSAGE,
borderless = flag == MessageAttachment.Flag.BORDERLESS,
gif = flag == MessageAttachment.Flag.GIF,
quote = false
)
}
return null
}
private fun Quote.QuotedAttachment.toLocalAttachment(): Attachment? {
return thumbnail?.toLocalAttachment(this.contentType, this.fileName)
?: if (this.contentType == null) null else PointerAttachment.forPointer(SignalServiceDataMessage.Quote.QuotedAttachment(contentType = this.contentType!!, fileName = this.fileName, thumbnail = null)).orNull()
?: if (this.contentType == null) null else PointerAttachment.forPointer(quotedAttachment = DataMessage.Quote.QuotedAttachment(contentType = this.contentType, fileName = this.fileName, thumbnail = null)).orNull()
}
private class MessageInsert(val contentValues: ContentValues, val followUp: ((Long) -> Unit)?)

4
app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/MessageTableBackupExtensions.kt
View File

@@ -16,7 +16,7 @@ import java.util.concurrent.TimeUnit
private val TAG = Log.tag(MessageTable::class.java)
private const val BASE_TYPE = "base_type"
fun MessageTable.getMessagesForBackup(backupTime: Long): ChatItemExportIterator {
fun MessageTable.getMessagesForBackup(backupTime: Long, archiveMedia: Boolean): ChatItemExportIterator {
val cursor = readableDatabase
.select(
MessageTable.ID,
@@ -64,7 +64,7 @@ fun MessageTable.getMessagesForBackup(backupTime: Long): ChatItemExportIterator
.orderBy("${MessageTable.DATE_RECEIVED} ASC")
.run()
return ChatItemExportIterator(cursor, 100)
return ChatItemExportIterator(cursor, 100, archiveMedia)
}
fun MessageTable.createChatItemInserter(backupState: BackupState): ChatItemImportInserter {

2
app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/ChatItemBackupProcessor.kt
View File

@@ -19,7 +19,7 @@ object ChatItemBackupProcessor {
val TAG = Log.tag(ChatItemBackupProcessor::class.java)
fun export(exportState: ExportState, emitter: BackupFrameEmitter) {
SignalDatabase.messages.getMessagesForBackup(exportState.backupTime).use { chatItems ->
SignalDatabase.messages.getMessagesForBackup(exportState.backupTime, exportState.allowMediaBackup).use { chatItems ->
for (chatItem in chatItems) {
if (exportState.threadIds.contains(chatItem.chatId)) {
emitter.emit(Frame(chatItem = chatItem))

21
app/src/main/java/org/thoughtcrime/securesms/backup/v2/ui/MessageBackupsTestRestoreActivity.kt
View File

@@ -26,6 +26,7 @@ import androidx.compose.material3.Surface
import androidx.compose.material3.Switch
import androidx.compose.material3.Text
import androidx.compose.runtime.Composable
import androidx.compose.runtime.SideEffect
import androidx.compose.runtime.getValue
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
@@ -80,6 +81,15 @@ class MessageBackupsTestRestoreActivity : BaseActivity() {
.fillMaxSize()
.padding(16.dp)
) {
Buttons.LargePrimary(
onClick = this@MessageBackupsTestRestoreActivity::restoreFromServer,
enabled = !state.importState.inProgress
) {
Text("Restore")
}
Spacer(modifier = Modifier.height(8.dp))
Row(
verticalAlignment = Alignment.CenterVertically
) {
@@ -120,9 +130,20 @@ class MessageBackupsTestRestoreActivity : BaseActivity() {
}
}
}
if (state.importState == MessageBackupsTestRestoreViewModel.ImportState.RESTORED) {
SideEffect {
RegistrationUtil.maybeMarkRegistrationComplete()
ApplicationDependencies.getJobManager().add(ProfileUploadJob())
startActivity(MainActivity.clearTop(this))
}
}
}
}
private fun restoreFromServer() {
viewModel.restore()
}
private fun continueRegistration() {
if (Recipient.self().profileName.isEmpty || !AvatarHelper.hasAvatar(this, Recipient.self().id)) {
val main = MainActivity.clearTop(this)

19
app/src/main/java/org/thoughtcrime/securesms/backup/v2/ui/MessageBackupsTestRestoreViewModel.kt
View File

@@ -15,8 +15,12 @@ import io.reactivex.rxjava3.disposables.CompositeDisposable
import io.reactivex.rxjava3.kotlin.plusAssign
import io.reactivex.rxjava3.kotlin.subscribeBy
import io.reactivex.rxjava3.schedulers.Schedulers
import org.signal.core.util.orNull
import org.signal.libsignal.zkgroup.profiles.ProfileKey
import org.thoughtcrime.securesms.backup.v2.BackupRepository
import org.thoughtcrime.securesms.dependencies.ApplicationDependencies
import org.thoughtcrime.securesms.jobmanager.JobTracker
import org.thoughtcrime.securesms.jobs.BackupRestoreJob
import org.thoughtcrime.securesms.recipients.Recipient
import java.io.InputStream
@@ -40,6 +44,19 @@ class MessageBackupsTestRestoreViewModel : ViewModel() {
}
}
fun restore() {
_state.value = _state.value.copy(importState = ImportState.IN_PROGRESS)
disposables += Single.fromCallable {
val jobState = ApplicationDependencies.getJobManager().runSynchronously(BackupRestoreJob(), 120_000)
jobState.orNull() == JobTracker.JobState.SUCCESS
}
.subscribeOn(Schedulers.io())
.observeOn(AndroidSchedulers.mainThread())
.subscribeBy {
_state.value = _state.value.copy(importState = ImportState.RESTORED)
}
}
fun onPlaintextToggled() {
_state.value = _state.value.copy(plaintext = !_state.value.plaintext)
}
@@ -54,6 +71,6 @@ class MessageBackupsTestRestoreViewModel : ViewModel() {
)
enum class ImportState(val inProgress: Boolean = false) {
NONE, IN_PROGRESS(true)
NONE, IN_PROGRESS(true), RESTORED
}
}

170
app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/backup/InternalBackupPlaygroundFragment.kt
View File

@@ -29,6 +29,11 @@ import androidx.compose.foundation.shape.RoundedCornerShape
import androidx.compose.material3.Button
import androidx.compose.material3.Checkbox
import androidx.compose.material3.CircularProgressIndicator
import androidx.compose.material3.DropdownMenu
import androidx.compose.material3.DropdownMenuItem
import androidx.compose.material3.ExperimentalMaterial3Api
import androidx.compose.material3.Icon
import androidx.compose.material3.IconButton
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.Scaffold
import androidx.compose.material3.SnackbarHostState
@@ -37,6 +42,8 @@ import androidx.compose.material3.Switch
import androidx.compose.material3.Tab
import androidx.compose.material3.TabRow
import androidx.compose.material3.Text
import androidx.compose.material3.TextButton
import androidx.compose.material3.TopAppBar
import androidx.compose.runtime.Composable
import androidx.compose.runtime.LaunchedEffect
import androidx.compose.runtime.getValue
@@ -46,10 +53,12 @@ import androidx.compose.runtime.remember
import androidx.compose.runtime.setValue
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.res.painterResource
import androidx.compose.ui.text.style.TextAlign
import androidx.compose.ui.tooling.preview.Preview
import androidx.compose.ui.unit.dp
import androidx.fragment.app.viewModels
import androidx.navigation.fragment.findNavController
import org.signal.core.ui.Buttons
import org.signal.core.ui.Dividers
import org.signal.core.ui.Snackbars
@@ -57,10 +66,13 @@ import org.signal.core.ui.theme.SignalTheme
import org.signal.core.util.bytes
import org.signal.core.util.getLength
import org.signal.core.util.roundedString
import org.thoughtcrime.securesms.R
import org.thoughtcrime.securesms.attachments.AttachmentId
import org.thoughtcrime.securesms.components.settings.app.internal.backup.InternalBackupPlaygroundViewModel.BackupState
import org.thoughtcrime.securesms.components.settings.app.internal.backup.InternalBackupPlaygroundViewModel.BackupUploadState
import org.thoughtcrime.securesms.components.settings.app.internal.backup.InternalBackupPlaygroundViewModel.ScreenState
import org.thoughtcrime.securesms.compose.ComposeFragment
import org.thoughtcrime.securesms.keyvalue.SignalStore
class InternalBackupPlaygroundFragment : ComposeFragment() {
@@ -114,6 +126,8 @@ class InternalBackupPlaygroundFragment : ComposeFragment() {
}
Tabs(
onBack = { findNavController().popBackStack() },
onDeleteAllArchivedMedia = { viewModel.deleteAllArchivedMedia() },
mainContent = {
Screen(
state = state,
@@ -149,25 +163,32 @@ class InternalBackupPlaygroundFragment : ComposeFragment() {
}
validateFileLauncher.launch(intent)
}
},
onTriggerBackupJobClicked = { viewModel.triggerBackupJob() },
onRestoreFromRemoteClicked = { viewModel.restoreFromRemote() }
)
},
mediaContent = { snackbarHostState ->
MediaList(
enabled = SignalStore.backup().canReadWriteToArchiveCdn,
state = mediaState,
snackbarHostState = snackbarHostState,
backupAttachmentMedia = { viewModel.backupAttachmentMedia(it) },
deleteBackupAttachmentMedia = { viewModel.deleteBackupAttachmentMedia(it) },
batchBackupAttachmentMedia = { viewModel.backupAttachmentMedia(it) },
batchDeleteBackupAttachmentMedia = { viewModel.deleteBackupAttachmentMedia(it) }
archiveAttachmentMedia = { viewModel.archiveAttachmentMedia(it) },
deleteArchivedMedia = { viewModel.deleteArchivedMedia(it) },
batchArchiveAttachmentMedia = { viewModel.archiveAttachmentMedia(it) },
batchDeleteBackupAttachmentMedia = { viewModel.deleteArchivedMedia(it) },
restoreArchivedMedia = { viewModel.restoreArchivedMedia(it) }
)
}
)
}
}
@OptIn(ExperimentalMaterial3Api::class)
@Composable
fun Tabs(
onBack: () -> Unit,
onDeleteAllArchivedMedia: () -> Unit,
mainContent: @Composable () -> Unit,
mediaContent: @Composable (snackbarHostState: SnackbarHostState) -> Unit
) {
@@ -179,13 +200,36 @@ fun Tabs(
Scaffold(
snackbarHost = { Snackbars.Host(snackbarHostState) },
topBar = {
TabRow(selectedTabIndex = tabIndex) {
tabs.forEachIndexed { index, tab ->
Tab(
text = { Text(tab) },
selected = index == tabIndex,
onClick = { tabIndex = index }
)
Column {
TopAppBar(
title = {
Text("Backup Playground")
},
navigationIcon = {
IconButton(onClick = onBack) {
Icon(
painter = painterResource(R.drawable.symbol_arrow_left_24),
tint = MaterialTheme.colorScheme.onSurface,
contentDescription = null
)
}
},
actions = {
if (tabIndex == 1 && SignalStore.backup().canReadWriteToArchiveCdn) {
TextButton(onClick = onDeleteAllArchivedMedia) {
Text(text = "Delete All")
}
}
}
)
TabRow(selectedTabIndex = tabIndex) {
tabs.forEachIndexed { index, tab ->
Tab(
text = { Text(tab) },
selected = index == tabIndex,
onClick = { tabIndex = index }
)
}
}
}
}
@@ -209,7 +253,9 @@ fun Screen(
onSaveToDiskClicked: () -> Unit = {},
onValidateFileClicked: () -> Unit = {},
onUploadToRemoteClicked: () -> Unit = {},
onCheckRemoteBackupStateClicked: () -> Unit = {}
onCheckRemoteBackupStateClicked: () -> Unit = {},
onTriggerBackupJobClicked: () -> Unit = {},
onRestoreFromRemoteClicked: () -> Unit = {}
) {
Surface {
Column(
@@ -239,6 +285,13 @@ fun Screen(
Text("Export")
}
Buttons.LargePrimary(
onClick = onTriggerBackupJobClicked,
enabled = !state.backupState.inProgress
) {
Text("Trigger Backup Job")
}
Dividers.Default()
Buttons.LargeTonal(
@@ -280,6 +333,10 @@ fun Screen(
}
}
BackupState.BACKUP_JOB_DONE -> {
StateLabel("Backup complete and uploaded")
}
BackupState.IMPORT_IN_PROGRESS -> {
StateLabel("Import in progress...")
}
@@ -324,6 +381,10 @@ fun Screen(
Spacer(modifier = Modifier.height(8.dp))
Buttons.LargePrimary(onClick = onRestoreFromRemoteClicked) {
Text("Restore from remote")
}
when (state.uploadState) {
BackupUploadState.NONE -> {
StateLabel("")
@@ -357,13 +418,24 @@ private fun StateLabel(text: String) {
@OptIn(ExperimentalFoundationApi::class)
@Composable
fun MediaList(
enabled: Boolean,
state: InternalBackupPlaygroundViewModel.MediaState,
snackbarHostState: SnackbarHostState,
backupAttachmentMedia: (InternalBackupPlaygroundViewModel.BackupAttachment) -> Unit,
deleteBackupAttachmentMedia: (InternalBackupPlaygroundViewModel.BackupAttachment) -> Unit,
batchBackupAttachmentMedia: (Set<String>) -> Unit,
batchDeleteBackupAttachmentMedia: (Set<String>) -> Unit
archiveAttachmentMedia: (InternalBackupPlaygroundViewModel.BackupAttachment) -> Unit,
deleteArchivedMedia: (InternalBackupPlaygroundViewModel.BackupAttachment) -> Unit,
batchArchiveAttachmentMedia: (Set<AttachmentId>) -> Unit,
batchDeleteBackupAttachmentMedia: (Set<AttachmentId>) -> Unit,
restoreArchivedMedia: (InternalBackupPlaygroundViewModel.BackupAttachment) -> Unit
) {
if (!enabled) {
Text(
text = "You do not have read/write to archive cdn enabled via SignalStore.backup()",
modifier = Modifier
.padding(16.dp)
)
return
}
LaunchedEffect(state.error?.id) {
state.error?.let {
snackbarHostState.showSnackbar(it.errorText)
@@ -384,51 +456,88 @@ fun MediaList(
.combinedClickable(
onClick = {
if (selectionState.selecting) {
selectionState = selectionState.copy(selected = if (selectionState.selected.contains(attachment.mediaId)) selectionState.selected - attachment.mediaId else selectionState.selected + attachment.mediaId)
selectionState = selectionState.copy(selected = if (selectionState.selected.contains(attachment.id)) selectionState.selected - attachment.id else selectionState.selected + attachment.id)
}
},
onLongClick = {
selectionState = if (selectionState.selecting) MediaMultiSelectState() else MediaMultiSelectState(selecting = true, selected = setOf(attachment.mediaId))
selectionState = if (selectionState.selecting) MediaMultiSelectState() else MediaMultiSelectState(selecting = true, selected = setOf(attachment.id))
}
)
.padding(horizontal = 16.dp, vertical = 8.dp)
) {
if (selectionState.selecting) {
Checkbox(
checked = selectionState.selected.contains(attachment.mediaId),
checked = selectionState.selected.contains(attachment.id),
onCheckedChange = { selected ->
selectionState = selectionState.copy(selected = if (selected) selectionState.selected + attachment.mediaId else selectionState.selected - attachment.mediaId)
selectionState = selectionState.copy(selected = if (selected) selectionState.selected + attachment.id else selectionState.selected - attachment.id)
}
)
}
Column(modifier = Modifier.weight(1f, true)) {
Text(text = "Attachment ${attachment.title}")
Text(text = attachment.title)
Text(text = "State: ${attachment.state}")
}
if (attachment.state == InternalBackupPlaygroundViewModel.BackupAttachment.State.INIT ||
attachment.state == InternalBackupPlaygroundViewModel.BackupAttachment.State.IN_PROGRESS
) {
if (attachment.state == InternalBackupPlaygroundViewModel.BackupAttachment.State.IN_PROGRESS) {
CircularProgressIndicator()
} else {
Button(
enabled = !selectionState.selecting,
onClick = {
when (attachment.state) {
InternalBackupPlaygroundViewModel.BackupAttachment.State.LOCAL_ONLY -> backupAttachmentMedia(attachment)
InternalBackupPlaygroundViewModel.BackupAttachment.State.UPLOADED -> deleteBackupAttachmentMedia(attachment)
InternalBackupPlaygroundViewModel.BackupAttachment.State.ATTACHMENT_CDN,
InternalBackupPlaygroundViewModel.BackupAttachment.State.LOCAL_ONLY -> archiveAttachmentMedia(attachment)
InternalBackupPlaygroundViewModel.BackupAttachment.State.UPLOADED_UNDOWNLOADED,
InternalBackupPlaygroundViewModel.BackupAttachment.State.UPLOADED_FINAL -> selectionState = selectionState.copy(expandedOption = attachment.dbAttachment.attachmentId)
else -> throw AssertionError("Unsupported state: ${attachment.state}")
}
}
) {
Text(
text = when (attachment.state) {
InternalBackupPlaygroundViewModel.BackupAttachment.State.ATTACHMENT_CDN,
InternalBackupPlaygroundViewModel.BackupAttachment.State.LOCAL_ONLY -> "Backup"
InternalBackupPlaygroundViewModel.BackupAttachment.State.UPLOADED -> "Remote Delete"
InternalBackupPlaygroundViewModel.BackupAttachment.State.UPLOADED_UNDOWNLOADED,
InternalBackupPlaygroundViewModel.BackupAttachment.State.UPLOADED_FINAL -> "Options..."
else -> throw AssertionError("Unsupported state: ${attachment.state}")
}
)
DropdownMenu(
expanded = attachment.dbAttachment.attachmentId == selectionState.expandedOption,
onDismissRequest = { selectionState = selectionState.copy(expandedOption = null) }
) {
DropdownMenuItem(
text = { Text("Remote Delete") },
onClick = {
selectionState = selectionState.copy(expandedOption = null)
deleteArchivedMedia(attachment)
}
)
DropdownMenuItem(
text = { Text("Pseudo Restore") },
onClick = {
selectionState = selectionState.copy(expandedOption = null)
restoreArchivedMedia(attachment)
}
)
if (attachment.dbAttachment.dataHash != null && attachment.state == InternalBackupPlaygroundViewModel.BackupAttachment.State.UPLOADED_UNDOWNLOADED) {
DropdownMenuItem(
text = { Text("Re-copy with hash") },
onClick = {
selectionState = selectionState.copy(expandedOption = null)
archiveAttachmentMedia(attachment)
}
)
}
}
}
}
}
@@ -451,7 +560,7 @@ fun MediaList(
Text("Cancel")
}
Button(onClick = {
batchBackupAttachmentMedia(selectionState.selected)
batchArchiveAttachmentMedia(selectionState.selected)
selectionState = MediaMultiSelectState()
}) {
Text("Backup")
@@ -469,7 +578,8 @@ fun MediaList(
private data class MediaMultiSelectState(
val selecting: Boolean = false,
val selected: Set<String> = emptySet()
val selected: Set<AttachmentId> = emptySet(),
val expandedOption: AttachmentId? = null
)
@Preview(name = "Light Theme", group = "screen", uiMode = Configuration.UI_MODE_NIGHT_NO)

273
app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/backup/InternalBackupPlaygroundViewModel.kt
View File

@@ -10,30 +10,38 @@ import androidx.compose.runtime.State
import androidx.compose.runtime.mutableStateOf
import androidx.lifecycle.ViewModel
import io.reactivex.rxjava3.android.schedulers.AndroidSchedulers
import io.reactivex.rxjava3.core.Completable
import io.reactivex.rxjava3.core.Single
import io.reactivex.rxjava3.disposables.CompositeDisposable
import io.reactivex.rxjava3.kotlin.plusAssign
import io.reactivex.rxjava3.kotlin.subscribeBy
import io.reactivex.rxjava3.schedulers.Schedulers
import org.signal.core.util.Base64
import org.signal.libsignal.zkgroup.profiles.ProfileKey
import org.thoughtcrime.securesms.attachments.AttachmentId
import org.thoughtcrime.securesms.attachments.DatabaseAttachment
import org.thoughtcrime.securesms.backup.v2.BackupMetadata
import org.thoughtcrime.securesms.backup.v2.BackupRepository
import org.thoughtcrime.securesms.database.MessageType
import org.thoughtcrime.securesms.database.SignalDatabase
import org.thoughtcrime.securesms.dependencies.ApplicationDependencies
import org.thoughtcrime.securesms.jobs.ArchiveAttachmentJob
import org.thoughtcrime.securesms.jobs.AttachmentDownloadJob
import org.thoughtcrime.securesms.jobs.AttachmentUploadJob
import org.thoughtcrime.securesms.jobs.BackupMessagesJob
import org.thoughtcrime.securesms.jobs.BackupRestoreJob
import org.thoughtcrime.securesms.jobs.BackupRestoreMediaJob
import org.thoughtcrime.securesms.keyvalue.SignalStore
import org.thoughtcrime.securesms.mms.IncomingMessage
import org.thoughtcrime.securesms.recipients.Recipient
import org.whispersystems.signalservice.api.NetworkResult
import org.whispersystems.signalservice.api.backup.BackupKey
import org.whispersystems.signalservice.api.backup.MediaName
import java.io.ByteArrayInputStream
import java.io.InputStream
import java.util.UUID
import kotlin.random.Random
import kotlin.time.Duration.Companion.seconds
class InternalBackupPlaygroundViewModel : ViewModel() {
private val backupKey = SignalStore.svr().getOrCreateMasterKey().deriveBackupKey()
var backupData: ByteArray? = null
val disposables = CompositeDisposable()
@@ -57,6 +65,17 @@ class InternalBackupPlaygroundViewModel : ViewModel() {
}
}
fun triggerBackupJob() {
_state.value = _state.value.copy(backupState = BackupState.EXPORT_IN_PROGRESS)
disposables += Single.fromCallable { ApplicationDependencies.getJobManager().runSynchronously(BackupMessagesJob(), 120_000) }
.subscribeOn(Schedulers.io())
.observeOn(AndroidSchedulers.mainThread())
.subscribeBy {
_state.value = _state.value.copy(backupState = BackupState.BACKUP_JOB_DONE)
}
}
fun import() {
backupData?.let {
_state.value = _state.value.copy(backupState = BackupState.IMPORT_IN_PROGRESS)
@@ -68,7 +87,7 @@ class InternalBackupPlaygroundViewModel : ViewModel() {
disposables += Single.fromCallable { BackupRepository.import(it.size.toLong(), { ByteArrayInputStream(it) }, selfData, plaintext = plaintext) }
.subscribeOn(Schedulers.io())
.observeOn(AndroidSchedulers.mainThread())
.subscribe { nothing ->
.subscribeBy {
backupData = null
_state.value = _state.value.copy(backupState = BackupState.NONE)
}
@@ -85,7 +104,7 @@ class InternalBackupPlaygroundViewModel : ViewModel() {
disposables += Single.fromCallable { BackupRepository.import(length, inputStreamFactory, selfData, plaintext = plaintext) }
.subscribeOn(Schedulers.io())
.observeOn(AndroidSchedulers.mainThread())
.subscribe { nothing ->
.subscribeBy {
backupData = null
_state.value = _state.value.copy(backupState = BackupState.NONE)
}
@@ -98,7 +117,7 @@ class InternalBackupPlaygroundViewModel : ViewModel() {
disposables += Single.fromCallable { BackupRepository.validate(length, inputStreamFactory, selfData) }
.subscribeOn(Schedulers.io())
.observeOn(AndroidSchedulers.mainThread())
.subscribe { nothing ->
.subscribeBy {
backupData = null
_state.value = _state.value.copy(backupState = BackupState.NONE)
}
@@ -142,47 +161,77 @@ class InternalBackupPlaygroundViewModel : ViewModel() {
}
}
fun restoreFromRemote() {
_state.value = _state.value.copy(backupState = BackupState.IMPORT_IN_PROGRESS)
disposables += Single.fromCallable {
ApplicationDependencies
.getJobManager()
.startChain(BackupRestoreJob())
.then(BackupRestoreMediaJob())
.enqueueAndBlockUntilCompletion(120.seconds.inWholeMilliseconds)
}
.subscribeOn(Schedulers.io())
.observeOn(AndroidSchedulers.mainThread())
.subscribeBy {
_state.value = _state.value.copy(backupState = BackupState.NONE)
}
}
fun loadMedia() {
disposables += Single
.fromCallable { SignalDatabase.attachments.debugGetLatestAttachments() }
.subscribeOn(Schedulers.io())
.observeOn(Schedulers.single())
.subscribeBy {
_mediaState.set { update(attachments = it.map { a -> BackupAttachment.from(backupKey, a) }) }
_mediaState.set { update(attachments = it.map { a -> BackupAttachment(dbAttachment = a) }) }
}
}
fun archiveAttachmentMedia(attachments: Set<AttachmentId>) {
disposables += Single
.fromCallable { BackupRepository.debugGetArchivedMediaState() }
.fromCallable {
val toArchive = mediaState.value
.attachments
.filter { attachments.contains(it.dbAttachment.attachmentId) }
.map { it.dbAttachment }
BackupRepository.archiveMedia(toArchive)
}
.subscribeOn(Schedulers.io())
.observeOn(Schedulers.single())
.doOnSubscribe { _mediaState.set { update(inProgress = inProgressMediaIds + attachments) } }
.doOnTerminate { _mediaState.set { update(inProgress = inProgressMediaIds - attachments) } }
.subscribeBy { result ->
when (result) {
is NetworkResult.Success -> _mediaState.set { update(archiveStateLoaded = true, backedUpMediaIds = result.result.map { it.mediaId }.toSet()) }
is NetworkResult.Success -> {
loadMedia()
result
.result
.sourceNotFoundResponses
.forEach {
reUploadAndArchiveMedia(result.result.mediaIdToAttachmentId(it.mediaId))
}
}
else -> _mediaState.set { copy(error = MediaStateError(errorText = "$result")) }
}
}
}
fun backupAttachmentMedia(mediaIds: Set<String>) {
disposables += Single.fromCallable { mediaIds.mapNotNull { mediaState.value.idToAttachment[it]?.dbAttachment }.toList() }
.map { BackupRepository.archiveMedia(it) }
fun archiveAttachmentMedia(attachment: BackupAttachment) {
disposables += Single.fromCallable { BackupRepository.archiveMedia(attachment.dbAttachment) }
.subscribeOn(Schedulers.io())
.observeOn(Schedulers.single())
.doOnSubscribe { _mediaState.set { update(inProgressMediaIds = inProgressMediaIds + mediaIds) } }
.doOnTerminate { _mediaState.set { update(inProgressMediaIds = inProgressMediaIds - mediaIds) } }
.doOnSubscribe { _mediaState.set { update(inProgress = inProgressMediaIds + attachment.dbAttachment.attachmentId) } }
.doOnTerminate { _mediaState.set { update(inProgress = inProgressMediaIds - attachment.dbAttachment.attachmentId) } }
.subscribeBy { result ->
when (result) {
is NetworkResult.Success -> {
val response = result.result
val successes = response.responses.filter { it.status == 200 }
val failures = response.responses - successes.toSet()
_mediaState.set {
var updated = update(backedUpMediaIds = backedUpMediaIds + successes.map { it.mediaId })
if (failures.isNotEmpty()) {
updated = updated.copy(error = MediaStateError(errorText = failures.toString()))
}
updated
is NetworkResult.Success -> loadMedia()
is NetworkResult.StatusCodeError -> {
if (result.code == 410) {
reUploadAndArchiveMedia(attachment.id)
} else {
_mediaState.set { copy(error = MediaStateError(errorText = "$result")) }
}
}
@@ -191,49 +240,107 @@ class InternalBackupPlaygroundViewModel : ViewModel() {
}
}
fun backupAttachmentMedia(attachment: BackupAttachment) {
disposables += Single.fromCallable { BackupRepository.archiveMedia(attachment.dbAttachment) }
private fun reUploadAndArchiveMedia(attachmentId: AttachmentId) {
disposables += Single
.fromCallable {
ApplicationDependencies
.getJobManager()
.startChain(AttachmentUploadJob(attachmentId))
.then(ArchiveAttachmentJob(attachmentId))
.enqueueAndBlockUntilCompletion(15.seconds.inWholeMilliseconds)
}
.subscribeOn(Schedulers.io())
.observeOn(Schedulers.single())
.doOnSubscribe { _mediaState.set { update(inProgressMediaIds = inProgressMediaIds + attachment.mediaId) } }
.doOnTerminate { _mediaState.set { update(inProgressMediaIds = inProgressMediaIds - attachment.mediaId) } }
.doOnSubscribe { _mediaState.set { update(inProgress = inProgressMediaIds + attachmentId) } }
.doOnTerminate { _mediaState.set { update(inProgress = inProgressMediaIds - attachmentId) } }
.subscribeBy {
when (it) {
is NetworkResult.Success -> {
_mediaState.set { update(backedUpMediaIds = backedUpMediaIds + attachment.mediaId) }
}
else -> _mediaState.set { copy(error = MediaStateError(errorText = "$it")) }
if (it.isPresent && it.get().isComplete) {
loadMedia()
} else {
_mediaState.set { copy(error = MediaStateError(errorText = "Reupload slow or failed, try again")) }
}
}
}
fun deleteBackupAttachmentMedia(mediaIds: Set<String>) {
deleteBackupAttachmentMedia(mediaIds.mapNotNull { mediaState.value.idToAttachment[it] }.toList())
fun deleteArchivedMedia(attachmentIds: Set<AttachmentId>) {
deleteArchivedMedia(mediaState.value.attachments.filter { attachmentIds.contains(it.dbAttachment.attachmentId) })
}
fun deleteBackupAttachmentMedia(attachment: BackupAttachment) {
deleteBackupAttachmentMedia(listOf(attachment))
fun deleteArchivedMedia(attachment: BackupAttachment) {
deleteArchivedMedia(listOf(attachment))
}
private fun deleteBackupAttachmentMedia(attachments: List<BackupAttachment>) {
val ids = attachments.map { it.mediaId }.toSet()
private fun deleteArchivedMedia(attachments: List<BackupAttachment>) {
val ids = attachments.map { it.dbAttachment.attachmentId }.toSet()
disposables += Single.fromCallable { BackupRepository.deleteArchivedMedia(attachments.map { it.dbAttachment }) }
.subscribeOn(Schedulers.io())
.observeOn(Schedulers.single())
.doOnSubscribe { _mediaState.set { update(inProgressMediaIds = inProgressMediaIds + ids) } }
.doOnTerminate { _mediaState.set { update(inProgressMediaIds = inProgressMediaIds - ids) } }
.doOnSubscribe { _mediaState.set { update(inProgress = inProgressMediaIds + ids) } }
.doOnTerminate { _mediaState.set { update(inProgress = inProgressMediaIds - ids) } }
.subscribeBy {
when (it) {
is NetworkResult.Success -> {
_mediaState.set { update(backedUpMediaIds = backedUpMediaIds - ids) }
}
is NetworkResult.Success -> loadMedia()
else -> _mediaState.set { copy(error = MediaStateError(errorText = "$it")) }
}
}
}
fun deleteAllArchivedMedia() {
disposables += Single
.fromCallable { BackupRepository.debugDeleteAllArchivedMedia() }
.subscribeOn(Schedulers.io())
.observeOn(Schedulers.single())
.subscribeBy { result ->
when (result) {
is NetworkResult.Success -> loadMedia()
else -> _mediaState.set { copy(error = MediaStateError(errorText = "$result")) }
}
}
}
fun restoreArchivedMedia(attachment: BackupAttachment) {
disposables += Completable
.fromCallable {
val recipientId = SignalStore.releaseChannelValues().releaseChannelRecipientId!!
val threadId = SignalDatabase.threads.getOrCreateThreadIdFor(Recipient.resolved(recipientId))
val message = IncomingMessage(
type = MessageType.NORMAL,
from = recipientId,
sentTimeMillis = System.currentTimeMillis(),
serverTimeMillis = System.currentTimeMillis(),
receivedTimeMillis = System.currentTimeMillis(),
body = "Restored from Archive!?",
serverGuid = UUID.randomUUID().toString()
)
val insertMessage = SignalDatabase.messages.insertMessageInbox(message, threadId).get()
SignalDatabase.attachments.debugCopyAttachmentForArchiveRestore(
insertMessage.messageId,
attachment.dbAttachment
)
val archivedAttachment = SignalDatabase.attachments.getAttachmentsForMessage(insertMessage.messageId).first()
ApplicationDependencies.getJobManager().add(
AttachmentDownloadJob(
messageId = insertMessage.messageId,
attachmentId = archivedAttachment.attachmentId,
manual = false,
forceArchiveDownload = true
)
)
}
.subscribeOn(Schedulers.io())
.observeOn(Schedulers.single())
.subscribeBy(
onError = {
_mediaState.set { copy(error = MediaStateError(errorText = "$it")) }
}
)
}
override fun onCleared() {
disposables.clear()
}
@@ -246,7 +353,7 @@ class InternalBackupPlaygroundViewModel : ViewModel() {
)
enum class BackupState(val inProgress: Boolean = false) {
NONE, EXPORT_IN_PROGRESS(true), EXPORT_DONE, IMPORT_IN_PROGRESS(true)
NONE, EXPORT_IN_PROGRESS(true), EXPORT_DONE, BACKUP_JOB_DONE, IMPORT_IN_PROGRESS(true)
}
enum class BackupUploadState(val inProgress: Boolean = false) {
@@ -261,67 +368,59 @@ class InternalBackupPlaygroundViewModel : ViewModel() {
}
data class MediaState(
val backupStateLoaded: Boolean = false,
val attachments: List<BackupAttachment> = emptyList(),
val backedUpMediaIds: Set<String> = emptySet(),
val inProgressMediaIds: Set<String> = emptySet(),
val inProgressMediaIds: Set<AttachmentId> = emptySet(),
val error: MediaStateError? = null
) {
val idToAttachment: Map<String, BackupAttachment> = attachments.associateBy { it.mediaId }
fun update(
archiveStateLoaded: Boolean = this.backupStateLoaded,
attachments: List<BackupAttachment> = this.attachments,
backedUpMediaIds: Set<String> = this.backedUpMediaIds,
inProgressMediaIds: Set<String> = this.inProgressMediaIds
inProgress: Set<AttachmentId> = this.inProgressMediaIds
): MediaState {
val updatedAttachments = if (archiveStateLoaded) {
attachments.map {
val state = if (inProgressMediaIds.contains(it.mediaId)) {
BackupAttachment.State.IN_PROGRESS
} else if (backedUpMediaIds.contains(it.mediaId)) {
BackupAttachment.State.UPLOADED
} else {
BackupAttachment.State.LOCAL_ONLY
}
val backupKey = SignalStore.svr().getOrCreateMasterKey().deriveBackupKey()
it.copy(state = state)
val updatedAttachments = attachments.map {
val state = if (inProgress.contains(it.dbAttachment.attachmentId)) {
BackupAttachment.State.IN_PROGRESS
} else if (it.dbAttachment.archiveMediaName != null) {
if (it.dbAttachment.remoteDigest != null) {
val mediaId = backupKey.deriveMediaId(MediaName(it.dbAttachment.archiveMediaName)).encode()
if (it.dbAttachment.archiveMediaId == mediaId) {
BackupAttachment.State.UPLOADED_FINAL
} else {
BackupAttachment.State.UPLOADED_UNDOWNLOADED
}
} else {
BackupAttachment.State.UPLOADED_UNDOWNLOADED
}
} else if (it.dbAttachment.dataHash == null) {
BackupAttachment.State.ATTACHMENT_CDN
} else {
BackupAttachment.State.LOCAL_ONLY
}
} else {
attachments
it.copy(state = state)
}
return copy(
backupStateLoaded = archiveStateLoaded,
attachments = updatedAttachments,
backedUpMediaIds = backedUpMediaIds
attachments = updatedAttachments
)
}
}
data class BackupAttachment(
val dbAttachment: DatabaseAttachment,
val state: State = State.INIT,
val mediaId: String = Base64.encodeUrlSafeWithPadding(Random.nextBytes(15))
val state: State = State.LOCAL_ONLY
) {
val id: Any = dbAttachment.attachmentId
val id: AttachmentId = dbAttachment.attachmentId
val title: String = dbAttachment.attachmentId.toString()
enum class State {
INIT,
ATTACHMENT_CDN,
LOCAL_ONLY,
UPLOADED,
UPLOADED_UNDOWNLOADED,
UPLOADED_FINAL,
IN_PROGRESS
}
companion object {
fun from(backupKey: BackupKey, dbAttachment: DatabaseAttachment): BackupAttachment {
return BackupAttachment(
dbAttachment = dbAttachment,
mediaId = backupKey.deriveMediaId(Base64.decode(dbAttachment.dataHash!!)).toString()
)
}
}
}
data class MediaStateError(

6
app/src/main/java/org/thoughtcrime/securesms/conversation/ConversationItem.java
View File

@@ -2450,7 +2450,8 @@ public final class ConversationItem extends RelativeLayout implements BindableCo
for (Slide slide : slides) {
ApplicationDependencies.getJobManager().add(new AttachmentDownloadJob(messageRecord.getId(),
((DatabaseAttachment) slide.asAttachment()).attachmentId,
true));
true,
false));
}
}
}
@@ -2476,7 +2477,8 @@ public final class ConversationItem extends RelativeLayout implements BindableCo
setup(v, slide);
jobManager.add(new AttachmentDownloadJob(messageRecord.getId(),
attachmentId,
true));
true,
false));
jobManager.addListener(queue, (job, jobState) -> {
if (jobState.isComplete()) {
cleanup();

8
app/src/main/java/org/thoughtcrime/securesms/conversation/v2/data/ConversationDataSource.kt
View File

@@ -10,6 +10,8 @@ import org.signal.core.util.Stopwatch
import org.signal.core.util.logging.Log
import org.signal.core.util.toInt
import org.signal.paging.PagedDataSource
import org.thoughtcrime.securesms.BuildConfig
import org.thoughtcrime.securesms.backup.v2.BackupRestoreManager
import org.thoughtcrime.securesms.conversation.ConversationData
import org.thoughtcrime.securesms.conversation.ConversationMessage
import org.thoughtcrime.securesms.conversation.ConversationMessage.ConversationMessageFactory
@@ -20,6 +22,7 @@ import org.thoughtcrime.securesms.database.model.InMemoryMessageRecord.Universal
import org.thoughtcrime.securesms.database.model.MessageRecord
import org.thoughtcrime.securesms.database.model.MmsMessageRecord
import org.thoughtcrime.securesms.dependencies.ApplicationDependencies
import org.thoughtcrime.securesms.keyvalue.SignalStore
import org.thoughtcrime.securesms.messagerequests.MessageRequestRepository
import org.thoughtcrime.securesms.recipients.Recipient
import org.thoughtcrime.securesms.util.adapter.mapping.MappingModel
@@ -122,6 +125,11 @@ class ConversationDataSource(
records = MessageDataFetcher.updateModelsWithData(records, extraData).toMutableList()
stopwatch.split("models")
if (BuildConfig.MESSAGE_BACKUP_RESTORE_ENABLED && SignalStore.backup().restoreState.inProgress) {
BackupRestoreManager.prioritizeAttachmentsIfNeeded(records)
stopwatch.split("restore")
}
val messages = records.map { record ->
ConversationMessageFactory.createWithUnresolvedData(
localContext,

265
app/src/main/java/org/thoughtcrime/securesms/database/AttachmentTable.kt
View File

@@ -52,13 +52,16 @@ import org.signal.core.util.requireInt
import org.signal.core.util.requireLong
import org.signal.core.util.requireNonNullBlob
import org.signal.core.util.requireNonNullString
import org.signal.core.util.requireObject
import org.signal.core.util.requireString
import org.signal.core.util.select
import org.signal.core.util.toInt
import org.signal.core.util.update
import org.signal.core.util.withinTransaction
import org.thoughtcrime.securesms.attachments.ArchivedAttachment
import org.thoughtcrime.securesms.attachments.Attachment
import org.thoughtcrime.securesms.attachments.AttachmentId
import org.thoughtcrime.securesms.attachments.Cdn
import org.thoughtcrime.securesms.attachments.DatabaseAttachment
import org.thoughtcrime.securesms.audio.AudioHash
import org.thoughtcrime.securesms.blurhash.BlurHash
@@ -140,6 +143,10 @@ class AttachmentTable(
const val TRANSFORM_PROPERTIES = "transform_properties"
const val DISPLAY_ORDER = "display_order"
const val UPLOAD_TIMESTAMP = "upload_timestamp"
const val ARCHIVE_CDN = "archive_cdn"
const val ARCHIVE_MEDIA_NAME = "archive_media_name"
const val ARCHIVE_MEDIA_ID = "archive_media_id"
const val ARCHIVE_TRANSFER_FILE = "archive_transfer_file"
const val ATTACHMENT_JSON_ALIAS = "attachment_json"
@@ -150,6 +157,8 @@ class AttachmentTable(
const val TRANSFER_PROGRESS_PENDING = 2
const val TRANSFER_PROGRESS_FAILED = 3
const val TRANSFER_PROGRESS_PERMANENT_FAILURE = 4
const val TRANSFER_NEEDS_RESTORE = 5
const val TRANSFER_RESTORE_IN_PROGRESS = 6
const val PREUPLOAD_MESSAGE_ID: Long = -8675309
private val PROJECTION = arrayOf(
@@ -185,7 +194,11 @@ class AttachmentTable(
DISPLAY_ORDER,
UPLOAD_TIMESTAMP,
DATA_HASH_START,
DATA_HASH_END
DATA_HASH_END,
ARCHIVE_CDN,
ARCHIVE_MEDIA_NAME,
ARCHIVE_MEDIA_ID,
ARCHIVE_TRANSFER_FILE
)
const val CREATE_TABLE = """
@@ -222,7 +235,11 @@ class AttachmentTable(
$DISPLAY_ORDER INTEGER DEFAULT 0,
$UPLOAD_TIMESTAMP INTEGER DEFAULT 0,
$DATA_HASH_START TEXT DEFAULT NULL,
$DATA_HASH_END TEXT DEFAULT NULL
$DATA_HASH_END TEXT DEFAULT NULL,
$ARCHIVE_CDN INTEGER DEFAULT 0,
$ARCHIVE_MEDIA_NAME TEXT DEFAULT NULL,
$ARCHIVE_MEDIA_ID TEXT DEFAULT NULL,
$ARCHIVE_TRANSFER_FILE TEXT DEFAULT NULL
)
"""
@@ -239,7 +256,6 @@ class AttachmentTable(
val ATTACHMENT_POINTER_REUSE_THRESHOLD = 7.days.inWholeMilliseconds
@JvmStatic
@JvmOverloads
@Throws(IOException::class)
fun newDataFile(context: Context): File {
val partsDirectory = context.getDir(DIRECTORY, Context.MODE_PRIVATE)
@@ -388,6 +404,27 @@ class AttachmentTable(
.flatten()
}
fun getArchivableAttachments(): Cursor {
return readableDatabase
.select(*PROJECTION)
.from(TABLE_NAME)
.where("$ARCHIVE_MEDIA_ID IS NULL AND $REMOTE_DIGEST IS NOT NULL AND ($TRANSFER_STATE = ? OR $TRANSFER_STATE = ?)", TRANSFER_PROGRESS_DONE.toString(), TRANSFER_NEEDS_RESTORE.toString())
.orderBy("$ID DESC")
.run()
}
fun getRestorableAttachments(batchSize: Int): List<DatabaseAttachment> {
return readableDatabase
.select(*PROJECTION)
.from(TABLE_NAME)
.where("$TRANSFER_STATE = ?", TRANSFER_NEEDS_RESTORE.toString())
.limit(batchSize)
.orderBy("$ID DESC")
.run().readToList {
it.readAttachments()
}.flatten()
}
fun deleteAttachmentsForMessage(mmsId: Long): Boolean {
Log.d(TAG, "[deleteAttachmentsForMessage] mmsId: $mmsId")
@@ -679,6 +716,7 @@ class AttachmentTable(
values.put(TRANSFER_STATE, TRANSFER_PROGRESS_DONE)
values.put(TRANSFER_FILE, null as String?)
values.put(TRANSFORM_PROPERTIES, TransformProperties.forSkipTransform().serialize())
values.put(ARCHIVE_TRANSFER_FILE, null as String?)
db.update(TABLE_NAME)
.values(values)
@@ -734,7 +772,7 @@ class AttachmentTable(
val values = contentValuesOf(
TRANSFER_STATE to TRANSFER_PROGRESS_DONE,
CDN_NUMBER to attachment.cdnNumber,
CDN_NUMBER to attachment.cdn.serialize(),
REMOTE_LOCATION to attachment.remoteLocation,
REMOTE_DIGEST to attachment.remoteDigest,
REMOTE_INCREMENTAL_DIGEST to attachment.incrementalDigest,
@@ -774,7 +812,7 @@ class AttachmentTable(
DATA_SIZE to sourceDataInfo.length,
DATA_RANDOM to sourceDataInfo.random,
TRANSFER_STATE to sourceAttachment.transferState,
CDN_NUMBER to sourceAttachment.cdnNumber,
CDN_NUMBER to sourceAttachment.cdn.serialize(),
REMOTE_LOCATION to sourceAttachment.remoteLocation,
REMOTE_DIGEST to sourceAttachment.remoteDigest,
REMOTE_INCREMENTAL_DIGEST to sourceAttachment.incrementalDigest,
@@ -865,7 +903,11 @@ class AttachmentTable(
val attachmentId = if (attachment.uri != null) {
insertAttachmentWithData(mmsId, attachment, attachment.quote)
} else {
insertUndownloadedAttachment(mmsId, attachment, attachment.quote)
if (attachment is ArchivedAttachment) {
insertArchivedAttachment(mmsId, attachment, attachment.quote)
} else {
insertUndownloadedAttachment(mmsId, attachment, attachment.quote)
}
}
insertedAttachments[attachment] = attachmentId
@@ -890,6 +932,75 @@ class AttachmentTable(
return insertedAttachments
}
fun debugCopyAttachmentForArchiveRestore(
mmsId: Long,
attachment: DatabaseAttachment
) {
val copy =
"""
INSERT INTO $TABLE_NAME
(
$MESSAGE_ID,
$CONTENT_TYPE,
$TRANSFER_STATE,
$CDN_NUMBER,
$REMOTE_LOCATION,
$REMOTE_DIGEST,
$REMOTE_INCREMENTAL_DIGEST,
$REMOTE_INCREMENTAL_DIGEST_CHUNK_SIZE,
$REMOTE_KEY,
$FILE_NAME,
$DATA_SIZE,
$VOICE_NOTE,
$BORDERLESS,
$VIDEO_GIF,
$WIDTH,
$HEIGHT,
$CAPTION,
$UPLOAD_TIMESTAMP,
$BLUR_HASH,
$DATA_SIZE,
$DATA_RANDOM,
$DATA_HASH_START,
$DATA_HASH_END,
$ARCHIVE_MEDIA_ID,
$ARCHIVE_MEDIA_NAME,
$ARCHIVE_CDN
)
SELECT
$mmsId,
$CONTENT_TYPE,
$TRANSFER_PROGRESS_PENDING,
$CDN_NUMBER,
$REMOTE_LOCATION,
$REMOTE_DIGEST,
$REMOTE_INCREMENTAL_DIGEST,
$REMOTE_INCREMENTAL_DIGEST_CHUNK_SIZE,
$REMOTE_KEY,
$FILE_NAME,
$DATA_SIZE,
$VOICE_NOTE,
$BORDERLESS,
$VIDEO_GIF,
$WIDTH,
$HEIGHT,
$CAPTION,
${System.currentTimeMillis()},
$BLUR_HASH,
$DATA_SIZE,
$DATA_RANDOM,
$DATA_HASH_START,
$DATA_HASH_END,
"${attachment.archiveMediaId}",
"${attachment.archiveMediaName}",
${attachment.archiveCdn}
FROM $TABLE_NAME
WHERE $ID = ${attachment.attachmentId.id}
"""
writableDatabase.execSQL(copy)
}
/**
* Updates the data stored for an existing attachment. This happens after transformations, like transcoding.
*/
@@ -956,6 +1067,24 @@ class AttachmentTable(
return transferFile
}
@Throws(IOException::class)
fun getOrCreateArchiveTransferFile(attachmentId: AttachmentId): File {
val existing = getArchiveTransferFile(writableDatabase, attachmentId)
if (existing != null) {
return existing
}
val transferFile = newTransferFile()
writableDatabase
.update(TABLE_NAME)
.values(ARCHIVE_TRANSFER_FILE to transferFile.absolutePath)
.where("$ID = ?", attachmentId.id)
.run()
return transferFile
}
fun getDataFileInfo(attachmentId: AttachmentId): DataFileInfo? {
return readableDatabase
.select(ID, DATA_FILE, DATA_SIZE, DATA_RANDOM, DATA_HASH_START, DATA_HASH_END, TRANSFORM_PROPERTIES, UPLOAD_TIMESTAMP)
@@ -1087,7 +1216,7 @@ class AttachmentTable(
transferProgress = jsonObject.getInt(TRANSFER_STATE),
size = jsonObject.getLong(DATA_SIZE),
fileName = jsonObject.getString(FILE_NAME),
cdnNumber = jsonObject.getInt(CDN_NUMBER),
cdn = Cdn.deserialize(jsonObject.getInt(CDN_NUMBER)),
location = jsonObject.getString(REMOTE_LOCATION),
key = jsonObject.getString(REMOTE_KEY),
digest = null,
@@ -1116,7 +1245,10 @@ class AttachmentTable(
transformProperties = TransformProperties.parse(jsonObject.getString(TRANSFORM_PROPERTIES)),
displayOrder = jsonObject.getInt(DISPLAY_ORDER),
uploadTimestamp = jsonObject.getLong(UPLOAD_TIMESTAMP),
dataHash = jsonObject.getString(DATA_HASH_END)
dataHash = jsonObject.getString(DATA_HASH_END),
archiveCdn = jsonObject.getInt(ARCHIVE_CDN),
archiveMediaName = jsonObject.getString(ARCHIVE_MEDIA_NAME),
archiveMediaId = jsonObject.getString(ARCHIVE_MEDIA_ID)
)
}
}
@@ -1156,6 +1288,45 @@ class AttachmentTable(
return readableDatabase.rawQuery(query, null)
}
fun setArchiveData(attachmentId: AttachmentId, archiveCdn: Int, archiveMediaName: String, archiveMediaId: String) {
writableDatabase
.update(TABLE_NAME)
.values(
ARCHIVE_CDN to archiveCdn,
ARCHIVE_MEDIA_ID to archiveMediaId,
ARCHIVE_MEDIA_NAME to archiveMediaName
)
.where("$ID = ?", attachmentId.id)
.run()
}
fun clearArchiveData(attachmentIds: List<AttachmentId>) {
SqlUtil.buildCollectionQuery(ID, attachmentIds.map { it.id })
.forEach { query ->
writableDatabase
.update(TABLE_NAME)
.values(
ARCHIVE_CDN to 0,
ARCHIVE_MEDIA_ID to null,
ARCHIVE_MEDIA_NAME to null
)
.where(query.where, query.whereArgs)
.run()
}
}
fun clearAllArchiveData() {
writableDatabase
.update(TABLE_NAME)
.values(
ARCHIVE_CDN to 0,
ARCHIVE_MEDIA_ID to null,
ARCHIVE_MEDIA_NAME to null
)
.where("$ARCHIVE_CDN > 0 OR $ARCHIVE_MEDIA_ID IS NOT NULL OR $ARCHIVE_MEDIA_NAME IS NOT NULL")
.run()
}
/**
* Deletes the data file if there's no strong references to other attachments.
* If deleted, it will also clear all weak references (i.e. quotes) of the attachment.
@@ -1338,7 +1509,7 @@ class AttachmentTable(
put(MESSAGE_ID, messageId)
put(CONTENT_TYPE, attachment.contentType)
put(TRANSFER_STATE, attachment.transferState)
put(CDN_NUMBER, attachment.cdnNumber)
put(CDN_NUMBER, attachment.cdn.serialize())
put(REMOTE_LOCATION, attachment.remoteLocation)
put(REMOTE_DIGEST, attachment.remoteDigest)
put(REMOTE_INCREMENTAL_DIGEST, attachment.incrementalDigest)
@@ -1373,6 +1544,59 @@ class AttachmentTable(
return attachmentId
}
/**
* Attachments need records in the database even if they haven't been downloaded yet. That allows us to store the info we need to download it, what message
* it's associated with, etc. We treat this case separately from attachments with data (see [insertAttachmentWithData]) because it's much simpler,
* and splitting the two use cases makes the code easier to understand.
*
* Callers are expected to later call [finalizeAttachmentAfterDownload] once they have downloaded the data for this attachment.
*/
@Throws(MmsException::class)
private fun insertArchivedAttachment(messageId: Long, attachment: ArchivedAttachment, quote: Boolean): AttachmentId {
Log.d(TAG, "[insertAttachment] Inserting attachment for messageId $messageId.")
val attachmentId: AttachmentId = writableDatabase.withinTransaction { db ->
val contentValues = ContentValues().apply {
put(MESSAGE_ID, messageId)
put(CONTENT_TYPE, attachment.contentType)
put(TRANSFER_STATE, attachment.transferState)
put(CDN_NUMBER, attachment.cdn.serialize())
put(REMOTE_LOCATION, attachment.remoteLocation)
put(REMOTE_DIGEST, attachment.remoteDigest)
put(REMOTE_INCREMENTAL_DIGEST, attachment.incrementalDigest)
put(REMOTE_INCREMENTAL_DIGEST_CHUNK_SIZE, attachment.incrementalMacChunkSize)
put(REMOTE_KEY, attachment.remoteKey)
put(FILE_NAME, StorageUtil.getCleanFileName(attachment.fileName))
put(DATA_SIZE, attachment.size)
put(FAST_PREFLIGHT_ID, attachment.fastPreflightId)
put(VOICE_NOTE, attachment.voiceNote.toInt())
put(BORDERLESS, attachment.borderless.toInt())
put(VIDEO_GIF, attachment.videoGif.toInt())
put(WIDTH, attachment.width)
put(HEIGHT, attachment.height)
put(QUOTE, quote)
put(CAPTION, attachment.caption)
put(UPLOAD_TIMESTAMP, attachment.uploadTimestamp)
put(ARCHIVE_CDN, attachment.archiveCdn)
put(ARCHIVE_MEDIA_NAME, attachment.archiveMediaName)
put(ARCHIVE_MEDIA_ID, attachment.archiveMediaId)
attachment.stickerLocator?.let { sticker ->
put(STICKER_PACK_ID, sticker.packId)
put(STICKER_PACK_KEY, sticker.packKey)
put(STICKER_ID, sticker.stickerId)
put(STICKER_EMOJI, sticker.emoji)
}
}
val rowId = db.insert(TABLE_NAME, null, contentValues)
AttachmentId(rowId)
}
notifyAttachmentListeners()
return attachmentId
}
/**
* Inserts an attachment with existing data. This is likely an outgoing attachment that we're in the process of sending.
*/
@@ -1462,7 +1686,7 @@ class AttachmentTable(
contentValues.put(MESSAGE_ID, messageId)
contentValues.put(CONTENT_TYPE, uploadTemplate?.contentType ?: attachment.contentType)
contentValues.put(TRANSFER_STATE, attachment.transferState) // Even if we have a template, we let AttachmentUploadJob have the final say so it can re-check and make sure the template is still valid
contentValues.put(CDN_NUMBER, uploadTemplate?.cdnNumber ?: 0)
contentValues.put(CDN_NUMBER, uploadTemplate?.cdn?.serialize() ?: Cdn.CDN_0.serialize())
contentValues.put(REMOTE_LOCATION, uploadTemplate?.remoteLocation)
contentValues.put(REMOTE_DIGEST, uploadTemplate?.remoteDigest)
contentValues.put(REMOTE_INCREMENTAL_DIGEST, uploadTemplate?.incrementalDigest)
@@ -1520,6 +1744,18 @@ class AttachmentTable(
}
}
private fun getArchiveTransferFile(db: SQLiteDatabase, attachmentId: AttachmentId): File? {
return db
.select(ARCHIVE_TRANSFER_FILE)
.from(TABLE_NAME)
.where("$ID = ?", attachmentId.id)
.limit(1)
.run()
.readToSingleObject { cursor ->
cursor.requireString(ARCHIVE_TRANSFER_FILE)?.let { File(it) }
}
}
private fun getAttachment(cursor: Cursor): DatabaseAttachment {
val contentType = cursor.requireString(CONTENT_TYPE)
@@ -1532,7 +1768,7 @@ class AttachmentTable(
transferProgress = cursor.requireInt(TRANSFER_STATE),
size = cursor.requireLong(DATA_SIZE),
fileName = cursor.requireString(FILE_NAME),
cdnNumber = cursor.requireInt(CDN_NUMBER),
cdn = cursor.requireObject(CDN_NUMBER, Cdn.Serializer),
location = cursor.requireString(REMOTE_LOCATION),
key = cursor.requireString(REMOTE_KEY),
digest = cursor.requireBlob(REMOTE_DIGEST),
@@ -1552,7 +1788,10 @@ class AttachmentTable(
transformProperties = TransformProperties.parse(cursor.requireString(TRANSFORM_PROPERTIES)),
displayOrder = cursor.requireInt(DISPLAY_ORDER),
uploadTimestamp = cursor.requireLong(UPLOAD_TIMESTAMP),
dataHash = cursor.requireString(DATA_HASH_END)
dataHash = cursor.requireString(DATA_HASH_END),
archiveCdn = cursor.requireInt(ARCHIVE_CDN),
archiveMediaName = cursor.requireString(ARCHIVE_MEDIA_NAME),
archiveMediaId = cursor.requireString(ARCHIVE_MEDIA_ID)
)
}
@@ -1603,7 +1842,7 @@ class AttachmentTable(
return readableDatabase
.select(*PROJECTION)
.from(TABLE_NAME)
.where("$TRANSFER_STATE == $TRANSFER_PROGRESS_DONE AND $REMOTE_LOCATION IS NOT NULL AND $DATA_HASH_END IS NOT NULL")
.where("$REMOTE_LOCATION IS NOT NULL AND $REMOTE_KEY IS NOT NULL")
.orderBy("$ID DESC")
.limit(30)
.run()

5
app/src/main/java/org/thoughtcrime/securesms/database/GroupTable.kt
View File

@@ -56,6 +56,7 @@ import org.whispersystems.signalservice.api.groupsv2.findRequestingByAci
import org.whispersystems.signalservice.api.groupsv2.toAciList
import org.whispersystems.signalservice.api.groupsv2.toAciListWithUnknowns
import org.whispersystems.signalservice.api.messages.SignalServiceAttachmentPointer
import org.whispersystems.signalservice.api.messages.SignalServiceAttachmentRemoteId
import org.whispersystems.signalservice.api.push.DistributionId
import org.whispersystems.signalservice.api.push.ServiceId
import org.whispersystems.signalservice.api.push.ServiceId.ACI
@@ -746,7 +747,7 @@ class GroupTable(context: Context?, databaseHelper: SignalDatabase?) : DatabaseT
values.put(MMS, groupId.isMms)
if (avatar != null) {
values.put(AVATAR_ID, avatar.remoteId.v2.get())
values.put(AVATAR_ID, (avatar.remoteId as SignalServiceAttachmentRemoteId.V2).cdnId)
values.put(AVATAR_KEY, avatar.key)
values.put(AVATAR_CONTENT_TYPE, avatar.contentType)
values.put(AVATAR_DIGEST, avatar.digest.orElse(null))
@@ -822,7 +823,7 @@ class GroupTable(context: Context?, databaseHelper: SignalDatabase?) : DatabaseT
}
if (avatar != null) {
put(AVATAR_ID, avatar.remoteId.v2.get())
put(AVATAR_ID, (avatar.remoteId as SignalServiceAttachmentRemoteId.V2).cdnId)
put(AVATAR_CONTENT_TYPE, avatar.contentType)
put(AVATAR_KEY, avatar.key)
put(AVATAR_DIGEST, avatar.digest.orElse(null))

3
app/src/main/java/org/thoughtcrime/securesms/database/MediaTable.kt
View File

@@ -50,6 +50,9 @@ class MediaTable internal constructor(context: Context?, databaseHelper: SignalD
${AttachmentTable.TABLE_NAME}.${AttachmentTable.REMOTE_INCREMENTAL_DIGEST},
${AttachmentTable.TABLE_NAME}.${AttachmentTable.REMOTE_INCREMENTAL_DIGEST_CHUNK_SIZE},
${AttachmentTable.TABLE_NAME}.${AttachmentTable.DATA_HASH_END},
${AttachmentTable.TABLE_NAME}.${AttachmentTable.ARCHIVE_CDN},
${AttachmentTable.TABLE_NAME}.${AttachmentTable.ARCHIVE_MEDIA_NAME},
${AttachmentTable.TABLE_NAME}.${AttachmentTable.ARCHIVE_MEDIA_ID},
${MessageTable.TABLE_NAME}.${MessageTable.TYPE},
${MessageTable.TABLE_NAME}.${MessageTable.DATE_SENT},
${MessageTable.TABLE_NAME}.${MessageTable.DATE_RECEIVED},

6
app/src/main/java/org/thoughtcrime/securesms/database/MessageTable.kt
View File

@@ -376,7 +376,11 @@ open class MessageTable(context: Context?, databaseHelper: SignalDatabase) : Dat
'${AttachmentTable.BLUR_HASH}', ${AttachmentTable.TABLE_NAME}.${AttachmentTable.BLUR_HASH},
'${AttachmentTable.TRANSFORM_PROPERTIES}', ${AttachmentTable.TABLE_NAME}.${AttachmentTable.TRANSFORM_PROPERTIES},
'${AttachmentTable.DISPLAY_ORDER}', ${AttachmentTable.TABLE_NAME}.${AttachmentTable.DISPLAY_ORDER},
'${AttachmentTable.UPLOAD_TIMESTAMP}', ${AttachmentTable.TABLE_NAME}.${AttachmentTable.UPLOAD_TIMESTAMP}
'${AttachmentTable.UPLOAD_TIMESTAMP}', ${AttachmentTable.TABLE_NAME}.${AttachmentTable.UPLOAD_TIMESTAMP},
'${AttachmentTable.DATA_HASH_END}', ${AttachmentTable.TABLE_NAME}.${AttachmentTable.DATA_HASH_END},
'${AttachmentTable.ARCHIVE_CDN}', ${AttachmentTable.TABLE_NAME}.${AttachmentTable.ARCHIVE_CDN},
'${AttachmentTable.ARCHIVE_MEDIA_NAME}', ${AttachmentTable.TABLE_NAME}.${AttachmentTable.ARCHIVE_MEDIA_NAME},
'${AttachmentTable.ARCHIVE_MEDIA_ID}', ${AttachmentTable.TABLE_NAME}.${AttachmentTable.ARCHIVE_MEDIA_ID}
)
) AS ${AttachmentTable.ATTACHMENT_JSON_ALIAS}
""".toSingleLine()

6
app/src/main/java/org/thoughtcrime/securesms/database/helpers/SignalDatabaseMigrations.kt
View File

@@ -81,6 +81,7 @@ import org.thoughtcrime.securesms.database.helpers.migration.V220_PreKeyConstrai
import org.thoughtcrime.securesms.database.helpers.migration.V221_AddReadColumnToCallEventsTable
import org.thoughtcrime.securesms.database.helpers.migration.V222_DataHashRefactor
import org.thoughtcrime.securesms.database.helpers.migration.V223_AddNicknameAndNoteFieldsToRecipientTable
import org.thoughtcrime.securesms.database.helpers.migration.V224_AddAttachmentArchiveColumns
/**
* Contains all of the database migrations for [SignalDatabase]. Broken into a separate file for cleanliness.
@@ -164,10 +165,11 @@ object SignalDatabaseMigrations {
220 to V220_PreKeyConstraints,
221 to V221_AddReadColumnToCallEventsTable,
222 to V222_DataHashRefactor,
223 to V223_AddNicknameAndNoteFieldsToRecipientTable
223 to V223_AddNicknameAndNoteFieldsToRecipientTable,
224 to V224_AddAttachmentArchiveColumns
)
const val DATABASE_VERSION = 223
const val DATABASE_VERSION = 224
@JvmStatic
fun migrate(context: Application, db: SQLiteDatabase, oldVersion: Int, newVersion: Int) {

22
app/src/main/java/org/thoughtcrime/securesms/database/helpers/migration/V224_AddAttachmentArchiveColumns.kt Normal file
View File

@@ -0,0 +1,22 @@
/*
* Copyright 2024 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.thoughtcrime.securesms.database.helpers.migration
import android.app.Application
import net.zetetic.database.sqlcipher.SQLiteDatabase
/**
* Adds archive_cdn and archive_media to attachment.
*/
@Suppress("ClassName")
object V224_AddAttachmentArchiveColumns : SignalDatabaseMigration {
override fun migrate(context: Application, db: SQLiteDatabase, oldVersion: Int, newVersion: Int) {
db.execSQL("ALTER TABLE attachment ADD COLUMN archive_cdn INTEGER DEFAULT 0")
db.execSQL("ALTER TABLE attachment ADD COLUMN archive_media_name TEXT DEFAULT NULL")
db.execSQL("ALTER TABLE attachment ADD COLUMN archive_media_id TEXT DEFAULT NULL")
db.execSQL("ALTER TABLE attachment ADD COLUMN archive_transfer_file TEXT DEFAULT NULL")
}
}

78
app/src/main/java/org/thoughtcrime/securesms/jobs/ArchiveAttachmentJob.kt Normal file
View File

@@ -0,0 +1,78 @@
package org.thoughtcrime.securesms.jobs
import org.signal.core.util.logging.Log
import org.thoughtcrime.securesms.attachments.AttachmentId
import org.thoughtcrime.securesms.backup.v2.BackupRepository
import org.thoughtcrime.securesms.database.SignalDatabase
import org.thoughtcrime.securesms.dependencies.ApplicationDependencies
import org.thoughtcrime.securesms.jobmanager.Job
import org.thoughtcrime.securesms.jobmanager.impl.NetworkConstraint
import org.thoughtcrime.securesms.jobs.protos.ArchiveAttachmentJobData
import org.thoughtcrime.securesms.keyvalue.SignalStore
import org.whispersystems.signalservice.api.push.exceptions.NonSuccessfulResponseCodeException
import java.io.IOException
import java.util.concurrent.TimeUnit
/**
* Copies and re-encrypts attachments from the attachment cdn to the archive cdn.
*
* Job will fail if the attachment isn't available on the attachment cdn, use [AttachmentUploadJob] to upload first if necessary.
*/
class ArchiveAttachmentJob private constructor(private val attachmentId: AttachmentId, parameters: Parameters) : BaseJob(parameters) {
companion object {
private val TAG = Log.tag(ArchiveAttachmentJob::class.java)
const val KEY = "ArchiveAttachmentJob"
fun enqueueIfPossible(attachmentId: AttachmentId) {
if (!SignalStore.backup().canReadWriteToArchiveCdn) {
return
}
ApplicationDependencies.getJobManager().add(ArchiveAttachmentJob(attachmentId))
}
}
constructor(attachmentId: AttachmentId) : this(
attachmentId = attachmentId,
parameters = Parameters.Builder()
.addConstraint(NetworkConstraint.KEY)
.setLifespan(TimeUnit.DAYS.toMillis(1))
.setMaxAttempts(Parameters.UNLIMITED)
.build()
)
override fun serialize(): ByteArray = ArchiveAttachmentJobData(attachmentId.id).encode()
override fun getFactoryKey(): String = KEY
override fun onRun() {
if (!SignalStore.backup().canReadWriteToArchiveCdn) {
Log.w(TAG, "Do not have permission to read/write to archive cdn")
return
}
val attachment = SignalDatabase.attachments.getAttachment(attachmentId)
if (attachment == null) {
Log.w(TAG, "Unable to find attachment to archive: $attachmentId")
return
}
BackupRepository.archiveMedia(attachment).successOrThrow()
}
override fun onShouldRetry(e: Exception): Boolean {
return e is IOException && e !is NonSuccessfulResponseCodeException
}
override fun onFailure() = Unit
class Factory : Job.Factory<ArchiveAttachmentJob> {
override fun create(parameters: Parameters, serializedData: ByteArray?): ArchiveAttachmentJob {
val jobData = ArchiveAttachmentJobData.ADAPTER.decode(serializedData!!)
return ArchiveAttachmentJob(AttachmentId(jobData.attachmentId), parameters)
}
}
}

350
app/src/main/java/org/thoughtcrime/securesms/jobs/AttachmentDownloadJob.java
View File

@@ -1,350 +0,0 @@
/*
* Copyright 2023 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.thoughtcrime.securesms.jobs;
import android.text.TextUtils;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import androidx.annotation.VisibleForTesting;
import org.greenrobot.eventbus.EventBus;
import org.signal.core.util.Hex;
import org.signal.core.util.logging.Log;
import org.signal.libsignal.protocol.InvalidMacException;
import org.signal.libsignal.protocol.InvalidMessageException;
import org.thoughtcrime.securesms.attachments.Attachment;
import org.thoughtcrime.securesms.attachments.AttachmentId;
import org.thoughtcrime.securesms.attachments.DatabaseAttachment;
import org.thoughtcrime.securesms.blurhash.BlurHash;
import org.thoughtcrime.securesms.database.AttachmentTable;
import org.thoughtcrime.securesms.database.SignalDatabase;
import org.thoughtcrime.securesms.dependencies.ApplicationDependencies;
import org.thoughtcrime.securesms.events.PartProgressEvent;
import org.thoughtcrime.securesms.jobmanager.Job;
import org.thoughtcrime.securesms.jobmanager.JobLogger;
import org.thoughtcrime.securesms.jobmanager.JsonJobData;
import org.thoughtcrime.securesms.jobmanager.impl.NetworkConstraint;
import org.thoughtcrime.securesms.jobmanager.persistence.JobSpec;
import org.thoughtcrime.securesms.mms.MmsException;
import org.thoughtcrime.securesms.notifications.v2.ConversationId;
import org.thoughtcrime.securesms.releasechannel.ReleaseChannel;
import org.thoughtcrime.securesms.s3.S3;
import org.thoughtcrime.securesms.transport.RetryLaterException;
import org.thoughtcrime.securesms.util.AttachmentUtil;
import org.signal.core.util.Base64;
import org.thoughtcrime.securesms.util.FeatureFlags;
import org.thoughtcrime.securesms.util.Util;
import org.whispersystems.signalservice.api.SignalServiceMessageReceiver;
import org.whispersystems.signalservice.api.messages.SignalServiceAttachment;
import org.whispersystems.signalservice.api.messages.SignalServiceAttachmentPointer;
import org.whispersystems.signalservice.api.messages.SignalServiceAttachmentRemoteId;
import org.whispersystems.signalservice.api.push.exceptions.MissingConfigurationException;
import org.whispersystems.signalservice.api.push.exceptions.NonSuccessfulResponseCodeException;
import org.whispersystems.signalservice.api.push.exceptions.PushNetworkException;
import org.whispersystems.signalservice.api.push.exceptions.RangeException;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.util.Objects;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
import okhttp3.Response;
import okhttp3.ResponseBody;
import okio.Okio;
public final class AttachmentDownloadJob extends BaseJob {
public static final String KEY = "AttachmentDownloadJob";
private static final String TAG = Log.tag(AttachmentDownloadJob.class);
private static final String KEY_MESSAGE_ID = "message_id";
private static final String KEY_ATTACHMENT_ID = "part_row_id";
private static final String KEY_MANUAL = "part_manual";
private final long messageId;
private final long attachmentId;
private final boolean manual;
public AttachmentDownloadJob(long messageId, AttachmentId attachmentId, boolean manual) {
this(new Job.Parameters.Builder()
.setQueue(constructQueueString(attachmentId))
.addConstraint(NetworkConstraint.KEY)
.setLifespan(TimeUnit.DAYS.toMillis(1))
.setMaxAttempts(Parameters.UNLIMITED)
.build(),
messageId,
attachmentId,
manual);
}
private AttachmentDownloadJob(@NonNull Job.Parameters parameters, long messageId, AttachmentId attachmentId, boolean manual) {
super(parameters);
this.messageId = messageId;
this.attachmentId = attachmentId.id;
this.manual = manual;
}
@Override
public @Nullable byte[] serialize() {
return new JsonJobData.Builder().putLong(KEY_MESSAGE_ID, messageId)
.putLong(KEY_ATTACHMENT_ID, attachmentId)
.putBoolean(KEY_MANUAL, manual)
.serialize();
}
@Override
public @NonNull String getFactoryKey() {
return KEY;
}
public static String constructQueueString(AttachmentId attachmentId) {
return "AttachmentDownloadJob-" + attachmentId.id;
}
@Override
public void onAdded() {
Log.i(TAG, "onAdded() messageId: " + messageId + " attachmentId: " + attachmentId + " manual: " + manual);
final AttachmentTable database = SignalDatabase.attachments();
final AttachmentId attachmentId = new AttachmentId(this.attachmentId);
final DatabaseAttachment attachment = database.getAttachment(attachmentId);
final boolean pending = attachment != null && attachment.transferState != AttachmentTable.TRANSFER_PROGRESS_DONE
&& attachment.transferState != AttachmentTable.TRANSFER_PROGRESS_PERMANENT_FAILURE;
if (pending && (manual || AttachmentUtil.isAutoDownloadPermitted(context, attachment))) {
Log.i(TAG, "onAdded() Marking attachment progress as 'started'");
database.setTransferState(messageId, attachmentId, AttachmentTable.TRANSFER_PROGRESS_STARTED);
}
}
@Override
public void onRun() throws Exception {
doWork();
if (!SignalDatabase.messages().isStory(messageId)) {
ApplicationDependencies.getMessageNotifier().updateNotification(context, ConversationId.forConversation(0));
}
}
public void doWork() throws IOException, RetryLaterException {
Log.i(TAG, "onRun() messageId: " + messageId + " attachmentId: " + attachmentId + " manual: " + manual);
final AttachmentTable database = SignalDatabase.attachments();
final AttachmentId attachmentId = new AttachmentId(this.attachmentId);
final DatabaseAttachment attachment = database.getAttachment(attachmentId);
if (attachment == null) {
Log.w(TAG, "attachment no longer exists.");
return;
}
if (attachment.isPermanentlyFailed()) {
Log.w(TAG, "Attachment was marked as a permanent failure. Refusing to download.");
return;
}
if (!attachment.isInProgress()) {
Log.w(TAG, "Attachment was already downloaded.");
return;
}
if (!manual && !AttachmentUtil.isAutoDownloadPermitted(context, attachment)) {
Log.w(TAG, "Attachment can't be auto downloaded...");
database.setTransferState(messageId, attachmentId, AttachmentTable.TRANSFER_PROGRESS_PENDING);
return;
}
Log.i(TAG, "Downloading push part " + attachmentId);
database.setTransferState(messageId, attachmentId, AttachmentTable.TRANSFER_PROGRESS_STARTED);
if (attachment.cdnNumber != ReleaseChannel.CDN_NUMBER) {
retrieveAttachment(messageId, attachmentId, attachment);
} else {
retrieveAttachmentForReleaseChannel(messageId, attachmentId, attachment);
}
}
@Override
public void onFailure() {
Log.w(TAG, JobLogger.format(this, "onFailure() messageId: " + messageId + " attachmentId: " + attachmentId + " manual: " + manual));
final AttachmentId attachmentId = new AttachmentId(this.attachmentId);
markFailed(messageId, attachmentId);
}
@Override
protected boolean onShouldRetry(@NonNull Exception exception) {
return exception instanceof PushNetworkException ||
exception instanceof RetryLaterException;
}
private void retrieveAttachment(long messageId,
final AttachmentId attachmentId,
final Attachment attachment)
throws IOException, RetryLaterException
{
long maxReceiveSize = FeatureFlags.maxAttachmentReceiveSizeBytes();
AttachmentTable database = SignalDatabase.attachments();
File attachmentFile = database.getOrCreateTransferFile(attachmentId);
try {
if (attachment.size > maxReceiveSize) {
throw new MmsException("Attachment too large, failing download");
}
SignalServiceMessageReceiver messageReceiver = ApplicationDependencies.getSignalServiceMessageReceiver();
SignalServiceAttachmentPointer pointer = createAttachmentPointer(attachment);
InputStream stream = messageReceiver.retrieveAttachment(pointer,
attachmentFile,
maxReceiveSize,
new SignalServiceAttachment.ProgressListener() {
@Override
public void onAttachmentProgress(long total, long progress) {
EventBus.getDefault().postSticky(new PartProgressEvent(attachment, PartProgressEvent.Type.NETWORK, total, progress));
}
@Override
public boolean shouldCancel() {
return isCanceled();
}
});
database.finalizeAttachmentAfterDownload(messageId, attachmentId, stream);
} catch (RangeException e) {
Log.w(TAG, "Range exception, file size " + attachmentFile.length(), e);
if (attachmentFile.delete()) {
Log.i(TAG, "Deleted temp download file to recover");
throw new RetryLaterException(e);
} else {
throw new IOException("Failed to delete temp download file following range exception");
}
} catch (InvalidPartException | NonSuccessfulResponseCodeException | MmsException | MissingConfigurationException e) {
Log.w(TAG, "Experienced exception while trying to download an attachment.", e);
markFailed(messageId, attachmentId);
} catch (InvalidMessageException e) {
Log.w(TAG, "Experienced an InvalidMessageException while trying to download an attachment.", e);
if (e.getCause() instanceof InvalidMacException) {
Log.w(TAG, "Detected an invalid mac. Treating as a permanent failure.");
markPermanentlyFailed(messageId, attachmentId);
} else {
markFailed(messageId, attachmentId);
}
}
}
private SignalServiceAttachmentPointer createAttachmentPointer(Attachment attachment) throws InvalidPartException {
if (TextUtils.isEmpty(attachment.remoteLocation)) {
throw new InvalidPartException("empty content id");
}
if (TextUtils.isEmpty(attachment.remoteKey)) {
throw new InvalidPartException("empty encrypted key");
}
try {
final SignalServiceAttachmentRemoteId remoteId = SignalServiceAttachmentRemoteId.from(attachment.remoteLocation);
final byte[] key = Base64.decode(attachment.remoteKey);
if (attachment.remoteDigest != null) {
Log.i(TAG, "Downloading attachment with digest: " + Hex.toString(attachment.remoteDigest));
} else {
Log.i(TAG, "Downloading attachment with no digest...");
}
return new SignalServiceAttachmentPointer(attachment.cdnNumber, remoteId, null, key,
Optional.of(Util.toIntExact(attachment.size)),
Optional.empty(),
0, 0,
Optional.ofNullable(attachment.remoteDigest),
Optional.ofNullable(attachment.getIncrementalDigest()),
attachment.incrementalMacChunkSize,
Optional.ofNullable(attachment.fileName),
attachment.voiceNote,
attachment.borderless,
attachment.videoGif,
Optional.empty(),
Optional.ofNullable(attachment.blurHash).map(BlurHash::getHash),
attachment.uploadTimestamp);
} catch (IOException | ArithmeticException e) {
Log.w(TAG, e);
throw new InvalidPartException(e);
}
}
private void retrieveAttachmentForReleaseChannel(long messageId,
final AttachmentId attachmentId,
final Attachment attachment)
throws IOException
{
try (Response response = S3.getObject(Objects.requireNonNull(attachment.fileName))) {
ResponseBody body = response.body();
if (body != null) {
if (body.contentLength() > FeatureFlags.maxAttachmentReceiveSizeBytes()) {
throw new MmsException("Attachment too large, failing download");
}
SignalDatabase.attachments().finalizeAttachmentAfterDownload(messageId, attachmentId, Okio.buffer(body.source()).inputStream());
}
} catch (MmsException e) {
Log.w(TAG, "Experienced exception while trying to download an attachment.", e);
markFailed(messageId, attachmentId);
}
}
private void markFailed(long messageId, AttachmentId attachmentId) {
try {
AttachmentTable database = SignalDatabase.attachments();
database.setTransferProgressFailed(attachmentId, messageId);
} catch (MmsException e) {
Log.w(TAG, e);
}
}
private void markPermanentlyFailed(long messageId, AttachmentId attachmentId) {
try {
AttachmentTable database = SignalDatabase.attachments();
database.setTransferProgressPermanentFailure(attachmentId, messageId);
} catch (MmsException e) {
Log.w(TAG, e);
}
}
public static boolean jobSpecMatchesAttachmentId(@NonNull JobSpec jobSpec, @NonNull AttachmentId attachmentId) {
if (!KEY.equals(jobSpec.getFactoryKey())) {
return false;
}
final byte[] serializedData = jobSpec.getSerializedData();
if (serializedData == null) {
return false;
}
JsonJobData data = JsonJobData.deserialize(serializedData);
final AttachmentId parsed = new AttachmentId(data.getLong(KEY_ATTACHMENT_ID));
return attachmentId.equals(parsed);
}
@VisibleForTesting
static class InvalidPartException extends Exception {
InvalidPartException(String s) {super(s);}
InvalidPartException(Exception e) {super(e);}
}
public static final class Factory implements Job.Factory<AttachmentDownloadJob> {
@Override
public @NonNull AttachmentDownloadJob create(@NonNull Parameters parameters, @Nullable byte[] serializedData) {
JsonJobData data = JsonJobData.deserialize(serializedData);
return new AttachmentDownloadJob(parameters,
data.getLong(KEY_MESSAGE_ID),
new AttachmentId(data.getLong(KEY_ATTACHMENT_ID)),
data.getBoolean(KEY_MANUAL));
}
}
}

424
app/src/main/java/org/thoughtcrime/securesms/jobs/AttachmentDownloadJob.kt Normal file
View File

@@ -0,0 +1,424 @@
/*
* Copyright 2023 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.thoughtcrime.securesms.jobs
import android.text.TextUtils
import androidx.annotation.VisibleForTesting
import okio.Source
import okio.buffer
import org.greenrobot.eventbus.EventBus
import org.signal.core.util.Base64
import org.signal.core.util.Hex
import org.signal.core.util.logging.Log
import org.signal.libsignal.protocol.InvalidMacException
import org.signal.libsignal.protocol.InvalidMessageException
import org.thoughtcrime.securesms.attachments.Attachment
import org.thoughtcrime.securesms.attachments.AttachmentId
import org.thoughtcrime.securesms.attachments.Cdn
import org.thoughtcrime.securesms.attachments.DatabaseAttachment
import org.thoughtcrime.securesms.backup.v2.BackupRepository
import org.thoughtcrime.securesms.database.AttachmentTable
import org.thoughtcrime.securesms.database.SignalDatabase
import org.thoughtcrime.securesms.dependencies.ApplicationDependencies
import org.thoughtcrime.securesms.events.PartProgressEvent
import org.thoughtcrime.securesms.jobmanager.Job
import org.thoughtcrime.securesms.jobmanager.JobLogger.format
import org.thoughtcrime.securesms.jobmanager.JsonJobData
import org.thoughtcrime.securesms.jobmanager.impl.NetworkConstraint
import org.thoughtcrime.securesms.jobmanager.persistence.JobSpec
import org.thoughtcrime.securesms.keyvalue.SignalStore
import org.thoughtcrime.securesms.mms.MmsException
import org.thoughtcrime.securesms.notifications.v2.ConversationId.Companion.forConversation
import org.thoughtcrime.securesms.s3.S3
import org.thoughtcrime.securesms.transport.RetryLaterException
import org.thoughtcrime.securesms.util.AttachmentUtil
import org.thoughtcrime.securesms.util.FeatureFlags
import org.thoughtcrime.securesms.util.Util
import org.whispersystems.signalservice.api.backup.MediaName
import org.whispersystems.signalservice.api.messages.SignalServiceAttachment
import org.whispersystems.signalservice.api.messages.SignalServiceAttachmentPointer
import org.whispersystems.signalservice.api.messages.SignalServiceAttachmentRemoteId
import org.whispersystems.signalservice.api.push.exceptions.MissingConfigurationException
import org.whispersystems.signalservice.api.push.exceptions.NonSuccessfulResponseCodeException
import org.whispersystems.signalservice.api.push.exceptions.PushNetworkException
import org.whispersystems.signalservice.api.push.exceptions.RangeException
import java.io.File
import java.io.IOException
import java.util.Optional
import java.util.concurrent.TimeUnit
/**
* Download attachment from locations as specified in their record.
*/
class AttachmentDownloadJob private constructor(
parameters: Parameters,
private val messageId: Long,
attachmentId: AttachmentId,
private val manual: Boolean,
private var forceArchiveDownload: Boolean
) : BaseJob(parameters) {
companion object {
const val KEY = "AttachmentDownloadJob"
private val TAG = Log.tag(AttachmentDownloadJob::class.java)
private const val KEY_MESSAGE_ID = "message_id"
private const val KEY_ATTACHMENT_ID = "part_row_id"
private const val KEY_MANUAL = "part_manual"
private const val KEY_FORCE_ARCHIVE = "force_archive"
@JvmStatic
fun constructQueueString(attachmentId: AttachmentId): String {
return "AttachmentDownloadJob-" + attachmentId.id
}
fun jobSpecMatchesAttachmentId(jobSpec: JobSpec, attachmentId: AttachmentId): Boolean {
if (KEY != jobSpec.factoryKey) {
return false
}
val serializedData = jobSpec.serializedData ?: return false
val data = JsonJobData.deserialize(serializedData)
val parsed = AttachmentId(data.getLong(KEY_ATTACHMENT_ID))
return attachmentId == parsed
}
}
private val attachmentId: Long
constructor(messageId: Long, attachmentId: AttachmentId, manual: Boolean, forceArchiveDownload: Boolean = false) : this(
Parameters.Builder()
.setQueue(constructQueueString(attachmentId))
.addConstraint(NetworkConstraint.KEY)
.setLifespan(TimeUnit.DAYS.toMillis(1))
.setMaxAttempts(Parameters.UNLIMITED)
.build(),
messageId,
attachmentId,
manual,
forceArchiveDownload
)
init {
this.attachmentId = attachmentId.id
}
override fun serialize(): ByteArray? {
return JsonJobData.Builder()
.putLong(KEY_MESSAGE_ID, messageId)
.putLong(KEY_ATTACHMENT_ID, attachmentId)
.putBoolean(KEY_MANUAL, manual)
.putBoolean(KEY_FORCE_ARCHIVE, forceArchiveDownload)
.serialize()
}
override fun getFactoryKey(): String {
return KEY
}
override fun onAdded() {
Log.i(TAG, "onAdded() messageId: $messageId attachmentId: $attachmentId manual: $manual")
val attachmentId = AttachmentId(attachmentId)
val attachment = SignalDatabase.attachments.getAttachment(attachmentId)
val pending = attachment != null && attachment.transferState != AttachmentTable.TRANSFER_PROGRESS_DONE && attachment.transferState != AttachmentTable.TRANSFER_PROGRESS_PERMANENT_FAILURE
if (pending && (manual || AttachmentUtil.isAutoDownloadPermitted(context, attachment))) {
Log.i(TAG, "onAdded() Marking attachment progress as 'started'")
SignalDatabase.attachments.setTransferState(messageId, attachmentId, AttachmentTable.TRANSFER_PROGRESS_STARTED)
}
}
@Throws(Exception::class)
public override fun onRun() {
doWork()
if (!SignalDatabase.messages.isStory(messageId)) {
ApplicationDependencies.getMessageNotifier().updateNotification(context, forConversation(0))
}
}
@Throws(IOException::class, RetryLaterException::class)
fun doWork() {
Log.i(TAG, "onRun() messageId: $messageId attachmentId: $attachmentId manual: $manual")
val attachmentId = AttachmentId(attachmentId)
val attachment = SignalDatabase.attachments.getAttachment(attachmentId)
if (attachment == null) {
Log.w(TAG, "attachment no longer exists.")
return
}
if (attachment.isPermanentlyFailed) {
Log.w(TAG, "Attachment was marked as a permanent failure. Refusing to download.")
return
}
if (!attachment.isInProgress) {
Log.w(TAG, "Attachment was already downloaded.")
return
}
if (!manual && !AttachmentUtil.isAutoDownloadPermitted(context, attachment)) {
Log.w(TAG, "Attachment can't be auto downloaded...")
SignalDatabase.attachments.setTransferState(messageId, attachmentId, AttachmentTable.TRANSFER_PROGRESS_PENDING)
return
}
Log.i(TAG, "Downloading push part $attachmentId")
SignalDatabase.attachments.setTransferState(messageId, attachmentId, AttachmentTable.TRANSFER_PROGRESS_STARTED)
when (attachment.cdn) {
Cdn.S3 -> retrieveAttachmentForReleaseChannel(messageId, attachmentId, attachment)
else -> retrieveAttachment(messageId, attachmentId, attachment)
}
}
override fun onFailure() {
Log.w(TAG, format(this, "onFailure() messageId: $messageId attachmentId: $attachmentId manual: $manual"))
val attachmentId = AttachmentId(attachmentId)
markFailed(messageId, attachmentId)
}
override fun onShouldRetry(exception: Exception): Boolean {
return exception is PushNetworkException ||
exception is RetryLaterException
}
@Throws(IOException::class, RetryLaterException::class)
private fun retrieveAttachment(
messageId: Long,
attachmentId: AttachmentId,
attachment: DatabaseAttachment
) {
val maxReceiveSize: Long = FeatureFlags.maxAttachmentReceiveSizeBytes()
val attachmentFile: File = SignalDatabase.attachments.getOrCreateTransferFile(attachmentId)
var archiveFile: File? = null
var useArchiveCdn = false
try {
if (attachment.size > maxReceiveSize) {
throw MmsException("Attachment too large, failing download")
}
useArchiveCdn = if (SignalStore.backup().canReadWriteToArchiveCdn && (forceArchiveDownload || attachment.remoteLocation == null)) {
if (attachment.archiveMediaName.isNullOrEmpty()) {
throw InvalidPartException("Invalid attachment configuration")
}
true
} else {
false
}
val messageReceiver = ApplicationDependencies.getSignalServiceMessageReceiver()
val pointer = createAttachmentPointer(attachment, useArchiveCdn)
val progressListener = object : SignalServiceAttachment.ProgressListener {
override fun onAttachmentProgress(total: Long, progress: Long) {
EventBus.getDefault().postSticky(PartProgressEvent(attachment, PartProgressEvent.Type.NETWORK, total, progress))
}
override fun shouldCancel(): Boolean {
return this@AttachmentDownloadJob.isCanceled
}
}
val stream = if (useArchiveCdn) {
archiveFile = SignalDatabase.attachments.getOrCreateArchiveTransferFile(attachmentId)
val cdnCredentials = BackupRepository.getCdnReadCredentials().successOrThrow().headers
messageReceiver
.retrieveArchivedAttachment(
SignalStore.svr().getOrCreateMasterKey().deriveBackupKey().deriveMediaSecrets(MediaName(attachment.archiveMediaName!!)),
cdnCredentials,
archiveFile,
pointer,
attachmentFile,
maxReceiveSize,
progressListener
)
} else {
messageReceiver
.retrieveAttachment(
pointer,
attachmentFile,
maxReceiveSize,
progressListener
)
}
SignalDatabase.attachments.finalizeAttachmentAfterDownload(messageId, attachmentId, stream)
} catch (e: RangeException) {
val transferFile = archiveFile ?: attachmentFile
Log.w(TAG, "Range exception, file size " + transferFile.length(), e)
if (transferFile.delete()) {
Log.i(TAG, "Deleted temp download file to recover")
throw RetryLaterException(e)
} else {
throw IOException("Failed to delete temp download file following range exception")
}
} catch (e: InvalidPartException) {
Log.w(TAG, "Experienced exception while trying to download an attachment.", e)
markFailed(messageId, attachmentId)
} catch (e: NonSuccessfulResponseCodeException) {
if (SignalStore.backup().canReadWriteToArchiveCdn) {
if (e.code == 404 && !useArchiveCdn && attachment.archiveMediaName?.isNotEmpty() == true) {
Log.i(TAG, "Retrying download from archive CDN")
forceArchiveDownload = true
retrieveAttachment(messageId, attachmentId, attachment)
return
} else if (e.code == 401 && useArchiveCdn) {
SignalStore.backup().cdnReadCredentials = null
throw RetryLaterException(e)
}
}
Log.w(TAG, "Experienced exception while trying to download an attachment.", e)
markFailed(messageId, attachmentId)
} catch (e: MmsException) {
Log.w(TAG, "Experienced exception while trying to download an attachment.", e)
markFailed(messageId, attachmentId)
} catch (e: MissingConfigurationException) {
Log.w(TAG, "Experienced exception while trying to download an attachment.", e)
markFailed(messageId, attachmentId)
} catch (e: InvalidMessageException) {
Log.w(TAG, "Experienced an InvalidMessageException while trying to download an attachment.", e)
if (e.cause is InvalidMacException) {
Log.w(TAG, "Detected an invalid mac. Treating as a permanent failure.")
markPermanentlyFailed(messageId, attachmentId)
} else {
markFailed(messageId, attachmentId)
}
}
}
@Throws(InvalidPartException::class)
private fun createAttachmentPointer(attachment: DatabaseAttachment, useArchiveCdn: Boolean): SignalServiceAttachmentPointer {
if (TextUtils.isEmpty(attachment.remoteKey)) {
throw InvalidPartException("empty encrypted key")
}
return try {
val remoteData: RemoteData = if (useArchiveCdn) {
val backupKey = SignalStore.svr().getOrCreateMasterKey().deriveBackupKey()
val backupDirectories = BackupRepository.getCdnBackupDirectories().successOrThrow()
RemoteData(
remoteId = SignalServiceAttachmentRemoteId.Backup(
backupDir = backupDirectories.backupDir,
mediaDir = backupDirectories.mediaDir,
mediaId = backupKey.deriveMediaId(MediaName(attachment.archiveMediaName!!)).encode()
),
cdnNumber = attachment.archiveCdn
)
} else {
if (attachment.remoteLocation.isNullOrEmpty()) {
throw InvalidPartException("empty content id")
}
RemoteData(
remoteId = SignalServiceAttachmentRemoteId.from(attachment.remoteLocation),
cdnNumber = attachment.cdn.cdnNumber
)
}
val key = Base64.decode(attachment.remoteKey!!)
if (attachment.remoteDigest != null) {
Log.i(TAG, "Downloading attachment with digest: " + Hex.toString(attachment.remoteDigest))
} else {
Log.i(TAG, "Downloading attachment with no digest...")
}
SignalServiceAttachmentPointer(
remoteData.cdnNumber,
remoteData.remoteId,
null,
key,
Optional.of(Util.toIntExact(attachment.size)),
Optional.empty(),
0,
0,
Optional.ofNullable(attachment.remoteDigest),
Optional.ofNullable(attachment.getIncrementalDigest()),
attachment.incrementalMacChunkSize,
Optional.ofNullable(attachment.fileName),
attachment.voiceNote,
attachment.borderless,
attachment.videoGif,
Optional.empty(),
Optional.ofNullable(attachment.blurHash).map { it.hash },
attachment.uploadTimestamp
)
} catch (e: IOException) {
Log.w(TAG, e)
throw InvalidPartException(e)
} catch (e: ArithmeticException) {
Log.w(TAG, e)
throw InvalidPartException(e)
}
}
@Throws(IOException::class)
private fun retrieveAttachmentForReleaseChannel(
messageId: Long,
attachmentId: AttachmentId,
attachment: Attachment
) {
try {
S3.getObject(attachment.fileName!!).use { response ->
val body = response.body()
if (body != null) {
if (body.contentLength() > FeatureFlags.maxAttachmentReceiveSizeBytes()) {
throw MmsException("Attachment too large, failing download")
}
SignalDatabase.attachments.finalizeAttachmentAfterDownload(messageId, attachmentId, (body.source() as Source).buffer().inputStream())
}
}
} catch (e: MmsException) {
Log.w(TAG, "Experienced exception while trying to download an attachment.", e)
markFailed(messageId, attachmentId)
}
}
private fun markFailed(messageId: Long, attachmentId: AttachmentId) {
try {
SignalDatabase.attachments.setTransferProgressFailed(attachmentId, messageId)
} catch (e: MmsException) {
Log.w(TAG, e)
}
}
private fun markPermanentlyFailed(messageId: Long, attachmentId: AttachmentId) {
try {
SignalDatabase.attachments.setTransferProgressPermanentFailure(attachmentId, messageId)
} catch (e: MmsException) {
Log.w(TAG, e)
}
}
@VisibleForTesting
internal class InvalidPartException : Exception {
constructor(s: String?) : super(s)
constructor(e: Exception?) : super(e)
}
private data class RemoteData(val remoteId: SignalServiceAttachmentRemoteId, val cdnNumber: Int)
class Factory : Job.Factory<AttachmentDownloadJob?> {
override fun create(parameters: Parameters, serializedData: ByteArray?): AttachmentDownloadJob {
val data = JsonJobData.deserialize(serializedData)
return AttachmentDownloadJob(
parameters = parameters,
messageId = data.getLong(KEY_MESSAGE_ID),
attachmentId = AttachmentId(data.getLong(KEY_ATTACHMENT_ID)),
manual = data.getBoolean(KEY_MANUAL),
forceArchiveDownload = data.getBooleanOrDefault(KEY_FORCE_ARCHIVE, false)
)
}
}
}

2
app/src/main/java/org/thoughtcrime/securesms/jobs/AvatarGroupsV1DownloadJob.java
View File

@@ -85,7 +85,7 @@ public final class AvatarGroupsV1DownloadJob extends BaseJob {
attachment.deleteOnExit();
SignalServiceMessageReceiver receiver = ApplicationDependencies.getSignalServiceMessageReceiver();
SignalServiceAttachmentPointer pointer = new SignalServiceAttachmentPointer(0, new SignalServiceAttachmentRemoteId(avatarId), contentType, key, Optional.of(0), Optional.empty(), 0, 0, digest, Optional.empty(), 0, fileName, false, false, false, Optional.empty(), Optional.empty(), System.currentTimeMillis());
SignalServiceAttachmentPointer pointer = new SignalServiceAttachmentPointer(0, new SignalServiceAttachmentRemoteId.V2(avatarId), contentType, key, Optional.of(0), Optional.empty(), 0, 0, digest, Optional.empty(), 0, fileName, false, false, false, Optional.empty(), Optional.empty(), System.currentTimeMillis());
InputStream inputStream = receiver.retrieveAttachment(pointer, attachment, AvatarHelper.AVATAR_DOWNLOAD_FAILSAFE_MAX_SIZE);
AvatarHelper.setAvatar(context, record.get().getRecipientId(), inputStream);

113
app/src/main/java/org/thoughtcrime/securesms/jobs/BackupMessagesJob.kt Normal file
View File

@@ -0,0 +1,113 @@
/*
* Copyright 2024 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.thoughtcrime.securesms.jobs
import android.database.Cursor
import org.signal.core.util.logging.Log
import org.thoughtcrime.securesms.BuildConfig
import org.thoughtcrime.securesms.attachments.DatabaseAttachment
import org.thoughtcrime.securesms.backup.v2.BackupRepository
import org.thoughtcrime.securesms.database.SignalDatabase
import org.thoughtcrime.securesms.dependencies.ApplicationDependencies
import org.thoughtcrime.securesms.jobmanager.Job
import org.thoughtcrime.securesms.jobmanager.impl.NetworkConstraint
import org.thoughtcrime.securesms.keyvalue.SignalStore
import org.thoughtcrime.securesms.providers.BlobProvider
import org.whispersystems.signalservice.api.NetworkResult
import java.io.FileInputStream
import java.io.FileOutputStream
/**
* Job that is responsible for exporting the DB as a backup proto and
* also uploading the resulting proto.
*/
class BackupMessagesJob private constructor(parameters: Parameters) : BaseJob(parameters) {
companion object {
private val TAG = Log.tag(BackupMessagesJob::class.java)
const val KEY = "BackupMessagesJob"
}
constructor() : this(
Parameters.Builder()
.addConstraint(NetworkConstraint.KEY)
.setMaxAttempts(Parameters.UNLIMITED)
.setMaxInstancesForFactory(2)
.build()
)
override fun serialize(): ByteArray? = null
override fun getFactoryKey(): String = KEY
override fun onFailure() = Unit
private fun archiveAttachments() {
if (BuildConfig.MESSAGE_BACKUP_RESTORE_ENABLED) {
SignalStore.backup().canReadWriteToArchiveCdn = true
}
val batchSize = 100
SignalDatabase.attachments.getArchivableAttachments().use { cursor ->
while (!cursor.isAfterLast) {
val attachments = cursor.readAttachmentBatch(batchSize)
when (val archiveResult = BackupRepository.archiveMedia(attachments)) {
is NetworkResult.Success -> {
for (success in archiveResult.result.sourceNotFoundResponses) {
val attachmentId = archiveResult.result.mediaIdToAttachmentId(success.mediaId)
ApplicationDependencies
.getJobManager()
.startChain(AttachmentUploadJob(attachmentId))
.then(ArchiveAttachmentJob(attachmentId))
.enqueue()
}
}
else -> {
Log.e(TAG, "Failed to archive $archiveResult")
}
}
}
}
}
private fun Cursor.readAttachmentBatch(batchSize: Int): List<DatabaseAttachment> {
val attachments = ArrayList<DatabaseAttachment>()
for (i in 0 until batchSize) {
if (this.moveToNext()) {
attachments.addAll(SignalDatabase.attachments.getAttachments(this))
} else {
break
}
}
return attachments
}
override fun onRun() {
val tempBackupFile = BlobProvider.getInstance().forNonAutoEncryptingSingleSessionOnDisk(ApplicationDependencies.getApplication())
val outputStream = FileOutputStream(tempBackupFile)
BackupRepository.export(outputStream = outputStream, append = { tempBackupFile.appendBytes(it) }, plaintext = false)
FileInputStream(tempBackupFile).use {
BackupRepository.uploadBackupFile(it, tempBackupFile.length())
}
archiveAttachments()
if (!tempBackupFile.delete()) {
Log.e(TAG, "Failed to delete temp backup file")
}
}
override fun onShouldRetry(e: Exception): Boolean = false
class Factory : Job.Factory<BackupMessagesJob> {
override fun create(parameters: Parameters, serializedData: ByteArray?): BackupMessagesJob {
return BackupMessagesJob(parameters)
}
}
}

105
app/src/main/java/org/thoughtcrime/securesms/jobs/BackupRestoreJob.kt Normal file
View File

@@ -0,0 +1,105 @@
/*
* Copyright 2024 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.thoughtcrime.securesms.jobs
import org.signal.core.util.logging.Log
import org.signal.libsignal.zkgroup.profiles.ProfileKey
import org.thoughtcrime.securesms.R
import org.thoughtcrime.securesms.backup.RestoreState
import org.thoughtcrime.securesms.backup.v2.BackupRepository
import org.thoughtcrime.securesms.dependencies.ApplicationDependencies
import org.thoughtcrime.securesms.jobmanager.Job
import org.thoughtcrime.securesms.jobmanager.impl.NetworkConstraint
import org.thoughtcrime.securesms.keyvalue.SignalStore
import org.thoughtcrime.securesms.net.NotPushRegisteredException
import org.thoughtcrime.securesms.providers.BlobProvider
import org.thoughtcrime.securesms.recipients.Recipient
import org.thoughtcrime.securesms.service.BackupProgressService
import org.whispersystems.signalservice.api.messages.SignalServiceAttachment.ProgressListener
import java.io.IOException
/**
* Job that is responsible for restoring a backup from the server
*/
class BackupRestoreJob private constructor(parameters: Parameters) : BaseJob(parameters) {
companion object {
private val TAG = Log.tag(BackupRestoreJob::class.java)
const val KEY = "BackupRestoreJob"
}
constructor() : this(
Parameters.Builder()
.addConstraint(NetworkConstraint.KEY)
.setMaxAttempts(Parameters.UNLIMITED)
.setMaxInstancesForFactory(1)
.build()
)
override fun serialize(): ByteArray? = null
override fun getFactoryKey(): String = KEY
override fun onFailure() = Unit
override fun onAdded() {
SignalStore.backup().restoreState = RestoreState.PENDING
}
override fun onRun() {
if (!SignalStore.account().isRegistered) {
Log.e(TAG, "Not registered, cannot restore!")
throw NotPushRegisteredException()
}
BackupProgressService.start(context, context.getString(R.string.BackupProgressService_title)).use {
restore(it)
}
}
private fun restore(controller: BackupProgressService.Controller) {
SignalStore.backup().restoreState = RestoreState.RESTORING_DB
val progressListener = object : ProgressListener {
override fun onAttachmentProgress(total: Long, progress: Long) {
controller.update(
title = context.getString(R.string.BackupProgressService_title_downloading),
progress = progress.toFloat() / total.toFloat(),
indeterminate = false
)
}
override fun shouldCancel() = isCanceled
}
val tempBackupFile = BlobProvider.getInstance().forNonAutoEncryptingSingleSessionOnDisk(ApplicationDependencies.getApplication())
if (!BackupRepository.downloadBackupFile(tempBackupFile, progressListener)) {
Log.e(TAG, "Failed to download backup file")
throw IOException()
}
controller.update(
title = context.getString(R.string.BackupProgressService_title),
progress = 0f,
indeterminate = true
)
val self = Recipient.self()
val selfData = BackupRepository.SelfData(self.aci.get(), self.pni.get(), self.e164.get(), ProfileKey(self.profileKey))
BackupRepository.import(length = tempBackupFile.length(), inputStreamFactory = tempBackupFile::inputStream, selfData = selfData, plaintext = false)
SignalStore.backup().restoreState = RestoreState.RESTORING_MEDIA
}
override fun onShouldRetry(e: Exception): Boolean = false
class Factory : Job.Factory<BackupRestoreJob> {
override fun create(parameters: Parameters, serializedData: ByteArray?): BackupRestoreJob {
return BackupRestoreJob(parameters)
}
}
}

83
app/src/main/java/org/thoughtcrime/securesms/jobs/BackupRestoreMediaJob.kt Normal file
View File

@@ -0,0 +1,83 @@
/*
* Copyright 2024 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.thoughtcrime.securesms.jobs
import org.signal.core.util.logging.Log
import org.thoughtcrime.securesms.database.SignalDatabase
import org.thoughtcrime.securesms.database.model.MmsMessageRecord
import org.thoughtcrime.securesms.dependencies.ApplicationDependencies
import org.thoughtcrime.securesms.jobmanager.Job
import org.thoughtcrime.securesms.jobmanager.impl.NetworkConstraint
import org.thoughtcrime.securesms.keyvalue.SignalStore
import org.thoughtcrime.securesms.net.NotPushRegisteredException
import kotlin.time.Duration.Companion.days
/**
* Job that is responsible for enqueueing attachment download
* jobs upon restore.
*/
class BackupRestoreMediaJob private constructor(parameters: Parameters) : BaseJob(parameters) {
companion object {
private val TAG = Log.tag(BackupRestoreMediaJob::class.java)
const val KEY = "BackupRestoreMediaJob"
}
constructor() : this(
Parameters.Builder()
.addConstraint(NetworkConstraint.KEY)
.setMaxAttempts(Parameters.UNLIMITED)
.setMaxInstancesForFactory(2)
.build()
)
override fun serialize(): ByteArray? = null
override fun getFactoryKey(): String = KEY
override fun onFailure() = Unit
override fun onRun() {
if (!SignalStore.account().isRegistered) {
Log.e(TAG, "Not registered, cannot restore!")
throw NotPushRegisteredException()
}
val jobManager = ApplicationDependencies.getJobManager()
val batchSize = 100
val restoreTime = System.currentTimeMillis()
var restoreJobBatch: List<RestoreAttachmentJob>
do {
val attachmentBatch = SignalDatabase.attachments.getRestorableAttachments(batchSize)
val messageIds = attachmentBatch.map { it.mmsId }.toSet()
val messageMap = SignalDatabase.messages.getMessages(messageIds).associate { it.id to (it as MmsMessageRecord) }
restoreJobBatch = SignalDatabase.attachments.getRestorableAttachments(batchSize).map { attachment ->
val message = messageMap[attachment.mmsId]!!
RestoreAttachmentJob(
messageId = attachment.mmsId,
attachmentId = attachment.attachmentId,
manual = false,
forceArchiveDownload = true,
fullSize = shouldRestoreFullSize(message, restoreTime, optimizeStorage = SignalStore.backup().optimizeStorage)
)
}
jobManager.addAll(restoreJobBatch)
} while (restoreJobBatch.isNotEmpty())
}
private fun shouldRestoreFullSize(message: MmsMessageRecord, restoreTime: Long, optimizeStorage: Boolean): Boolean {
return ((restoreTime - message.dateSent) < 30.days.inWholeMilliseconds) || !optimizeStorage
}
override fun onShouldRetry(e: Exception): Boolean = false
class Factory : Job.Factory<BackupRestoreMediaJob> {
override fun create(parameters: Parameters, serializedData: ByteArray?): BackupRestoreMediaJob {
return BackupRestoreMediaJob(parameters)
}
}
}

5
app/src/main/java/org/thoughtcrime/securesms/jobs/JobManagerFactories.java
View File

@@ -100,6 +100,7 @@ public final class JobManagerFactories {
return new HashMap<String, Job.Factory>() {{
put(AccountConsistencyWorkerJob.KEY, new AccountConsistencyWorkerJob.Factory());
put(AnalyzeDatabaseJob.KEY, new AnalyzeDatabaseJob.Factory());
put(ArchiveAttachmentJob.KEY, new ArchiveAttachmentJob.Factory());
put(AttachmentCompressionJob.KEY, new AttachmentCompressionJob.Factory());
put(AttachmentCopyJob.KEY, new AttachmentCopyJob.Factory());
put(AttachmentDownloadJob.KEY, new AttachmentDownloadJob.Factory());
@@ -109,6 +110,9 @@ public final class JobManagerFactories {
put(AutomaticSessionResetJob.KEY, new AutomaticSessionResetJob.Factory());
put(AvatarGroupsV1DownloadJob.KEY, new AvatarGroupsV1DownloadJob.Factory());
put(AvatarGroupsV2DownloadJob.KEY, new AvatarGroupsV2DownloadJob.Factory());
put(BackupMessagesJob.KEY, new BackupMessagesJob.Factory());
put(BackupRestoreJob.KEY, new BackupRestoreJob.Factory());
put(BackupRestoreMediaJob.KEY, new BackupRestoreMediaJob.Factory());
put(BoostReceiptRequestResponseJob.KEY, new BoostReceiptRequestResponseJob.Factory());
put(CallLinkPeekJob.KEY, new CallLinkPeekJob.Factory());
put(CallLinkUpdateSendJob.KEY, new CallLinkUpdateSendJob.Factory());
@@ -193,6 +197,7 @@ public final class JobManagerFactories {
put(ResumableUploadSpecJob.KEY, new ResumableUploadSpecJob.Factory());
put(RequestGroupV2InfoWorkerJob.KEY, new RequestGroupV2InfoWorkerJob.Factory());
put(RequestGroupV2InfoJob.KEY, new RequestGroupV2InfoJob.Factory());
put(RestoreAttachmentJob.KEY, new RestoreAttachmentJob.Factory());
put(RetrieveProfileAvatarJob.KEY, new RetrieveProfileAvatarJob.Factory());
put(RetrieveProfileJob.KEY, new RetrieveProfileJob.Factory());
put(RetrieveRemoteAnnouncementsJob.KEY, new RetrieveRemoteAnnouncementsJob.Factory());

2
app/src/main/java/org/thoughtcrime/securesms/jobs/PushSendJob.java
View File

@@ -288,7 +288,7 @@ public abstract class PushSendJob extends SendJob {
}
}
return new SignalServiceAttachmentPointer(attachment.cdnNumber,
return new SignalServiceAttachmentPointer(attachment.cdn.getCdnNumber(),
remoteId,
attachment.contentType,
key,

400
app/src/main/java/org/thoughtcrime/securesms/jobs/RestoreAttachmentJob.kt Normal file
View File

@@ -0,0 +1,400 @@
/*
* Copyright 2023 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.thoughtcrime.securesms.jobs
import android.text.TextUtils
import androidx.annotation.VisibleForTesting
import org.greenrobot.eventbus.EventBus
import org.signal.core.util.Base64
import org.signal.core.util.Hex
import org.signal.core.util.logging.Log
import org.signal.libsignal.protocol.InvalidMacException
import org.signal.libsignal.protocol.InvalidMessageException
import org.thoughtcrime.securesms.attachments.AttachmentId
import org.thoughtcrime.securesms.attachments.DatabaseAttachment
import org.thoughtcrime.securesms.backup.v2.BackupRepository
import org.thoughtcrime.securesms.database.AttachmentTable
import org.thoughtcrime.securesms.database.SignalDatabase
import org.thoughtcrime.securesms.dependencies.ApplicationDependencies
import org.thoughtcrime.securesms.events.PartProgressEvent
import org.thoughtcrime.securesms.jobmanager.Job
import org.thoughtcrime.securesms.jobmanager.JobLogger.format
import org.thoughtcrime.securesms.jobmanager.JsonJobData
import org.thoughtcrime.securesms.jobmanager.impl.NetworkConstraint
import org.thoughtcrime.securesms.jobmanager.persistence.JobSpec
import org.thoughtcrime.securesms.keyvalue.SignalStore
import org.thoughtcrime.securesms.mms.MmsException
import org.thoughtcrime.securesms.notifications.v2.ConversationId.Companion.forConversation
import org.thoughtcrime.securesms.transport.RetryLaterException
import org.thoughtcrime.securesms.util.FeatureFlags
import org.thoughtcrime.securesms.util.Util
import org.whispersystems.signalservice.api.backup.MediaName
import org.whispersystems.signalservice.api.messages.SignalServiceAttachment
import org.whispersystems.signalservice.api.messages.SignalServiceAttachmentPointer
import org.whispersystems.signalservice.api.messages.SignalServiceAttachmentRemoteId
import org.whispersystems.signalservice.api.push.exceptions.MissingConfigurationException
import org.whispersystems.signalservice.api.push.exceptions.NonSuccessfulResponseCodeException
import org.whispersystems.signalservice.api.push.exceptions.PushNetworkException
import org.whispersystems.signalservice.api.push.exceptions.RangeException
import java.io.File
import java.io.IOException
import java.util.Optional
import java.util.concurrent.TimeUnit
/**
* Download attachment from locations as specified in their record.
*/
class RestoreAttachmentJob private constructor(
parameters: Parameters,
private val messageId: Long,
attachmentId: AttachmentId,
private val manual: Boolean,
private var forceArchiveDownload: Boolean,
private val fullSize: Boolean
) : BaseJob(parameters) {
companion object {
const val KEY = "RestoreAttachmentJob"
private val TAG = Log.tag(AttachmentDownloadJob::class.java)
private const val KEY_MESSAGE_ID = "message_id"
private const val KEY_ATTACHMENT_ID = "part_row_id"
private const val KEY_MANUAL = "part_manual"
private const val KEY_FORCE_ARCHIVE = "force_archive"
private const val KEY_FULL_SIZE = "full_size"
@JvmStatic
fun constructQueueString(attachmentId: AttachmentId): String {
// TODO: decide how many queues
return "RestoreAttachmentJob"
}
fun jobSpecMatchesAnyAttachmentId(jobSpec: JobSpec, ids: Set<AttachmentId>): Boolean {
if (KEY != jobSpec.factoryKey) {
return false
}
val serializedData = jobSpec.serializedData ?: return false
val data = JsonJobData.deserialize(serializedData)
val parsed = AttachmentId(data.getLong(KEY_ATTACHMENT_ID))
return ids.contains(parsed)
}
fun modifyPriorities(ids: Set<AttachmentId>, priority: Int) {
val jobManager = ApplicationDependencies.getJobManager()
jobManager.update { spec ->
if (jobSpecMatchesAnyAttachmentId(spec, ids) && spec.priority != priority) {
spec.copy(priority = priority)
} else {
spec
}
}
}
}
private val attachmentId: Long
constructor(messageId: Long, attachmentId: AttachmentId, manual: Boolean, forceArchiveDownload: Boolean = false, fullSize: Boolean = true) : this(
Parameters.Builder()
.setQueue(constructQueueString(attachmentId))
.addConstraint(NetworkConstraint.KEY)
.setLifespan(TimeUnit.DAYS.toMillis(1))
.setMaxAttempts(Parameters.UNLIMITED)
.build(),
messageId,
attachmentId,
manual,
forceArchiveDownload,
fullSize
)
init {
this.attachmentId = attachmentId.id
}
override fun serialize(): ByteArray? {
return JsonJobData.Builder()
.putLong(KEY_MESSAGE_ID, messageId)
.putLong(KEY_ATTACHMENT_ID, attachmentId)
.putBoolean(KEY_MANUAL, manual)
.putBoolean(KEY_FORCE_ARCHIVE, forceArchiveDownload)
.putBoolean(KEY_FULL_SIZE, fullSize)
.serialize()
}
override fun getFactoryKey(): String {
return KEY
}
override fun onAdded() {
Log.i(TAG, "onAdded() messageId: $messageId attachmentId: $attachmentId manual: $manual")
val attachmentId = AttachmentId(attachmentId)
val attachment = SignalDatabase.attachments.getAttachment(attachmentId)
val pending = attachment != null && attachment.transferState != AttachmentTable.TRANSFER_PROGRESS_DONE && attachment.transferState != AttachmentTable.TRANSFER_PROGRESS_PERMANENT_FAILURE
if (attachment?.transferState == AttachmentTable.TRANSFER_NEEDS_RESTORE) {
Log.i(TAG, "onAdded() Marking attachment restore progress as 'started'")
SignalDatabase.attachments.setTransferState(messageId, attachmentId, AttachmentTable.TRANSFER_RESTORE_IN_PROGRESS)
}
}
@Throws(Exception::class)
public override fun onRun() {
doWork()
if (!SignalDatabase.messages.isStory(messageId)) {
ApplicationDependencies.getMessageNotifier().updateNotification(context, forConversation(0))
}
}
@Throws(IOException::class, RetryLaterException::class)
fun doWork() {
Log.i(TAG, "onRun() messageId: $messageId attachmentId: $attachmentId manual: $manual")
val attachmentId = AttachmentId(attachmentId)
val attachment = SignalDatabase.attachments.getAttachment(attachmentId)
if (attachment == null) {
Log.w(TAG, "attachment no longer exists.")
return
}
if (attachment.isPermanentlyFailed) {
Log.w(TAG, "Attachment was marked as a permanent failure. Refusing to download.")
return
}
if (attachment.transferState != AttachmentTable.TRANSFER_NEEDS_RESTORE && attachment.transferState != AttachmentTable.TRANSFER_RESTORE_IN_PROGRESS) {
Log.w(TAG, "Attachment does not need to be restored.")
return
}
retrieveAttachment(messageId, attachmentId, attachment)
}
override fun onFailure() {
Log.w(TAG, format(this, "onFailure() messageId: $messageId attachmentId: $attachmentId manual: $manual"))
val attachmentId = AttachmentId(attachmentId)
markFailed(messageId, attachmentId)
}
override fun onShouldRetry(exception: Exception): Boolean {
return exception is PushNetworkException ||
exception is RetryLaterException
}
@Throws(IOException::class, RetryLaterException::class)
private fun retrieveAttachment(
messageId: Long,
attachmentId: AttachmentId,
attachment: DatabaseAttachment
) {
val maxReceiveSize: Long = FeatureFlags.maxAttachmentReceiveSizeBytes()
val attachmentFile: File = SignalDatabase.attachments.getOrCreateTransferFile(attachmentId)
var archiveFile: File? = null
var useArchiveCdn = false
try {
if (attachment.size > maxReceiveSize) {
throw MmsException("Attachment too large, failing download")
}
useArchiveCdn = if (SignalStore.backup().canReadWriteToArchiveCdn && (forceArchiveDownload || attachment.remoteLocation == null)) {
if (attachment.archiveMediaName.isNullOrEmpty()) {
throw InvalidPartException("Invalid attachment configuration")
}
true
} else {
false
}
val messageReceiver = ApplicationDependencies.getSignalServiceMessageReceiver()
val pointer = createAttachmentPointer(attachment, useArchiveCdn)
val progressListener = object : SignalServiceAttachment.ProgressListener {
override fun onAttachmentProgress(total: Long, progress: Long) {
EventBus.getDefault().postSticky(PartProgressEvent(attachment, PartProgressEvent.Type.NETWORK, total, progress))
}
override fun shouldCancel(): Boolean {
return this@RestoreAttachmentJob.isCanceled
}
}
val stream = if (useArchiveCdn) {
archiveFile = SignalDatabase.attachments.getOrCreateArchiveTransferFile(attachmentId)
val cdnCredentials = BackupRepository.getCdnReadCredentials().successOrThrow().headers
messageReceiver
.retrieveArchivedAttachment(
SignalStore.svr().getOrCreateMasterKey().deriveBackupKey().deriveMediaSecrets(MediaName(attachment.archiveMediaName!!)),
cdnCredentials,
archiveFile,
pointer,
attachmentFile,
maxReceiveSize,
progressListener
)
} else {
messageReceiver
.retrieveAttachment(
pointer,
attachmentFile,
maxReceiveSize,
progressListener
)
}
SignalDatabase.attachments.finalizeAttachmentAfterDownload(messageId, attachmentId, stream)
} catch (e: RangeException) {
val transferFile = archiveFile ?: attachmentFile
Log.w(TAG, "Range exception, file size " + transferFile.length(), e)
if (transferFile.delete()) {
Log.i(TAG, "Deleted temp download file to recover")
throw RetryLaterException(e)
} else {
throw IOException("Failed to delete temp download file following range exception")
}
} catch (e: InvalidPartException) {
Log.w(TAG, "Experienced exception while trying to download an attachment.", e)
markFailed(messageId, attachmentId)
} catch (e: NonSuccessfulResponseCodeException) {
if (SignalStore.backup().canReadWriteToArchiveCdn) {
if (e.code == 404 && !useArchiveCdn && attachment.archiveMediaName?.isNotEmpty() == true) {
Log.i(TAG, "Retrying download from archive CDN")
forceArchiveDownload = true
retrieveAttachment(messageId, attachmentId, attachment)
return
} else if (e.code == 401 && useArchiveCdn) {
SignalStore.backup().cdnReadCredentials = null
throw RetryLaterException(e)
}
}
Log.w(TAG, "Experienced exception while trying to download an attachment.", e)
markFailed(messageId, attachmentId)
} catch (e: MmsException) {
Log.w(TAG, "Experienced exception while trying to download an attachment.", e)
markFailed(messageId, attachmentId)
} catch (e: MissingConfigurationException) {
Log.w(TAG, "Experienced exception while trying to download an attachment.", e)
markFailed(messageId, attachmentId)
} catch (e: InvalidMessageException) {
Log.w(TAG, "Experienced an InvalidMessageException while trying to download an attachment.", e)
if (e.cause is InvalidMacException) {
Log.w(TAG, "Detected an invalid mac. Treating as a permanent failure.")
markPermanentlyFailed(messageId, attachmentId)
} else {
markFailed(messageId, attachmentId)
}
}
}
@Throws(InvalidPartException::class)
private fun createAttachmentPointer(attachment: DatabaseAttachment, useArchiveCdn: Boolean): SignalServiceAttachmentPointer {
if (TextUtils.isEmpty(attachment.remoteKey)) {
throw InvalidPartException("empty encrypted key")
}
return try {
val remoteData: RemoteData = if (useArchiveCdn) {
val backupKey = SignalStore.svr().getOrCreateMasterKey().deriveBackupKey()
val backupDirectories = BackupRepository.getCdnBackupDirectories().successOrThrow()
RemoteData(
remoteId = SignalServiceAttachmentRemoteId.Backup(
backupDir = backupDirectories.backupDir,
mediaDir = backupDirectories.mediaDir,
mediaId = backupKey.deriveMediaId(MediaName(attachment.archiveMediaName!!)).encode()
),
cdnNumber = attachment.archiveCdn
)
} else {
if (attachment.remoteLocation.isNullOrEmpty()) {
throw InvalidPartException("empty content id")
}
RemoteData(
remoteId = SignalServiceAttachmentRemoteId.from(attachment.remoteLocation),
cdnNumber = attachment.cdn.cdnNumber
)
}
val key = Base64.decode(attachment.remoteKey!!)
if (attachment.remoteDigest != null) {
Log.i(TAG, "Downloading attachment with digest: " + Hex.toString(attachment.remoteDigest))
} else {
Log.i(TAG, "Downloading attachment with no digest...")
}
SignalServiceAttachmentPointer(
remoteData.cdnNumber,
remoteData.remoteId,
null,
key,
Optional.of(Util.toIntExact(attachment.size)),
Optional.empty(),
0,
0,
Optional.ofNullable(attachment.remoteDigest),
Optional.ofNullable(attachment.getIncrementalDigest()),
attachment.incrementalMacChunkSize,
Optional.ofNullable(attachment.fileName),
attachment.voiceNote,
attachment.borderless,
attachment.videoGif,
Optional.empty(),
Optional.ofNullable(attachment.blurHash).map { it.hash },
attachment.uploadTimestamp
)
} catch (e: IOException) {
Log.w(TAG, e)
throw InvalidPartException(e)
} catch (e: ArithmeticException) {
Log.w(TAG, e)
throw InvalidPartException(e)
}
}
private fun markFailed(messageId: Long, attachmentId: AttachmentId) {
try {
SignalDatabase.attachments.setTransferProgressFailed(attachmentId, messageId)
} catch (e: MmsException) {
Log.w(TAG, e)
}
}
private fun markPermanentlyFailed(messageId: Long, attachmentId: AttachmentId) {
try {
SignalDatabase.attachments.setTransferProgressPermanentFailure(attachmentId, messageId)
} catch (e: MmsException) {
Log.w(TAG, e)
}
}
@VisibleForTesting
internal class InvalidPartException : Exception {
constructor(s: String?) : super(s)
constructor(e: Exception?) : super(e)
}
private data class RemoteData(val remoteId: SignalServiceAttachmentRemoteId, val cdnNumber: Int)
class Factory : Job.Factory<RestoreAttachmentJob?> {
override fun create(parameters: Parameters, serializedData: ByteArray?): RestoreAttachmentJob {
val data = JsonJobData.deserialize(serializedData)
return RestoreAttachmentJob(
parameters = parameters,
messageId = data.getLong(KEY_MESSAGE_ID),
attachmentId = AttachmentId(data.getLong(KEY_ATTACHMENT_ID)),
manual = data.getBoolean(KEY_MANUAL),
forceArchiveDownload = data.getBooleanOrDefault(KEY_FORCE_ARCHIVE, false),
fullSize = data.getBooleanOrDefault(KEY_FULL_SIZE, true)
)
}
}
}

47
app/src/main/java/org/thoughtcrime/securesms/keyvalue/BackupValues.kt
View File

@@ -2,21 +2,44 @@ package org.thoughtcrime.securesms.keyvalue
import com.fasterxml.jackson.annotation.JsonProperty
import org.signal.core.util.logging.Log
import org.thoughtcrime.securesms.backup.RestoreState
import org.whispersystems.signalservice.api.archive.ArchiveServiceCredential
import org.whispersystems.signalservice.api.archive.GetArchiveCdnCredentialsResponse
import org.whispersystems.signalservice.internal.util.JsonUtil
import java.io.IOException
import kotlin.time.Duration
import kotlin.time.Duration.Companion.days
import kotlin.time.Duration.Companion.hours
internal class BackupValues(store: KeyValueStore) : SignalStoreValues(store) {
companion object {
val TAG = Log.tag(BackupValues::class.java)
val KEY_CREDENTIALS = "backup.credentials"
private const val KEY_CREDENTIALS = "backup.credentials"
private const val KEY_CDN_CAN_READ_WRITE = "backup.cdn.canReadWrite"
private const val KEY_CDN_READ_CREDENTIALS = "backup.cdn.readCredentials"
private const val KEY_CDN_READ_CREDENTIALS_TIMESTAMP = "backup.cdn.readCredentials.timestamp"
private const val KEY_RESTORE_STATE = "backup.restoreState"
private const val KEY_CDN_BACKUP_DIRECTORY = "backup.cdn.directory"
private const val KEY_CDN_BACKUP_MEDIA_DIRECTORY = "backup.cdn.mediaDirectory"
private const val KEY_OPTIMIZE_STORAGE = "backup.optimizeStorage"
private val cachedCdnCredentialsExpiresIn: Duration = 12.hours
}
private var cachedCdnCredentialsTimestamp: Long by longValue(KEY_CDN_READ_CREDENTIALS_TIMESTAMP, 0L)
private var cachedCdnCredentials: String? by stringValue(KEY_CDN_READ_CREDENTIALS, null)
var cachedBackupDirectory: String? by stringValue(KEY_CDN_BACKUP_DIRECTORY, null)
var cachedBackupMediaDirectory: String? by stringValue(KEY_CDN_BACKUP_MEDIA_DIRECTORY, null)
override fun onFirstEverAppLaunch() = Unit
override fun getKeysToIncludeInBackup(): List<String> = emptyList()
var canReadWriteToArchiveCdn: Boolean by booleanValue(KEY_CDN_CAN_READ_WRITE, false)
var restoreState: RestoreState by enumValue(KEY_RESTORE_STATE, RestoreState.NONE, RestoreState.serializer)
var optimizeStorage: Boolean by booleanValue(KEY_OPTIMIZE_STORAGE, false)
/**
* Retrieves the stored credentials, mapped by the day they're valid. The day is represented as
* the unix time (in seconds) of the start of the day. Wrapped in a [ArchiveServiceCredentials]
@@ -36,6 +59,28 @@ internal class BackupValues(store: KeyValueStore) : SignalStoreValues(store) {
}
}
var cdnReadCredentials: GetArchiveCdnCredentialsResponse?
get() {
val cacheAge = System.currentTimeMillis() - cachedCdnCredentialsTimestamp
val cached = cachedCdnCredentials
return if (cached != null && (cacheAge > 0 && cacheAge < cachedCdnCredentialsExpiresIn.inWholeMilliseconds)) {
try {
JsonUtil.fromJson(cached, GetArchiveCdnCredentialsResponse::class.java)
} catch (e: IOException) {
Log.w(TAG, "Invalid JSON! Clearing.", e)
cachedCdnCredentials = null
null
}
} else {
null
}
}
set(value) {
cachedCdnCredentials = value?.let { JsonUtil.toJson(it) }
cachedCdnCredentialsTimestamp = System.currentTimeMillis()
}
/**
* Adds the given credentials to the existing list of stored credentials.
*/

2
app/src/main/java/org/thoughtcrime/securesms/migrations/LegacyMigrationJob.java
View File

@@ -255,7 +255,7 @@ public class LegacyMigrationJob extends MigrationJob {
attachmentDb.setTransferState(attachment.mmsId, attachment.attachmentId, AttachmentTable.TRANSFER_PROGRESS_DONE);
} else if (record != null && !record.isOutgoing() && record.isPush()) {
Log.i(TAG, "queuing new attachment download job for incoming push part " + attachment.attachmentId + ".");
ApplicationDependencies.getJobManager().add(new AttachmentDownloadJob(attachment.mmsId, attachment.attachmentId, false));
ApplicationDependencies.getJobManager().add(new AttachmentDownloadJob(attachment.mmsId, attachment.attachmentId, false, false));
}
reader.close();
}

1
app/src/main/java/org/thoughtcrime/securesms/notifications/NotificationIds.java
View File

@@ -8,6 +8,7 @@ public final class NotificationIds {
public static final int FCM_FAILURE = 12;
public static final int ATTACHMENT_PROGRESS = 50;
public static final int BACKUP_PROGRESS = 51;
public static final int APK_UPDATE_PROMPT_INSTALL = 666;
public static final int APK_UPDATE_FAILED_INSTALL = 667;
public static final int APK_UPDATE_SUCCESSFUL_INSTALL = 668;

7
app/src/main/java/org/thoughtcrime/securesms/releasechannel/ReleaseChannel.kt
View File

@@ -1,5 +1,6 @@
package org.thoughtcrime.securesms.releasechannel
import org.thoughtcrime.securesms.attachments.Cdn
import org.thoughtcrime.securesms.attachments.PointerAttachment
import org.thoughtcrime.securesms.database.MessageTable
import org.thoughtcrime.securesms.database.MessageType
@@ -20,8 +21,6 @@ import java.util.UUID
*/
object ReleaseChannel {
const val CDN_NUMBER = -1
fun insertReleaseChannelMessage(
recipientId: RecipientId,
body: String,
@@ -36,8 +35,8 @@ object ReleaseChannel {
): MessageTable.InsertResult? {
val attachments: Optional<List<SignalServiceAttachment>> = if (media != null) {
val attachment = SignalServiceAttachmentPointer(
CDN_NUMBER,
SignalServiceAttachmentRemoteId.from(""),
Cdn.S3.cdnNumber,
SignalServiceAttachmentRemoteId.S3,
mediaType,
null,
Optional.empty(),

2
app/src/main/java/org/thoughtcrime/securesms/service/AttachmentProgressService.kt
View File

@@ -150,8 +150,8 @@ class AttachmentProgressService : SafeForegroundService() {
/** Has to have separate setter to avoid infinite loops when [progress] and [indeterminate] interact. */
fun setIndeterminate(value: Boolean) {
indeterminate = value
progress = 0f
indeterminate = value
onControllersChanged(context)
}

120
app/src/main/java/org/thoughtcrime/securesms/service/BackupProgressService.kt Normal file
View File

@@ -0,0 +1,120 @@
/*
* Copyright 2024 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.thoughtcrime.securesms.service
import android.annotation.SuppressLint
import android.app.Notification
import android.app.PendingIntent
import android.content.Context
import android.content.Intent
import androidx.core.app.NotificationCompat
import androidx.core.app.NotificationManagerCompat
import org.signal.core.util.PendingIntentFlags
import org.signal.core.util.logging.Log
import org.thoughtcrime.securesms.MainActivity
import org.thoughtcrime.securesms.R
import org.thoughtcrime.securesms.notifications.NotificationChannels
import org.thoughtcrime.securesms.notifications.NotificationIds
import java.util.concurrent.locks.ReentrantLock
import javax.annotation.CheckReturnValue
import kotlin.concurrent.withLock
/**
* Foreground service to provide "long" run support to backup jobs.
*/
class BackupProgressService : SafeForegroundService() {
companion object {
private val TAG = Log.tag(BackupProgressService::class)
@SuppressLint("StaticFieldLeak")
private var controller: Controller? = null
private val controllerLock = ReentrantLock()
private var title: String = ""
private var progress: Float = 0f
private var indeterminate: Boolean = true
@CheckReturnValue
fun start(context: Context, startingTitle: String): Controller {
controllerLock.withLock {
if (controller != null) {
Log.w(TAG, "Starting service with existing controller")
}
controller = Controller(context, startingTitle)
val started = SafeForegroundService.start(context, BackupProgressService::class.java)
if (started) {
Log.i(TAG, "[start] Starting")
} else {
Log.w(TAG, "[start] Service already started")
}
return controller!!
}
}
private fun stop(context: Context) {
SafeForegroundService.stop(context, BackupProgressService::class.java)
controllerLock.withLock {
controller = null
}
}
private fun getForegroundNotification(context: Context): Notification {
return NotificationCompat.Builder(context, NotificationChannels.getInstance().OTHER)
.setSmallIcon(R.drawable.ic_notification)
.setContentTitle(title)
.setProgress(100, (progress * 100).toInt(), indeterminate)
.setContentIntent(PendingIntent.getActivity(context, 0, MainActivity.clearTop(context), PendingIntentFlags.mutable()))
.setVibrate(longArrayOf(0))
.build()
}
}
override val tag: String = TAG
override val notificationId: Int = NotificationIds.BACKUP_PROGRESS
override fun getForegroundNotification(intent: Intent): Notification {
return getForegroundNotification(this)
}
/**
* Use to update notification progress/state.
*/
class Controller(private val context: Context, startingTitle: String) : AutoCloseable {
init {
title = startingTitle
progress = 0f
indeterminate = true
}
fun update(title: String, progress: Float, indeterminate: Boolean) {
controllerLock.withLock {
if (this != controller) {
return
}
BackupProgressService.title = title
BackupProgressService.progress = progress
BackupProgressService.indeterminate = indeterminate
if (NotificationManagerCompat.from(context).activeNotifications.any { n -> n.id == NotificationIds.BACKUP_PROGRESS }) {
NotificationManagerCompat.from(context).notify(NotificationIds.BACKUP_PROGRESS, getForegroundNotification(context))
}
}
}
override fun close() {
controllerLock.withLock {
if (this == controller) {
stop(context)
}
}
}
}
}

2
app/src/main/java/org/thoughtcrime/securesms/stories/Stories.kt
View File

@@ -141,7 +141,7 @@ object Stories {
if (record.hasLinkPreview() && record.linkPreviews[0].attachmentId != null) {
ApplicationDependencies.getJobManager().add(
AttachmentDownloadJob(record.id, record.linkPreviews[0].attachmentId, true)
AttachmentDownloadJob(record.id, record.linkPreviews[0].attachmentId!!, true)
)
}
}