mirror of
https://github.com/signalapp/Signal-Android.git
synced 2026-04-25 19:29:54 +01:00
Refactor and cleanup backupv2 media restore.
This commit is contained in:
Cody Henthorne
@@ -13,6 +13,7 @@ import org.thoughtcrime.securesms.attachments.AttachmentId
|
||||
import org.thoughtcrime.securesms.attachments.DatabaseAttachment
|
||||
import org.thoughtcrime.securesms.attachments.PointerAttachment
|
||||
import org.thoughtcrime.securesms.backup.v2.BackupRepository
|
||||
import org.thoughtcrime.securesms.backup.v2.BackupRepository.getMediaName
|
||||
import org.thoughtcrime.securesms.backup.v2.BackupRepository.getThumbnailMediaName
|
||||
import org.thoughtcrime.securesms.database.SignalDatabase
|
||||
import org.thoughtcrime.securesms.dependencies.AppDependencies
|
||||
@@ -29,7 +30,6 @@ import org.whispersystems.signalservice.api.messages.SignalServiceAttachmentStre
|
||||
import org.whispersystems.signalservice.internal.push.http.ResumableUploadSpec
|
||||
import java.io.ByteArrayInputStream
|
||||
import java.io.IOException
|
||||
import java.lang.RuntimeException
|
||||
import java.util.Optional
|
||||
import kotlin.time.Duration.Companion.days
|
||||
|
||||
@@ -52,7 +52,7 @@ class ArchiveThumbnailUploadJob private constructor(
|
||||
}
|
||||
}
|
||||
|
||||
constructor(attachmentId: AttachmentId) : this(
|
||||
private constructor(attachmentId: AttachmentId) : this(
|
||||
Parameters.Builder()
|
||||
.setQueue("ArchiveThumbnailUploadJob")
|
||||
.addConstraint(NetworkConstraint.KEY)
|
||||
@@ -82,11 +82,15 @@ class ArchiveThumbnailUploadJob private constructor(
|
||||
return Result.success()
|
||||
}
|
||||
|
||||
// TODO [backups] Decide if we fail a job when associated attachment not already backed up
|
||||
// TODO [backups] Determine if we actually need to upload or are reusing a thumbnail from another attachment
|
||||
|
||||
val thumbnailResult = generateThumbnailIfPossible(attachment)
|
||||
if (thumbnailResult == null) {
|
||||
Log.w(TAG, "Unable to generate a thumbnail result for $attachmentId")
|
||||
return Result.success()
|
||||
}
|
||||
|
||||
val backupKey = SignalStore.svr.getOrCreateMasterKey().deriveBackupKey()
|
||||
|
||||
val resumableUpload = when (val result = BackupRepository.getMediaUploadSpec(secretKey = backupKey.deriveThumbnailTransitKey(attachment.getThumbnailMediaName()))) {
|
||||
@@ -123,6 +127,10 @@ class ArchiveThumbnailUploadJob private constructor(
|
||||
|
||||
return when (val result = BackupRepository.archiveThumbnail(attachmentPointer, attachment)) {
|
||||
is NetworkResult.Success -> {
|
||||
// save attachment thumbnail
|
||||
val archiveMediaId = attachment.archiveMediaId ?: backupKey.deriveMediaId(attachment.getMediaName()).encode()
|
||||
SignalDatabase.attachments.finalizeAttachmentThumbnailAfterUpload(attachmentId, archiveMediaId, mediaSecrets.id, thumbnailResult.data)
|
||||
|
||||
Log.i(RestoreAttachmentJob.TAG, "Restore: Thumbnail mediaId=${mediaSecrets.id.encode()} backupDir=${backupDirectories.backupDir} mediaDir=${backupDirectories.mediaDir}")
|
||||
Log.d(TAG, "Successfully archived thumbnail for $attachmentId mediaName=${attachment.getThumbnailMediaName()}")
|
||||
Result.success()
|
||||
|
||||
@@ -4,8 +4,6 @@
|
||||
*/
|
||||
package org.thoughtcrime.securesms.jobs
|
||||
|
||||
import android.text.TextUtils
|
||||
import androidx.annotation.VisibleForTesting
|
||||
import okio.Source
|
||||
import okio.buffer
|
||||
import org.greenrobot.eventbus.EventBus
|
||||
@@ -19,7 +17,7 @@ import org.thoughtcrime.securesms.attachments.Attachment
|
||||
import org.thoughtcrime.securesms.attachments.AttachmentId
|
||||
import org.thoughtcrime.securesms.attachments.Cdn
|
||||
import org.thoughtcrime.securesms.attachments.DatabaseAttachment
|
||||
import org.thoughtcrime.securesms.backup.v2.BackupRepository
|
||||
import org.thoughtcrime.securesms.attachments.InvalidAttachmentException
|
||||
import org.thoughtcrime.securesms.database.AttachmentTable
|
||||
import org.thoughtcrime.securesms.database.SignalDatabase
|
||||
import org.thoughtcrime.securesms.dependencies.AppDependencies
|
||||
@@ -37,7 +35,6 @@ import org.thoughtcrime.securesms.transport.RetryLaterException
|
||||
import org.thoughtcrime.securesms.util.AttachmentUtil
|
||||
import org.thoughtcrime.securesms.util.RemoteConfig
|
||||
import org.thoughtcrime.securesms.util.Util
|
||||
import org.whispersystems.signalservice.api.backup.MediaName
|
||||
import org.whispersystems.signalservice.api.messages.SignalServiceAttachment
|
||||
import org.whispersystems.signalservice.api.messages.SignalServiceAttachmentPointer
|
||||
import org.whispersystems.signalservice.api.messages.SignalServiceAttachmentRemoteId
|
||||
@@ -56,9 +53,8 @@ import java.util.concurrent.TimeUnit
|
||||
class AttachmentDownloadJob private constructor(
|
||||
parameters: Parameters,
|
||||
private val messageId: Long,
|
||||
attachmentId: AttachmentId,
|
||||
private val manual: Boolean,
|
||||
private var forceArchiveDownload: Boolean
|
||||
private val attachmentId: AttachmentId,
|
||||
private val manual: Boolean
|
||||
) : BaseJob(parameters) {
|
||||
|
||||
companion object {
|
||||
@@ -68,7 +64,6 @@ class AttachmentDownloadJob private constructor(
|
||||
private const val KEY_MESSAGE_ID = "message_id"
|
||||
private const val KEY_ATTACHMENT_ID = "part_row_id"
|
||||
private const val KEY_MANUAL = "part_manual"
|
||||
private const val KEY_FORCE_ARCHIVE = "force_archive"
|
||||
|
||||
@JvmStatic
|
||||
fun constructQueueString(attachmentId: AttachmentId): String {
|
||||
@@ -89,25 +84,37 @@ class AttachmentDownloadJob private constructor(
|
||||
@JvmStatic
|
||||
fun downloadAttachmentIfNeeded(databaseAttachment: DatabaseAttachment): String? {
|
||||
return when (val transferState = databaseAttachment.transferState) {
|
||||
AttachmentTable.TRANSFER_RESTORE_OFFLOADED -> RestoreAttachmentJob.restoreOffloadedAttachment(databaseAttachment)
|
||||
AttachmentTable.TRANSFER_RESTORE_OFFLOADED,
|
||||
AttachmentTable.TRANSFER_NEEDS_RESTORE -> RestoreAttachmentJob.restoreAttachment(databaseAttachment)
|
||||
|
||||
AttachmentTable.TRANSFER_PROGRESS_PENDING,
|
||||
AttachmentTable.TRANSFER_PROGRESS_FAILED,
|
||||
AttachmentTable.TRANSFER_NEEDS_RESTORE -> {
|
||||
val downloadJob = AttachmentDownloadJob(
|
||||
messageId = databaseAttachment.mmsId,
|
||||
attachmentId = databaseAttachment.attachmentId,
|
||||
manual = true,
|
||||
forceArchiveDownload = false
|
||||
)
|
||||
AppDependencies.jobManager.add(downloadJob)
|
||||
downloadJob.id
|
||||
AttachmentTable.TRANSFER_PROGRESS_FAILED -> {
|
||||
if (SignalStore.backup.backsUpMedia && databaseAttachment.remoteLocation == null) {
|
||||
if (databaseAttachment.archiveMediaName.isNullOrEmpty()) {
|
||||
Log.w(TAG, "No remote location or archive media name, can't download")
|
||||
null
|
||||
} else {
|
||||
Log.i(TAG, "Trying to restore attachment from archive cdn")
|
||||
RestoreAttachmentJob.restoreAttachment(databaseAttachment)
|
||||
}
|
||||
} else {
|
||||
val downloadJob = AttachmentDownloadJob(
|
||||
messageId = databaseAttachment.mmsId,
|
||||
attachmentId = databaseAttachment.attachmentId,
|
||||
manual = true
|
||||
)
|
||||
AppDependencies.jobManager.add(downloadJob)
|
||||
downloadJob.id
|
||||
}
|
||||
}
|
||||
|
||||
AttachmentTable.TRANSFER_RESTORE_IN_PROGRESS,
|
||||
AttachmentTable.TRANSFER_PROGRESS_DONE,
|
||||
AttachmentTable.TRANSFER_PROGRESS_STARTED,
|
||||
AttachmentTable.TRANSFER_PROGRESS_PERMANENT_FAILURE -> null
|
||||
AttachmentTable.TRANSFER_PROGRESS_PERMANENT_FAILURE -> {
|
||||
Log.d(TAG, "$databaseAttachment is downloading, downloaded already or permanently failed, transferState: $transferState")
|
||||
null
|
||||
}
|
||||
|
||||
else -> {
|
||||
Log.w(TAG, "Attempted to download attachment with unknown transfer state: $transferState")
|
||||
@@ -117,9 +124,7 @@ class AttachmentDownloadJob private constructor(
|
||||
}
|
||||
}
|
||||
|
||||
private val attachmentId: Long
|
||||
|
||||
constructor(messageId: Long, attachmentId: AttachmentId, manual: Boolean, forceArchiveDownload: Boolean = false) : this(
|
||||
constructor(messageId: Long, attachmentId: AttachmentId, manual: Boolean) : this(
|
||||
Parameters.Builder()
|
||||
.setQueue(constructQueueString(attachmentId))
|
||||
.addConstraint(NetworkConstraint.KEY)
|
||||
@@ -128,20 +133,14 @@ class AttachmentDownloadJob private constructor(
|
||||
.build(),
|
||||
messageId,
|
||||
attachmentId,
|
||||
manual,
|
||||
forceArchiveDownload
|
||||
manual
|
||||
)
|
||||
|
||||
init {
|
||||
this.attachmentId = attachmentId.id
|
||||
}
|
||||
|
||||
override fun serialize(): ByteArray? {
|
||||
return JsonJobData.Builder()
|
||||
.putLong(KEY_MESSAGE_ID, messageId)
|
||||
.putLong(KEY_ATTACHMENT_ID, attachmentId)
|
||||
.putLong(KEY_ATTACHMENT_ID, attachmentId.id)
|
||||
.putBoolean(KEY_MANUAL, manual)
|
||||
.putBoolean(KEY_FORCE_ARCHIVE, forceArchiveDownload)
|
||||
.serialize()
|
||||
}
|
||||
|
||||
@@ -152,7 +151,6 @@ class AttachmentDownloadJob private constructor(
|
||||
override fun onAdded() {
|
||||
Log.i(TAG, "onAdded() messageId: $messageId attachmentId: $attachmentId manual: $manual")
|
||||
|
||||
val attachmentId = AttachmentId(attachmentId)
|
||||
val attachment = SignalDatabase.attachments.getAttachment(attachmentId)
|
||||
val pending = attachment != null && attachment.transferState != AttachmentTable.TRANSFER_PROGRESS_DONE && attachment.transferState != AttachmentTable.TRANSFER_PROGRESS_PERMANENT_FAILURE
|
||||
|
||||
@@ -175,7 +173,6 @@ class AttachmentDownloadJob private constructor(
|
||||
fun doWork() {
|
||||
Log.i(TAG, "onRun() messageId: $messageId attachmentId: $attachmentId manual: $manual")
|
||||
|
||||
val attachmentId = AttachmentId(attachmentId)
|
||||
val attachment = SignalDatabase.attachments.getAttachment(attachmentId)
|
||||
|
||||
if (attachment == null) {
|
||||
@@ -199,6 +196,17 @@ class AttachmentDownloadJob private constructor(
|
||||
return
|
||||
}
|
||||
|
||||
if (SignalStore.backup.backsUpMedia && attachment.remoteLocation == null) {
|
||||
if (attachment.archiveMediaName.isNullOrEmpty()) {
|
||||
throw InvalidAttachmentException("No remote location or archive media name")
|
||||
}
|
||||
|
||||
Log.i(TAG, "Trying to restore attachment from archive cdn instead")
|
||||
RestoreAttachmentJob.restoreAttachment(attachment)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
Log.i(TAG, "Downloading push part $attachmentId")
|
||||
SignalDatabase.attachments.setTransferState(messageId, attachmentId, AttachmentTable.TRANSFER_PROGRESS_STARTED)
|
||||
|
||||
@@ -218,7 +226,6 @@ class AttachmentDownloadJob private constructor(
|
||||
override fun onFailure() {
|
||||
Log.w(TAG, format(this, "onFailure() messageId: $messageId attachmentId: $attachmentId manual: $manual"))
|
||||
|
||||
val attachmentId = AttachmentId(attachmentId)
|
||||
markFailed(messageId, attachmentId)
|
||||
}
|
||||
|
||||
@@ -235,25 +242,13 @@ class AttachmentDownloadJob private constructor(
|
||||
) {
|
||||
val maxReceiveSize: Long = RemoteConfig.maxAttachmentReceiveSizeBytes
|
||||
val attachmentFile: File = SignalDatabase.attachments.getOrCreateTransferFile(attachmentId)
|
||||
var archiveFile: File? = null
|
||||
var useArchiveCdn = false
|
||||
|
||||
try {
|
||||
if (attachment.size > maxReceiveSize) {
|
||||
throw MmsException("Attachment too large, failing download")
|
||||
}
|
||||
|
||||
useArchiveCdn = if (SignalStore.backup.backsUpMedia && (forceArchiveDownload || attachment.remoteLocation == null)) {
|
||||
if (attachment.archiveMediaName.isNullOrEmpty()) {
|
||||
throw InvalidPartException("Invalid attachment configuration")
|
||||
}
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
|
||||
val messageReceiver = AppDependencies.signalServiceMessageReceiver
|
||||
val pointer = createAttachmentPointer(attachment, useArchiveCdn)
|
||||
val pointer = createAttachmentPointer(attachment)
|
||||
|
||||
val progressListener = object : SignalServiceAttachment.ProgressListener {
|
||||
override fun onAttachmentProgress(total: Long, progress: Long) {
|
||||
@@ -265,55 +260,32 @@ class AttachmentDownloadJob private constructor(
|
||||
}
|
||||
}
|
||||
|
||||
val downloadResult = if (useArchiveCdn) {
|
||||
archiveFile = SignalDatabase.attachments.getOrCreateArchiveTransferFile(attachmentId)
|
||||
val cdnCredentials = BackupRepository.getCdnReadCredentials(attachment.archiveCdn).successOrThrow().headers
|
||||
|
||||
messageReceiver
|
||||
.retrieveArchivedAttachment(
|
||||
SignalStore.svr.getOrCreateMasterKey().deriveBackupKey().deriveMediaSecrets(MediaName(attachment.archiveMediaName!!)),
|
||||
cdnCredentials,
|
||||
archiveFile,
|
||||
pointer,
|
||||
attachmentFile,
|
||||
maxReceiveSize,
|
||||
false,
|
||||
progressListener
|
||||
)
|
||||
} else {
|
||||
messageReceiver
|
||||
.retrieveAttachment(
|
||||
pointer,
|
||||
attachmentFile,
|
||||
maxReceiveSize,
|
||||
progressListener
|
||||
)
|
||||
}
|
||||
val downloadResult = AppDependencies
|
||||
.signalServiceMessageReceiver
|
||||
.retrieveAttachment(
|
||||
pointer,
|
||||
attachmentFile,
|
||||
maxReceiveSize,
|
||||
progressListener
|
||||
)
|
||||
|
||||
SignalDatabase.attachments.finalizeAttachmentAfterDownload(messageId, attachmentId, downloadResult.dataStream, downloadResult.iv)
|
||||
} catch (e: RangeException) {
|
||||
val transferFile = archiveFile ?: attachmentFile
|
||||
Log.w(TAG, "Range exception, file size " + transferFile.length(), e)
|
||||
if (transferFile.delete()) {
|
||||
Log.w(TAG, "Range exception, file size " + attachmentFile.length(), e)
|
||||
if (attachmentFile.delete()) {
|
||||
Log.i(TAG, "Deleted temp download file to recover")
|
||||
throw RetryLaterException(e)
|
||||
} else {
|
||||
throw IOException("Failed to delete temp download file following range exception")
|
||||
}
|
||||
} catch (e: InvalidPartException) {
|
||||
} catch (e: InvalidAttachmentException) {
|
||||
Log.w(TAG, "Experienced exception while trying to download an attachment.", e)
|
||||
markFailed(messageId, attachmentId)
|
||||
} catch (e: NonSuccessfulResponseCodeException) {
|
||||
if (SignalStore.backup.backsUpMedia) {
|
||||
if (e.code == 404 && !useArchiveCdn && attachment.archiveMediaName?.isNotEmpty() == true) {
|
||||
Log.i(TAG, "Retrying download from archive CDN")
|
||||
forceArchiveDownload = true
|
||||
retrieveAttachment(messageId, attachmentId, attachment)
|
||||
return
|
||||
} else if (e.code == 401 && useArchiveCdn) {
|
||||
SignalStore.backup.cdnReadCredentials = null
|
||||
throw RetryLaterException(e)
|
||||
}
|
||||
if (SignalStore.backup.backsUpMedia && e.code == 404 && attachment.archiveMediaName?.isNotEmpty() == true) {
|
||||
Log.i(TAG, "Retrying download from archive CDN")
|
||||
RestoreAttachmentJob.restoreAttachment(attachment)
|
||||
return
|
||||
}
|
||||
|
||||
Log.w(TAG, "Experienced exception while trying to download an attachment.", e)
|
||||
@@ -335,47 +307,31 @@ class AttachmentDownloadJob private constructor(
|
||||
}
|
||||
}
|
||||
|
||||
@Throws(InvalidPartException::class)
|
||||
private fun createAttachmentPointer(attachment: DatabaseAttachment, useArchiveCdn: Boolean): SignalServiceAttachmentPointer {
|
||||
if (TextUtils.isEmpty(attachment.remoteKey)) {
|
||||
throw InvalidPartException("empty encrypted key")
|
||||
@Throws(InvalidAttachmentException::class)
|
||||
private fun createAttachmentPointer(attachment: DatabaseAttachment): SignalServiceAttachmentPointer {
|
||||
if (attachment.remoteKey.isNullOrEmpty()) {
|
||||
throw InvalidAttachmentException("empty encrypted key")
|
||||
}
|
||||
|
||||
if (attachment.remoteLocation.isNullOrEmpty()) {
|
||||
throw InvalidAttachmentException("empty content id")
|
||||
}
|
||||
|
||||
return try {
|
||||
val remoteData: RemoteData = if (useArchiveCdn) {
|
||||
val backupKey = SignalStore.svr.getOrCreateMasterKey().deriveBackupKey()
|
||||
val backupDirectories = BackupRepository.getCdnBackupDirectories().successOrThrow()
|
||||
val remoteId = SignalServiceAttachmentRemoteId.from(attachment.remoteLocation)
|
||||
val cdnNumber = attachment.cdn.cdnNumber
|
||||
|
||||
RemoteData(
|
||||
remoteId = SignalServiceAttachmentRemoteId.Backup(
|
||||
backupDir = backupDirectories.backupDir,
|
||||
mediaDir = backupDirectories.mediaDir,
|
||||
mediaId = backupKey.deriveMediaId(MediaName(attachment.archiveMediaName!!)).encode()
|
||||
),
|
||||
cdnNumber = attachment.archiveCdn
|
||||
)
|
||||
} else {
|
||||
if (attachment.remoteLocation.isNullOrEmpty()) {
|
||||
throw InvalidPartException("empty content id")
|
||||
}
|
||||
|
||||
RemoteData(
|
||||
remoteId = SignalServiceAttachmentRemoteId.from(attachment.remoteLocation),
|
||||
cdnNumber = attachment.cdn.cdnNumber
|
||||
)
|
||||
}
|
||||
|
||||
val key = Base64.decode(attachment.remoteKey!!)
|
||||
val key = Base64.decode(attachment.remoteKey)
|
||||
|
||||
if (attachment.remoteDigest != null) {
|
||||
Log.i(TAG, "Downloading attachment with digest: " + Hex.toString(attachment.remoteDigest))
|
||||
} else {
|
||||
throw InvalidPartException("Null remote digest for $attachmentId")
|
||||
throw InvalidAttachmentException("Null remote digest for $attachmentId")
|
||||
}
|
||||
|
||||
SignalServiceAttachmentPointer(
|
||||
remoteData.cdnNumber,
|
||||
remoteData.remoteId,
|
||||
cdnNumber,
|
||||
remoteId,
|
||||
null,
|
||||
key,
|
||||
Optional.of(Util.toIntExact(attachment.size)),
|
||||
@@ -396,10 +352,10 @@ class AttachmentDownloadJob private constructor(
|
||||
)
|
||||
} catch (e: IOException) {
|
||||
Log.w(TAG, e)
|
||||
throw InvalidPartException(e)
|
||||
throw InvalidAttachmentException(e)
|
||||
} catch (e: ArithmeticException) {
|
||||
Log.w(TAG, e)
|
||||
throw InvalidPartException(e)
|
||||
throw InvalidAttachmentException(e)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -442,14 +398,6 @@ class AttachmentDownloadJob private constructor(
|
||||
SignalDatabase.attachments.setTransferProgressPermanentFailure(attachmentId, messageId)
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
internal class InvalidPartException : Exception {
|
||||
constructor(s: String?) : super(s)
|
||||
constructor(e: Exception?) : super(e)
|
||||
}
|
||||
|
||||
private data class RemoteData(val remoteId: SignalServiceAttachmentRemoteId, val cdnNumber: Int)
|
||||
|
||||
class Factory : Job.Factory<AttachmentDownloadJob?> {
|
||||
override fun create(parameters: Parameters, serializedData: ByteArray?): AttachmentDownloadJob {
|
||||
val data = JsonJobData.deserialize(serializedData)
|
||||
@@ -457,8 +405,7 @@ class AttachmentDownloadJob private constructor(
|
||||
parameters = parameters,
|
||||
messageId = data.getLong(KEY_MESSAGE_ID),
|
||||
attachmentId = AttachmentId(data.getLong(KEY_ATTACHMENT_ID)),
|
||||
manual = data.getBoolean(KEY_MANUAL),
|
||||
forceArchiveDownload = data.getBooleanOrDefault(KEY_FORCE_ARCHIVE, false)
|
||||
manual = data.getBoolean(KEY_MANUAL)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,7 +6,9 @@
|
||||
package org.thoughtcrime.securesms.jobs
|
||||
|
||||
import org.signal.core.util.logging.Log
|
||||
import org.signal.core.util.withinTransaction
|
||||
import org.thoughtcrime.securesms.database.AttachmentTable
|
||||
import org.thoughtcrime.securesms.database.AttachmentTable.RestorableAttachment
|
||||
import org.thoughtcrime.securesms.database.SignalDatabase
|
||||
import org.thoughtcrime.securesms.database.model.MmsMessageRecord
|
||||
import org.thoughtcrime.securesms.dependencies.AppDependencies
|
||||
@@ -48,40 +50,64 @@ class BackupRestoreMediaJob private constructor(parameters: Parameters) : BaseJo
|
||||
throw NotPushRegisteredException()
|
||||
}
|
||||
|
||||
SignalStore.backup.totalRestorableAttachmentSize = SignalDatabase.attachments.getRemainingRestorableAttachmentSize()
|
||||
val jobManager = AppDependencies.jobManager
|
||||
val batchSize = 100
|
||||
val batchSize = 500
|
||||
val restoreTime = System.currentTimeMillis()
|
||||
var restoreJobBatch: List<Job>
|
||||
|
||||
do {
|
||||
val restoreThumbnailJobs: MutableList<RestoreAttachmentThumbnailJob> = mutableListOf()
|
||||
val restoreFullAttachmentJobs: MutableMap<RestorableAttachment, RestoreAttachmentJob> = mutableMapOf()
|
||||
|
||||
val restoreThumbnailOnlyAttachments: MutableList<RestorableAttachment> = mutableListOf()
|
||||
val notRestorable: MutableList<RestorableAttachment> = mutableListOf()
|
||||
|
||||
val attachmentBatch = SignalDatabase.attachments.getRestorableAttachments(batchSize)
|
||||
val messageIds = attachmentBatch.map { it.mmsId }.toSet()
|
||||
val messageMap = SignalDatabase.messages.getMessages(messageIds).associate { it.id to (it as MmsMessageRecord) }
|
||||
restoreJobBatch = SignalDatabase.attachments.getRestorableAttachments(batchSize).map { attachment ->
|
||||
val message = messageMap[attachment.mmsId]!!
|
||||
|
||||
for (attachment in attachmentBatch) {
|
||||
val message = messageMap[attachment.mmsId]
|
||||
if (message == null) {
|
||||
Log.w(TAG, "Unable to find message for ${attachment.attachmentId}")
|
||||
notRestorable += attachment
|
||||
continue
|
||||
}
|
||||
|
||||
restoreThumbnailJobs += RestoreAttachmentThumbnailJob(
|
||||
messageId = attachment.mmsId,
|
||||
attachmentId = attachment.attachmentId,
|
||||
highPriority = false
|
||||
)
|
||||
|
||||
if (shouldRestoreFullSize(message, restoreTime, SignalStore.backup.optimizeStorage)) {
|
||||
RestoreAttachmentJob(
|
||||
restoreFullAttachmentJobs += attachment to RestoreAttachmentJob(
|
||||
messageId = attachment.mmsId,
|
||||
attachmentId = attachment.attachmentId,
|
||||
manual = false,
|
||||
forceArchiveDownload = true,
|
||||
restoreMode = RestoreAttachmentJob.RestoreMode.ORIGINAL
|
||||
attachmentId = attachment.attachmentId
|
||||
)
|
||||
} else {
|
||||
SignalDatabase.attachments.setTransferState(
|
||||
messageId = attachment.mmsId,
|
||||
attachmentId = attachment.attachmentId,
|
||||
transferState = AttachmentTable.TRANSFER_RESTORE_OFFLOADED
|
||||
)
|
||||
RestoreAttachmentThumbnailJob(
|
||||
messageId = attachment.mmsId,
|
||||
attachmentId = attachment.attachmentId,
|
||||
highPriority = false
|
||||
)
|
||||
restoreThumbnailOnlyAttachments += attachment
|
||||
}
|
||||
}
|
||||
jobManager.addAll(restoreJobBatch)
|
||||
} while (restoreJobBatch.isNotEmpty())
|
||||
|
||||
SignalDatabase.rawDatabase.withinTransaction {
|
||||
// Mark not restorable thumbnails and attachments as failed
|
||||
SignalDatabase.attachments.setThumbnailRestoreState(notRestorable.map { it.attachmentId }, AttachmentTable.ThumbnailRestoreState.PERMANENT_FAILURE)
|
||||
SignalDatabase.attachments.setRestoreTransferState(notRestorable, AttachmentTable.TRANSFER_PROGRESS_FAILED)
|
||||
|
||||
// Mark restorable thumbnails and attachments as in progress
|
||||
SignalDatabase.attachments.setThumbnailRestoreState(restoreThumbnailJobs.map { it.attachmentId }, AttachmentTable.ThumbnailRestoreState.IN_PROGRESS)
|
||||
SignalDatabase.attachments.setRestoreTransferState(restoreFullAttachmentJobs.keys, AttachmentTable.TRANSFER_RESTORE_IN_PROGRESS)
|
||||
|
||||
// Set thumbnail only attachments as offloaded
|
||||
SignalDatabase.attachments.setRestoreTransferState(restoreThumbnailOnlyAttachments, AttachmentTable.TRANSFER_RESTORE_OFFLOADED)
|
||||
|
||||
jobManager.addAll(restoreThumbnailJobs + restoreFullAttachmentJobs.values)
|
||||
}
|
||||
} while (restoreThumbnailJobs.isNotEmpty() && restoreFullAttachmentJobs.isNotEmpty() && notRestorable.isNotEmpty())
|
||||
|
||||
SignalStore.backup.totalRestorableAttachmentSize = SignalDatabase.attachments.getRemainingRestorableAttachmentSize()
|
||||
|
||||
jobManager.add(CheckRestoreMediaLeftJob(RestoreAttachmentJob.constructQueueString()))
|
||||
}
|
||||
|
||||
private fun shouldRestoreFullSize(message: MmsMessageRecord, restoreTime: Long, optimizeStorage: Boolean): Boolean {
|
||||
|
||||
@@ -4,44 +4,36 @@
|
||||
*/
|
||||
package org.thoughtcrime.securesms.jobs
|
||||
|
||||
import android.text.TextUtils
|
||||
import androidx.annotation.VisibleForTesting
|
||||
import org.greenrobot.eventbus.EventBus
|
||||
import org.signal.core.util.Base64
|
||||
import org.signal.core.util.Hex
|
||||
import org.signal.core.util.logging.Log
|
||||
import org.signal.libsignal.protocol.InvalidMacException
|
||||
import org.signal.libsignal.protocol.InvalidMessageException
|
||||
import org.thoughtcrime.securesms.attachments.AttachmentId
|
||||
import org.thoughtcrime.securesms.attachments.DatabaseAttachment
|
||||
import org.thoughtcrime.securesms.attachments.InvalidAttachmentException
|
||||
import org.thoughtcrime.securesms.backup.v2.BackupRepository
|
||||
import org.thoughtcrime.securesms.backup.v2.BackupRepository.getThumbnailMediaName
|
||||
import org.thoughtcrime.securesms.backup.v2.database.createArchiveAttachmentPointer
|
||||
import org.thoughtcrime.securesms.database.AttachmentTable
|
||||
import org.thoughtcrime.securesms.database.SignalDatabase
|
||||
import org.thoughtcrime.securesms.dependencies.AppDependencies
|
||||
import org.thoughtcrime.securesms.events.PartProgressEvent
|
||||
import org.thoughtcrime.securesms.jobmanager.Job
|
||||
import org.thoughtcrime.securesms.jobmanager.JobLogger.format
|
||||
import org.thoughtcrime.securesms.jobmanager.JsonJobData
|
||||
import org.thoughtcrime.securesms.jobmanager.impl.NetworkConstraint
|
||||
import org.thoughtcrime.securesms.jobmanager.persistence.JobSpec
|
||||
import org.thoughtcrime.securesms.jobs.protos.RestoreAttachmentJobData
|
||||
import org.thoughtcrime.securesms.keyvalue.SignalStore
|
||||
import org.thoughtcrime.securesms.mms.MmsException
|
||||
import org.thoughtcrime.securesms.notifications.v2.ConversationId.Companion.forConversation
|
||||
import org.thoughtcrime.securesms.transport.RetryLaterException
|
||||
import org.thoughtcrime.securesms.util.RemoteConfig
|
||||
import org.thoughtcrime.securesms.util.Util
|
||||
import org.whispersystems.signalservice.api.backup.MediaName
|
||||
import org.whispersystems.signalservice.api.messages.SignalServiceAttachment
|
||||
import org.whispersystems.signalservice.api.messages.SignalServiceAttachmentPointer
|
||||
import org.whispersystems.signalservice.api.messages.SignalServiceAttachmentRemoteId
|
||||
import org.whispersystems.signalservice.api.push.exceptions.MissingConfigurationException
|
||||
import org.whispersystems.signalservice.api.push.exceptions.NonSuccessfulResponseCodeException
|
||||
import org.whispersystems.signalservice.api.push.exceptions.PushNetworkException
|
||||
import org.whispersystems.signalservice.api.push.exceptions.RangeException
|
||||
import java.io.File
|
||||
import java.io.IOException
|
||||
import java.util.Optional
|
||||
import java.util.concurrent.TimeUnit
|
||||
|
||||
/**
|
||||
@@ -50,83 +42,46 @@ import java.util.concurrent.TimeUnit
|
||||
class RestoreAttachmentJob private constructor(
|
||||
parameters: Parameters,
|
||||
private val messageId: Long,
|
||||
attachmentId: AttachmentId,
|
||||
private val manual: Boolean,
|
||||
private var forceArchiveDownload: Boolean,
|
||||
private val restoreMode: RestoreMode
|
||||
private val attachmentId: AttachmentId,
|
||||
private val offloaded: Boolean
|
||||
) : BaseJob(parameters) {
|
||||
|
||||
companion object {
|
||||
const val KEY = "RestoreAttachmentJob"
|
||||
val TAG = Log.tag(RestoreAttachmentJob::class.java)
|
||||
|
||||
private const val KEY_MESSAGE_ID = "message_id"
|
||||
private const val KEY_ATTACHMENT_ID = "part_row_id"
|
||||
private const val KEY_MANUAL = "part_manual"
|
||||
private const val KEY_FORCE_ARCHIVE = "force_archive"
|
||||
private const val KEY_RESTORE_MODE = "restore_mode"
|
||||
|
||||
@JvmStatic
|
||||
fun constructQueueString(attachmentId: AttachmentId): String {
|
||||
fun constructQueueString(): String {
|
||||
// TODO: decide how many queues
|
||||
return "RestoreAttachmentJob"
|
||||
}
|
||||
|
||||
private fun getJsonJobData(jobSpec: JobSpec): JsonJobData? {
|
||||
if (KEY != jobSpec.factoryKey) {
|
||||
return null
|
||||
}
|
||||
|
||||
val serializedData = jobSpec.serializedData ?: return null
|
||||
return JsonJobData.deserialize(serializedData)
|
||||
}
|
||||
|
||||
fun jobSpecMatchesAnyAttachmentId(data: JsonJobData?, ids: Set<AttachmentId>): Boolean {
|
||||
if (data == null) {
|
||||
return false
|
||||
}
|
||||
val parsed = AttachmentId(data.getLong(KEY_ATTACHMENT_ID))
|
||||
return ids.contains(parsed)
|
||||
}
|
||||
|
||||
@JvmStatic
|
||||
fun restoreOffloadedAttachment(attachment: DatabaseAttachment): String {
|
||||
fun restoreAttachment(attachment: DatabaseAttachment): String {
|
||||
val restoreJob = RestoreAttachmentJob(
|
||||
messageId = attachment.mmsId,
|
||||
attachmentId = attachment.attachmentId,
|
||||
manual = false,
|
||||
forceArchiveDownload = true,
|
||||
restoreMode = RestoreAttachmentJob.RestoreMode.ORIGINAL
|
||||
offloaded = true
|
||||
)
|
||||
AppDependencies.jobManager.add(restoreJob)
|
||||
return restoreJob.id
|
||||
}
|
||||
}
|
||||
|
||||
private val attachmentId: Long = attachmentId.id
|
||||
|
||||
constructor(messageId: Long, attachmentId: AttachmentId, manual: Boolean, forceArchiveDownload: Boolean = false, restoreMode: RestoreMode = RestoreMode.ORIGINAL) : this(
|
||||
constructor(messageId: Long, attachmentId: AttachmentId, offloaded: Boolean = false) : this(
|
||||
Parameters.Builder()
|
||||
.setQueue(constructQueueString(attachmentId))
|
||||
.setQueue(constructQueueString())
|
||||
.addConstraint(NetworkConstraint.KEY)
|
||||
.setLifespan(TimeUnit.DAYS.toMillis(1))
|
||||
.setMaxAttempts(Parameters.UNLIMITED)
|
||||
.build(),
|
||||
messageId,
|
||||
attachmentId,
|
||||
manual,
|
||||
forceArchiveDownload,
|
||||
restoreMode
|
||||
offloaded
|
||||
)
|
||||
|
||||
override fun serialize(): ByteArray? {
|
||||
return JsonJobData.Builder()
|
||||
.putLong(KEY_MESSAGE_ID, messageId)
|
||||
.putLong(KEY_ATTACHMENT_ID, attachmentId)
|
||||
.putBoolean(KEY_MANUAL, manual)
|
||||
.putBoolean(KEY_FORCE_ARCHIVE, forceArchiveDownload)
|
||||
.putInt(KEY_RESTORE_MODE, restoreMode.value)
|
||||
.serialize()
|
||||
return RestoreAttachmentJobData(messageId = messageId, attachmentId = attachmentId.id, offloaded = offloaded).encode()
|
||||
}
|
||||
|
||||
override fun getFactoryKey(): String {
|
||||
@@ -134,14 +89,14 @@ class RestoreAttachmentJob private constructor(
|
||||
}
|
||||
|
||||
override fun onAdded() {
|
||||
Log.i(TAG, "onAdded() messageId: $messageId attachmentId: $attachmentId manual: $manual")
|
||||
if (offloaded) {
|
||||
Log.i(TAG, "onAdded() messageId: $messageId attachmentId: $attachmentId")
|
||||
|
||||
val attachmentId = AttachmentId(attachmentId)
|
||||
val attachment = SignalDatabase.attachments.getAttachment(attachmentId)
|
||||
val pending = attachment != null && attachment.transferState != AttachmentTable.TRANSFER_PROGRESS_DONE && attachment.transferState != AttachmentTable.TRANSFER_PROGRESS_PERMANENT_FAILURE
|
||||
if (attachment?.transferState == AttachmentTable.TRANSFER_NEEDS_RESTORE || attachment?.transferState == AttachmentTable.TRANSFER_RESTORE_OFFLOADED) {
|
||||
Log.i(TAG, "onAdded() Marking attachment restore progress as 'started'")
|
||||
SignalDatabase.attachments.setTransferState(messageId, attachmentId, AttachmentTable.TRANSFER_RESTORE_IN_PROGRESS)
|
||||
val attachment = SignalDatabase.attachments.getAttachment(attachmentId)
|
||||
if (attachment?.transferState == AttachmentTable.TRANSFER_NEEDS_RESTORE || attachment?.transferState == AttachmentTable.TRANSFER_RESTORE_OFFLOADED) {
|
||||
Log.i(TAG, "onAdded() Marking attachment restore progress as 'started'")
|
||||
SignalDatabase.attachments.setTransferState(messageId, attachmentId, AttachmentTable.TRANSFER_RESTORE_IN_PROGRESS)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -152,17 +107,12 @@ class RestoreAttachmentJob private constructor(
|
||||
if (!SignalDatabase.messages.isStory(messageId)) {
|
||||
AppDependencies.messageNotifier.updateNotification(context, forConversation(0))
|
||||
}
|
||||
|
||||
if (SignalDatabase.attachments.getRemainingRestorableAttachmentSize() == 0L) {
|
||||
SignalStore.backup.totalRestorableAttachmentSize = 0L
|
||||
}
|
||||
}
|
||||
|
||||
@Throws(IOException::class, RetryLaterException::class)
|
||||
fun doWork() {
|
||||
Log.i(TAG, "onRun() messageId: $messageId attachmentId: $attachmentId manual: $manual")
|
||||
Log.i(TAG, "onRun() messageId: $messageId attachmentId: $attachmentId")
|
||||
|
||||
val attachmentId = AttachmentId(attachmentId)
|
||||
val attachment = SignalDatabase.attachments.getAttachment(attachmentId)
|
||||
|
||||
if (attachment == null) {
|
||||
@@ -177,23 +127,18 @@ class RestoreAttachmentJob private constructor(
|
||||
|
||||
if (attachment.transferState != AttachmentTable.TRANSFER_NEEDS_RESTORE &&
|
||||
attachment.transferState != AttachmentTable.TRANSFER_RESTORE_IN_PROGRESS &&
|
||||
(attachment.transferState != AttachmentTable.TRANSFER_RESTORE_OFFLOADED || restoreMode == RestoreMode.THUMBNAIL)
|
||||
(attachment.transferState != AttachmentTable.TRANSFER_RESTORE_OFFLOADED)
|
||||
) {
|
||||
Log.w(TAG, "Attachment does not need to be restored.")
|
||||
return
|
||||
}
|
||||
if (attachment.thumbnailUri == null && (restoreMode == RestoreMode.THUMBNAIL || restoreMode == RestoreMode.BOTH)) {
|
||||
downloadThumbnail(attachmentId, attachment)
|
||||
}
|
||||
if (restoreMode == RestoreMode.ORIGINAL || restoreMode == RestoreMode.BOTH) {
|
||||
retrieveAttachment(messageId, attachmentId, attachment)
|
||||
}
|
||||
|
||||
retrieveAttachment(messageId, attachmentId, attachment)
|
||||
}
|
||||
|
||||
override fun onFailure() {
|
||||
Log.w(TAG, format(this, "onFailure() messageId: $messageId attachmentId: $attachmentId manual: $manual"))
|
||||
Log.w(TAG, format(this, "onFailure() messageId: $messageId attachmentId: $attachmentId"))
|
||||
|
||||
val attachmentId = AttachmentId(attachmentId)
|
||||
markFailed(messageId, attachmentId)
|
||||
}
|
||||
|
||||
@@ -206,7 +151,8 @@ class RestoreAttachmentJob private constructor(
|
||||
private fun retrieveAttachment(
|
||||
messageId: Long,
|
||||
attachmentId: AttachmentId,
|
||||
attachment: DatabaseAttachment
|
||||
attachment: DatabaseAttachment,
|
||||
forceTransitTier: Boolean = false
|
||||
) {
|
||||
val maxReceiveSize: Long = RemoteConfig.maxAttachmentReceiveSizeBytes
|
||||
val attachmentFile: File = SignalDatabase.attachments.getOrCreateTransferFile(attachmentId)
|
||||
@@ -218,9 +164,9 @@ class RestoreAttachmentJob private constructor(
|
||||
throw MmsException("Attachment too large, failing download")
|
||||
}
|
||||
|
||||
useArchiveCdn = if (SignalStore.backup.backsUpMedia && (forceArchiveDownload || attachment.remoteLocation == null)) {
|
||||
useArchiveCdn = if (SignalStore.backup.backsUpMedia && !forceTransitTier) {
|
||||
if (attachment.archiveMediaName.isNullOrEmpty()) {
|
||||
throw InvalidPartException("Invalid attachment configuration")
|
||||
throw InvalidAttachmentException("Invalid attachment configuration")
|
||||
}
|
||||
true
|
||||
} else {
|
||||
@@ -228,7 +174,7 @@ class RestoreAttachmentJob private constructor(
|
||||
}
|
||||
|
||||
val messageReceiver = AppDependencies.signalServiceMessageReceiver
|
||||
val pointer = createAttachmentPointer(attachment, useArchiveCdn)
|
||||
val pointer = attachment.createArchiveAttachmentPointer(useArchiveCdn)
|
||||
|
||||
val progressListener = object : SignalServiceAttachment.ProgressListener {
|
||||
override fun onAttachmentProgress(total: Long, progress: Long) {
|
||||
@@ -275,15 +221,14 @@ class RestoreAttachmentJob private constructor(
|
||||
} else {
|
||||
throw IOException("Failed to delete temp download file following range exception")
|
||||
}
|
||||
} catch (e: InvalidPartException) {
|
||||
} catch (e: InvalidAttachmentException) {
|
||||
Log.w(TAG, "Experienced exception while trying to download an attachment.", e)
|
||||
markFailed(messageId, attachmentId)
|
||||
} catch (e: NonSuccessfulResponseCodeException) {
|
||||
if (SignalStore.backup.backsUpMedia) {
|
||||
if (e.code == 404 && !useArchiveCdn && attachment.archiveMediaName?.isNotEmpty() == true) {
|
||||
Log.i(TAG, "Retrying download from archive CDN")
|
||||
forceArchiveDownload = true
|
||||
retrieveAttachment(messageId, attachmentId, attachment)
|
||||
if (e.code == 404 && !forceTransitTier && attachment.remoteLocation?.isNotBlank() == true) {
|
||||
Log.i(TAG, "Retrying download from transit CDN")
|
||||
retrieveAttachment(messageId, attachmentId, attachment, true)
|
||||
return
|
||||
} else if (e.code == 401 && useArchiveCdn) {
|
||||
SignalStore.backup.cdnReadCredentials = null
|
||||
@@ -310,212 +255,22 @@ class RestoreAttachmentJob private constructor(
|
||||
}
|
||||
}
|
||||
|
||||
@Throws(InvalidPartException::class)
|
||||
private fun createAttachmentPointer(attachment: DatabaseAttachment, useArchiveCdn: Boolean): SignalServiceAttachmentPointer {
|
||||
if (TextUtils.isEmpty(attachment.remoteKey)) {
|
||||
throw InvalidPartException("empty encrypted key")
|
||||
}
|
||||
|
||||
return try {
|
||||
val remoteData: RemoteData = if (useArchiveCdn) {
|
||||
val backupKey = SignalStore.svr.getOrCreateMasterKey().deriveBackupKey()
|
||||
val backupDirectories = BackupRepository.getCdnBackupDirectories().successOrThrow()
|
||||
|
||||
RemoteData(
|
||||
remoteId = SignalServiceAttachmentRemoteId.Backup(
|
||||
backupDir = backupDirectories.backupDir,
|
||||
mediaDir = backupDirectories.mediaDir,
|
||||
mediaId = backupKey.deriveMediaId(MediaName(attachment.archiveMediaName!!)).encode()
|
||||
),
|
||||
cdnNumber = attachment.archiveCdn
|
||||
)
|
||||
} else {
|
||||
if (attachment.remoteLocation.isNullOrEmpty()) {
|
||||
throw InvalidPartException("empty content id")
|
||||
}
|
||||
|
||||
RemoteData(
|
||||
remoteId = SignalServiceAttachmentRemoteId.from(attachment.remoteLocation),
|
||||
cdnNumber = attachment.cdn.cdnNumber
|
||||
)
|
||||
}
|
||||
|
||||
val key = Base64.decode(attachment.remoteKey!!)
|
||||
|
||||
if (attachment.remoteDigest != null) {
|
||||
Log.i(TAG, "Downloading attachment with digest: " + Hex.toString(attachment.remoteDigest))
|
||||
} else {
|
||||
Log.i(TAG, "Downloading attachment with no digest...")
|
||||
}
|
||||
|
||||
SignalServiceAttachmentPointer(
|
||||
remoteData.cdnNumber,
|
||||
remoteData.remoteId,
|
||||
null,
|
||||
key,
|
||||
Optional.of(Util.toIntExact(attachment.size)),
|
||||
Optional.empty(),
|
||||
0,
|
||||
0,
|
||||
Optional.ofNullable(attachment.remoteDigest),
|
||||
Optional.ofNullable(attachment.getIncrementalDigest()),
|
||||
attachment.incrementalMacChunkSize,
|
||||
Optional.ofNullable(attachment.fileName),
|
||||
attachment.voiceNote,
|
||||
attachment.borderless,
|
||||
attachment.videoGif,
|
||||
Optional.empty(),
|
||||
Optional.ofNullable(attachment.blurHash).map { it.hash },
|
||||
attachment.uploadTimestamp,
|
||||
attachment.uuid
|
||||
)
|
||||
} catch (e: IOException) {
|
||||
Log.w(TAG, e)
|
||||
throw InvalidPartException(e)
|
||||
} catch (e: ArithmeticException) {
|
||||
Log.w(TAG, e)
|
||||
throw InvalidPartException(e)
|
||||
}
|
||||
}
|
||||
|
||||
@Throws(InvalidPartException::class)
|
||||
private fun createThumbnailPointer(attachment: DatabaseAttachment): SignalServiceAttachmentPointer {
|
||||
if (TextUtils.isEmpty(attachment.remoteKey)) {
|
||||
throw InvalidPartException("empty encrypted key")
|
||||
}
|
||||
|
||||
val backupKey = SignalStore.svr.getOrCreateMasterKey().deriveBackupKey()
|
||||
val backupDirectories = BackupRepository.getCdnBackupDirectories().successOrThrow()
|
||||
return try {
|
||||
val key = backupKey.deriveThumbnailTransitKey(attachment.getThumbnailMediaName())
|
||||
val mediaId = backupKey.deriveMediaId(attachment.getThumbnailMediaName()).encode()
|
||||
SignalServiceAttachmentPointer(
|
||||
attachment.archiveThumbnailCdn,
|
||||
SignalServiceAttachmentRemoteId.Backup(
|
||||
backupDir = backupDirectories.backupDir,
|
||||
mediaDir = backupDirectories.mediaDir,
|
||||
mediaId = mediaId
|
||||
),
|
||||
null,
|
||||
key,
|
||||
Optional.empty(),
|
||||
Optional.empty(),
|
||||
0,
|
||||
0,
|
||||
Optional.empty(),
|
||||
Optional.empty(),
|
||||
attachment.incrementalMacChunkSize,
|
||||
Optional.empty(),
|
||||
attachment.voiceNote,
|
||||
attachment.borderless,
|
||||
attachment.videoGif,
|
||||
Optional.empty(),
|
||||
Optional.ofNullable(attachment.blurHash).map { it.hash },
|
||||
attachment.uploadTimestamp,
|
||||
attachment.uuid
|
||||
)
|
||||
} catch (e: IOException) {
|
||||
Log.w(TAG, e)
|
||||
throw InvalidPartException(e)
|
||||
} catch (e: ArithmeticException) {
|
||||
Log.w(TAG, e)
|
||||
throw InvalidPartException(e)
|
||||
}
|
||||
}
|
||||
|
||||
private fun downloadThumbnail(attachmentId: AttachmentId, attachment: DatabaseAttachment) {
|
||||
if (attachment.thumbnailRestoreState == AttachmentTable.ThumbnailRestoreState.FINISHED) {
|
||||
Log.w(TAG, "$attachmentId already has thumbnail downloaded")
|
||||
return
|
||||
}
|
||||
if (attachment.thumbnailRestoreState == AttachmentTable.ThumbnailRestoreState.NONE) {
|
||||
Log.w(TAG, "$attachmentId has no thumbnail state")
|
||||
return
|
||||
}
|
||||
if (attachment.thumbnailRestoreState == AttachmentTable.ThumbnailRestoreState.PERMANENT_FAILURE) {
|
||||
Log.w(TAG, "$attachmentId thumbnail permanently failed")
|
||||
return
|
||||
}
|
||||
if (attachment.archiveMediaName == null) {
|
||||
Log.w(TAG, "$attachmentId was never archived! Cannot proceed.")
|
||||
return
|
||||
}
|
||||
|
||||
val maxThumbnailSize: Long = RemoteConfig.maxAttachmentReceiveSizeBytes
|
||||
val thumbnailTransferFile: File = SignalDatabase.attachments.createArchiveThumbnailTransferFile()
|
||||
val thumbnailFile: File = SignalDatabase.attachments.createArchiveThumbnailTransferFile()
|
||||
|
||||
val progressListener = object : SignalServiceAttachment.ProgressListener {
|
||||
override fun onAttachmentProgress(total: Long, progress: Long) {
|
||||
}
|
||||
|
||||
override fun shouldCancel(): Boolean {
|
||||
return this@RestoreAttachmentJob.isCanceled
|
||||
}
|
||||
}
|
||||
|
||||
val cdnCredentials = BackupRepository.getCdnReadCredentials(attachment.archiveCdn).successOrThrow().headers
|
||||
val messageReceiver = AppDependencies.signalServiceMessageReceiver
|
||||
val pointer = createThumbnailPointer(attachment)
|
||||
|
||||
Log.w(TAG, "Downloading thumbnail for $attachmentId mediaName=${attachment.getThumbnailMediaName()}")
|
||||
val downloadResult = messageReceiver
|
||||
.retrieveArchivedAttachment(
|
||||
SignalStore.svr.getOrCreateMasterKey().deriveBackupKey().deriveMediaSecrets(attachment.getThumbnailMediaName()),
|
||||
cdnCredentials,
|
||||
thumbnailTransferFile,
|
||||
pointer,
|
||||
thumbnailFile,
|
||||
maxThumbnailSize,
|
||||
true,
|
||||
progressListener
|
||||
)
|
||||
|
||||
SignalDatabase.attachments.finalizeAttachmentThumbnailAfterDownload(attachmentId, attachment.archiveMediaId!!, downloadResult.dataStream, thumbnailTransferFile)
|
||||
}
|
||||
|
||||
private fun markFailed(messageId: Long, attachmentId: AttachmentId) {
|
||||
try {
|
||||
SignalDatabase.attachments.setTransferProgressFailed(attachmentId, messageId)
|
||||
} catch (e: MmsException) {
|
||||
Log.w(TAG, e)
|
||||
}
|
||||
SignalDatabase.attachments.setTransferProgressFailed(attachmentId, messageId)
|
||||
}
|
||||
|
||||
private fun markPermanentlyFailed(messageId: Long, attachmentId: AttachmentId) {
|
||||
SignalDatabase.attachments.setTransferProgressPermanentFailure(attachmentId, messageId)
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
internal class InvalidPartException : Exception {
|
||||
constructor(s: String?) : super(s)
|
||||
constructor(e: Exception?) : super(e)
|
||||
}
|
||||
|
||||
enum class RestoreMode(val value: Int) {
|
||||
THUMBNAIL(0),
|
||||
ORIGINAL(1),
|
||||
BOTH(2);
|
||||
|
||||
companion object {
|
||||
fun deserialize(value: Int): RestoreMode {
|
||||
return values().firstOrNull { it.value == value } ?: ORIGINAL
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private data class RemoteData(val remoteId: SignalServiceAttachmentRemoteId, val cdnNumber: Int)
|
||||
|
||||
class Factory : Job.Factory<RestoreAttachmentJob?> {
|
||||
override fun create(parameters: Parameters, serializedData: ByteArray?): RestoreAttachmentJob {
|
||||
val data = JsonJobData.deserialize(serializedData)
|
||||
val data = RestoreAttachmentJobData.ADAPTER.decode(serializedData!!)
|
||||
return RestoreAttachmentJob(
|
||||
parameters = parameters,
|
||||
messageId = data.getLong(KEY_MESSAGE_ID),
|
||||
attachmentId = AttachmentId(data.getLong(KEY_ATTACHMENT_ID)),
|
||||
manual = data.getBoolean(KEY_MANUAL),
|
||||
forceArchiveDownload = data.getBooleanOrDefault(KEY_FORCE_ARCHIVE, false),
|
||||
restoreMode = RestoreMode.deserialize(data.getIntOrDefault(KEY_RESTORE_MODE, RestoreMode.ORIGINAL.value))
|
||||
messageId = data.messageId,
|
||||
attachmentId = AttachmentId(data.attachmentId),
|
||||
offloaded = data.offloaded
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,13 +4,13 @@
|
||||
*/
|
||||
package org.thoughtcrime.securesms.jobs
|
||||
|
||||
import android.text.TextUtils
|
||||
import androidx.annotation.VisibleForTesting
|
||||
import org.signal.core.util.logging.Log
|
||||
import org.signal.libsignal.protocol.InvalidMessageException
|
||||
import org.thoughtcrime.securesms.attachments.AttachmentId
|
||||
import org.thoughtcrime.securesms.attachments.DatabaseAttachment
|
||||
import org.thoughtcrime.securesms.attachments.InvalidAttachmentException
|
||||
import org.thoughtcrime.securesms.backup.v2.BackupRepository
|
||||
import org.thoughtcrime.securesms.backup.v2.BackupRepository.getThumbnailMediaName
|
||||
import org.thoughtcrime.securesms.backup.v2.database.createArchiveThumbnailPointer
|
||||
import org.thoughtcrime.securesms.database.AttachmentTable
|
||||
import org.thoughtcrime.securesms.database.SignalDatabase
|
||||
import org.thoughtcrime.securesms.dependencies.AppDependencies
|
||||
@@ -19,16 +19,11 @@ import org.thoughtcrime.securesms.jobmanager.JobLogger.format
|
||||
import org.thoughtcrime.securesms.jobmanager.JsonJobData
|
||||
import org.thoughtcrime.securesms.jobmanager.impl.NetworkConstraint
|
||||
import org.thoughtcrime.securesms.keyvalue.SignalStore
|
||||
import org.thoughtcrime.securesms.notifications.v2.ConversationId.Companion.forConversation
|
||||
import org.thoughtcrime.securesms.transport.RetryLaterException
|
||||
import org.thoughtcrime.securesms.util.RemoteConfig
|
||||
import org.whispersystems.signalservice.api.messages.SignalServiceAttachment
|
||||
import org.whispersystems.signalservice.api.messages.SignalServiceAttachmentPointer
|
||||
import org.whispersystems.signalservice.api.messages.SignalServiceAttachmentRemoteId
|
||||
import org.whispersystems.signalservice.api.push.exceptions.PushNetworkException
|
||||
import org.whispersystems.signalservice.api.push.exceptions.MissingConfigurationException
|
||||
import java.io.File
|
||||
import java.io.IOException
|
||||
import java.util.Optional
|
||||
import java.util.concurrent.TimeUnit
|
||||
|
||||
/**
|
||||
@@ -37,7 +32,7 @@ import java.util.concurrent.TimeUnit
|
||||
class RestoreAttachmentThumbnailJob private constructor(
|
||||
parameters: Parameters,
|
||||
private val messageId: Long,
|
||||
attachmentId: AttachmentId
|
||||
val attachmentId: AttachmentId
|
||||
) : BaseJob(parameters) {
|
||||
|
||||
companion object {
|
||||
@@ -54,8 +49,6 @@ class RestoreAttachmentThumbnailJob private constructor(
|
||||
}
|
||||
}
|
||||
|
||||
private val attachmentId: Long
|
||||
|
||||
constructor(messageId: Long, attachmentId: AttachmentId, highPriority: Boolean = false) : this(
|
||||
Parameters.Builder()
|
||||
.setQueue(constructQueueString(attachmentId))
|
||||
@@ -68,14 +61,10 @@ class RestoreAttachmentThumbnailJob private constructor(
|
||||
attachmentId
|
||||
)
|
||||
|
||||
init {
|
||||
this.attachmentId = attachmentId.id
|
||||
}
|
||||
|
||||
override fun serialize(): ByteArray? {
|
||||
return JsonJobData.Builder()
|
||||
.putLong(KEY_MESSAGE_ID, messageId)
|
||||
.putLong(KEY_ATTACHMENT_ID, attachmentId)
|
||||
.putLong(KEY_ATTACHMENT_ID, attachmentId.id)
|
||||
.serialize()
|
||||
}
|
||||
|
||||
@@ -83,35 +72,10 @@ class RestoreAttachmentThumbnailJob private constructor(
|
||||
return KEY
|
||||
}
|
||||
|
||||
override fun onAdded() {
|
||||
Log.i(TAG, "onAdded() messageId: $messageId attachmentId: $attachmentId")
|
||||
|
||||
val attachmentId = AttachmentId(attachmentId)
|
||||
val attachment = SignalDatabase.attachments.getAttachment(attachmentId)
|
||||
val pending = attachment != null &&
|
||||
attachment.thumbnailRestoreState != AttachmentTable.ThumbnailRestoreState.FINISHED &&
|
||||
attachment.thumbnailRestoreState != AttachmentTable.ThumbnailRestoreState.PERMANENT_FAILURE &&
|
||||
attachment.thumbnailRestoreState != AttachmentTable.ThumbnailRestoreState.NONE
|
||||
if (pending) {
|
||||
Log.i(TAG, "onAdded() Marking thumbnail restore progress as 'started'")
|
||||
SignalDatabase.attachments.setThumbnailTransferState(messageId, attachmentId, AttachmentTable.ThumbnailRestoreState.IN_PROGRESS)
|
||||
}
|
||||
}
|
||||
|
||||
@Throws(Exception::class)
|
||||
@Throws(Exception::class, IOException::class, InvalidAttachmentException::class, InvalidMessageException::class, MissingConfigurationException::class)
|
||||
public override fun onRun() {
|
||||
doWork()
|
||||
|
||||
if (!SignalDatabase.messages.isStory(messageId)) {
|
||||
AppDependencies.messageNotifier.updateNotification(context, forConversation(0))
|
||||
}
|
||||
}
|
||||
|
||||
@Throws(IOException::class, RetryLaterException::class)
|
||||
fun doWork() {
|
||||
Log.i(TAG, "onRun() messageId: $messageId attachmentId: $attachmentId")
|
||||
|
||||
val attachmentId = AttachmentId(attachmentId)
|
||||
val attachment = SignalDatabase.attachments.getAttachment(attachmentId)
|
||||
|
||||
if (attachment == null) {
|
||||
@@ -119,79 +83,21 @@ class RestoreAttachmentThumbnailJob private constructor(
|
||||
return
|
||||
}
|
||||
|
||||
downloadThumbnail(attachmentId, attachment)
|
||||
}
|
||||
|
||||
override fun onFailure() {
|
||||
Log.w(TAG, format(this, "onFailure() thumbnail messageId: $messageId attachmentId: $attachmentId "))
|
||||
|
||||
val attachmentId = AttachmentId(attachmentId)
|
||||
markFailed(messageId, attachmentId)
|
||||
}
|
||||
|
||||
override fun onShouldRetry(exception: Exception): Boolean {
|
||||
return exception is PushNetworkException ||
|
||||
exception is RetryLaterException
|
||||
}
|
||||
|
||||
@Throws(InvalidPartException::class)
|
||||
private fun createThumbnailPointer(attachment: DatabaseAttachment): SignalServiceAttachmentPointer {
|
||||
if (TextUtils.isEmpty(attachment.remoteKey)) {
|
||||
throw InvalidPartException("empty encrypted key")
|
||||
}
|
||||
|
||||
val backupKey = SignalStore.svr.getOrCreateMasterKey().deriveBackupKey()
|
||||
val backupDirectories = BackupRepository.getCdnBackupDirectories().successOrThrow()
|
||||
return try {
|
||||
val key = backupKey.deriveThumbnailTransitKey(attachment.getThumbnailMediaName())
|
||||
val mediaId = backupKey.deriveMediaId(attachment.getThumbnailMediaName()).encode()
|
||||
SignalServiceAttachmentPointer(
|
||||
attachment.archiveThumbnailCdn,
|
||||
SignalServiceAttachmentRemoteId.Backup(
|
||||
backupDir = backupDirectories.backupDir,
|
||||
mediaDir = backupDirectories.mediaDir,
|
||||
mediaId = mediaId
|
||||
),
|
||||
null,
|
||||
key,
|
||||
Optional.empty(),
|
||||
Optional.empty(),
|
||||
0,
|
||||
0,
|
||||
Optional.empty(),
|
||||
Optional.empty(),
|
||||
attachment.incrementalMacChunkSize,
|
||||
Optional.empty(),
|
||||
attachment.voiceNote,
|
||||
attachment.borderless,
|
||||
attachment.videoGif,
|
||||
Optional.empty(),
|
||||
Optional.ofNullable(attachment.blurHash).map { it.hash },
|
||||
attachment.uploadTimestamp,
|
||||
attachment.uuid
|
||||
)
|
||||
} catch (e: IOException) {
|
||||
Log.w(TAG, e)
|
||||
throw InvalidPartException(e)
|
||||
} catch (e: ArithmeticException) {
|
||||
Log.w(TAG, e)
|
||||
throw InvalidPartException(e)
|
||||
}
|
||||
}
|
||||
|
||||
private fun downloadThumbnail(attachmentId: AttachmentId, attachment: DatabaseAttachment) {
|
||||
if (attachment.thumbnailRestoreState == AttachmentTable.ThumbnailRestoreState.FINISHED) {
|
||||
Log.w(TAG, "$attachmentId already has thumbnail downloaded")
|
||||
return
|
||||
}
|
||||
|
||||
if (attachment.thumbnailRestoreState == AttachmentTable.ThumbnailRestoreState.NONE) {
|
||||
Log.w(TAG, "$attachmentId has no thumbnail state")
|
||||
return
|
||||
}
|
||||
|
||||
if (attachment.thumbnailRestoreState == AttachmentTable.ThumbnailRestoreState.PERMANENT_FAILURE) {
|
||||
Log.w(TAG, "$attachmentId thumbnail permanently failed")
|
||||
return
|
||||
}
|
||||
|
||||
if (attachment.archiveMediaName == null) {
|
||||
Log.w(TAG, "$attachmentId was never archived! Cannot proceed.")
|
||||
return
|
||||
@@ -202,20 +108,15 @@ class RestoreAttachmentThumbnailJob private constructor(
|
||||
val thumbnailFile: File = SignalDatabase.attachments.createArchiveThumbnailTransferFile()
|
||||
|
||||
val progressListener = object : SignalServiceAttachment.ProgressListener {
|
||||
override fun onAttachmentProgress(total: Long, progress: Long) {
|
||||
}
|
||||
|
||||
override fun shouldCancel(): Boolean {
|
||||
return this@RestoreAttachmentThumbnailJob.isCanceled
|
||||
}
|
||||
override fun onAttachmentProgress(total: Long, progress: Long) = Unit
|
||||
override fun shouldCancel(): Boolean = this@RestoreAttachmentThumbnailJob.isCanceled
|
||||
}
|
||||
|
||||
val cdnCredentials = BackupRepository.getCdnReadCredentials(attachment.archiveCdn).successOrThrow().headers
|
||||
val messageReceiver = AppDependencies.signalServiceMessageReceiver
|
||||
val pointer = createThumbnailPointer(attachment)
|
||||
val pointer = attachment.createArchiveThumbnailPointer()
|
||||
|
||||
Log.w(TAG, "Downloading thumbnail for $attachmentId")
|
||||
val downloadResult = messageReceiver
|
||||
Log.i(TAG, "Downloading thumbnail for $attachmentId")
|
||||
val downloadResult = AppDependencies.signalServiceMessageReceiver
|
||||
.retrieveArchivedAttachment(
|
||||
SignalStore.svr.getOrCreateMasterKey().deriveBackupKey().deriveMediaSecrets(attachment.getThumbnailMediaName()),
|
||||
cdnCredentials,
|
||||
@@ -228,16 +129,20 @@ class RestoreAttachmentThumbnailJob private constructor(
|
||||
)
|
||||
|
||||
SignalDatabase.attachments.finalizeAttachmentThumbnailAfterDownload(attachmentId, attachment.archiveMediaId!!, downloadResult.dataStream, thumbnailTransferFile)
|
||||
|
||||
if (!SignalDatabase.messages.isStory(messageId)) {
|
||||
AppDependencies.messageNotifier.updateNotification(context)
|
||||
}
|
||||
}
|
||||
|
||||
private fun markFailed(messageId: Long, attachmentId: AttachmentId) {
|
||||
override fun onFailure() {
|
||||
Log.w(TAG, format(this, "onFailure() thumbnail messageId: $messageId attachmentId: $attachmentId "))
|
||||
|
||||
SignalDatabase.attachments.setThumbnailRestoreProgressFailed(attachmentId, messageId)
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
internal class InvalidPartException : Exception {
|
||||
constructor(s: String?) : super(s)
|
||||
constructor(e: Exception?) : super(e)
|
||||
override fun onShouldRetry(exception: Exception): Boolean {
|
||||
return exception is IOException
|
||||
}
|
||||
|
||||
class Factory : Job.Factory<RestoreAttachmentThumbnailJob?> {
|
||||
|
||||
@@ -15,7 +15,7 @@ import org.signal.libsignal.protocol.InvalidMessageException
|
||||
import org.thoughtcrime.securesms.attachments.AttachmentId
|
||||
import org.thoughtcrime.securesms.backup.v2.local.ArchiveFileSystem
|
||||
import org.thoughtcrime.securesms.database.AttachmentTable
|
||||
import org.thoughtcrime.securesms.database.AttachmentTable.LocalRestorableAttachment
|
||||
import org.thoughtcrime.securesms.database.AttachmentTable.RestorableAttachment
|
||||
import org.thoughtcrime.securesms.database.SignalDatabase
|
||||
import org.thoughtcrime.securesms.dependencies.AppDependencies
|
||||
import org.thoughtcrime.securesms.jobmanager.Job
|
||||
@@ -47,9 +47,9 @@ class RestoreLocalAttachmentJob private constructor(
|
||||
var restoreAttachmentJobs: MutableList<Job>
|
||||
|
||||
do {
|
||||
val possibleRestorableAttachments: List<LocalRestorableAttachment> = SignalDatabase.attachments.getLocalRestorableAttachments(500)
|
||||
val restorableAttachments = ArrayList<LocalRestorableAttachment>(possibleRestorableAttachments.size)
|
||||
val notRestorableAttachments = ArrayList<LocalRestorableAttachment>(possibleRestorableAttachments.size)
|
||||
val possibleRestorableAttachments: List<RestorableAttachment> = SignalDatabase.attachments.getRestorableAttachments(500)
|
||||
val restorableAttachments = ArrayList<RestorableAttachment>(possibleRestorableAttachments.size)
|
||||
val notRestorableAttachments = ArrayList<RestorableAttachment>(possibleRestorableAttachments.size)
|
||||
|
||||
restoreAttachmentJobs = ArrayList(possibleRestorableAttachments.size)
|
||||
|
||||
@@ -71,8 +71,8 @@ class RestoreLocalAttachmentJob private constructor(
|
||||
}
|
||||
|
||||
SignalDatabase.rawDatabase.withinTransaction {
|
||||
SignalDatabase.attachments.setRestoreInProgressTransferState(restorableAttachments)
|
||||
SignalDatabase.attachments.setRestoreFailedTransferState(notRestorableAttachments)
|
||||
SignalDatabase.attachments.setRestoreTransferState(restorableAttachments, AttachmentTable.TRANSFER_RESTORE_IN_PROGRESS)
|
||||
SignalDatabase.attachments.setRestoreTransferState(notRestorableAttachments, AttachmentTable.TRANSFER_PROGRESS_FAILED)
|
||||
|
||||
SignalStore.backup.totalRestorableAttachmentSize = SignalDatabase.attachments.getRemainingRestorableAttachmentSize()
|
||||
AppDependencies.jobManager.addAll(restoreAttachmentJobs)
|
||||
@@ -92,7 +92,7 @@ class RestoreLocalAttachmentJob private constructor(
|
||||
}
|
||||
}
|
||||
|
||||
private constructor(queue: String, attachment: LocalRestorableAttachment, info: DocumentFileInfo) : this(
|
||||
private constructor(queue: String, attachment: RestorableAttachment, info: DocumentFileInfo) : this(
|
||||
Parameters.Builder()
|
||||
.setQueue(queue)
|
||||
.setLifespan(Parameters.IMMORTAL)
|
||||
|
||||
Reference in New Issue
Block a user