Refactor and cleanup backupv2 media restore.

This commit is contained in:
Cody Henthorne
2024-09-11 12:38:19 -04:00
parent baa6032770
commit 816006c67e
25 changed files with 498 additions and 671 deletions

View File

@@ -29,9 +29,6 @@ class DatabaseAttachment : Attachment {
@JvmField
val archiveCdn: Int
@JvmField
val archiveThumbnailCdn: Int
@JvmField
val archiveMediaName: String?
@@ -78,7 +75,6 @@ class DatabaseAttachment : Attachment {
uploadTimestamp: Long,
dataHash: String?,
archiveCdn: Int,
archiveThumbnailCdn: Int,
archiveMediaName: String?,
archiveMediaId: String?,
thumbnailRestoreState: AttachmentTable.ThumbnailRestoreState,
@@ -117,7 +113,6 @@ class DatabaseAttachment : Attachment {
this.hasArchiveThumbnail = hasArchiveThumbnail
this.displayOrder = displayOrder
this.archiveCdn = archiveCdn
this.archiveThumbnailCdn = archiveThumbnailCdn
this.archiveMediaName = archiveMediaName
this.archiveMediaId = archiveMediaId
this.thumbnailRestoreState = thumbnailRestoreState
@@ -131,7 +126,6 @@ class DatabaseAttachment : Attachment {
mmsId = parcel.readLong()
displayOrder = parcel.readInt()
archiveCdn = parcel.readInt()
archiveThumbnailCdn = parcel.readInt()
archiveMediaName = parcel.readString()
archiveMediaId = parcel.readString()
hasArchiveThumbnail = ParcelUtil.readBoolean(parcel)
@@ -147,7 +141,6 @@ class DatabaseAttachment : Attachment {
dest.writeLong(mmsId)
dest.writeInt(displayOrder)
dest.writeInt(archiveCdn)
dest.writeInt(archiveThumbnailCdn)
dest.writeString(archiveMediaName)
dest.writeString(archiveMediaId)
ParcelUtil.writeBoolean(dest, hasArchiveThumbnail)

View File

@@ -0,0 +1,14 @@
/*
* Copyright 2024 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.thoughtcrime.securesms.attachments
/**
* Thrown by jobs unable to rehydrate enough attachment information to download it.
*/
class InvalidAttachmentException : Exception {
constructor(s: String?) : super(s)
constructor(e: Exception?) : super(e)
}

View File

@@ -30,6 +30,7 @@ object BackupRestoreManager {
SignalExecutors.BOUNDED.execute {
synchronized(this) {
val restoringAttachments = messageRecords
.asSequence()
.mapNotNull { (it as? MmsMessageRecord?)?.slideDeck?.slides }
.flatten()
.mapNotNull { it.asAttachment() as? DatabaseAttachment }
@@ -41,10 +42,11 @@ object BackupRestoreManager {
.toSet()
reprioritizedAttachments += restoringAttachments.map { it.first }
val thumbnailJobs = restoringAttachments.map {
val (attachmentId, mmsId) = it
val thumbnailJobs = restoringAttachments.map { (attachmentId, mmsId) ->
RestoreAttachmentThumbnailJob(attachmentId = attachmentId, messageId = mmsId, highPriority = true)
}
if (thumbnailJobs.isNotEmpty()) {
AppDependencies.jobManager.addAll(thumbnailJobs)
}

View File

@@ -0,0 +1,129 @@
/*
* Copyright 2024 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.thoughtcrime.securesms.backup.v2.database
import android.text.TextUtils
import org.signal.core.util.Base64
import org.thoughtcrime.securesms.attachments.DatabaseAttachment
import org.thoughtcrime.securesms.attachments.InvalidAttachmentException
import org.thoughtcrime.securesms.backup.v2.BackupRepository
import org.thoughtcrime.securesms.backup.v2.BackupRepository.getThumbnailMediaName
import org.thoughtcrime.securesms.keyvalue.SignalStore
import org.thoughtcrime.securesms.util.Util
import org.whispersystems.signalservice.api.backup.MediaName
import org.whispersystems.signalservice.api.messages.SignalServiceAttachmentPointer
import org.whispersystems.signalservice.api.messages.SignalServiceAttachmentRemoteId
import java.io.IOException
import java.util.Optional
/**
* Creates a [SignalServiceAttachmentPointer] for the archived attachment of the given [DatabaseAttachment].
*/
@Throws(InvalidAttachmentException::class)
fun DatabaseAttachment.createArchiveAttachmentPointer(useArchiveCdn: Boolean): SignalServiceAttachmentPointer {
if (remoteKey.isNullOrBlank()) {
throw InvalidAttachmentException("empty encrypted key")
}
if (remoteDigest == null) {
throw InvalidAttachmentException("no digest")
}
return try {
val (remoteId, cdnNumber) = if (useArchiveCdn) {
val backupKey = SignalStore.svr.getOrCreateMasterKey().deriveBackupKey()
val backupDirectories = BackupRepository.getCdnBackupDirectories().successOrThrow()
val id = SignalServiceAttachmentRemoteId.Backup(
backupDir = backupDirectories.backupDir,
mediaDir = backupDirectories.mediaDir,
mediaId = backupKey.deriveMediaId(MediaName(archiveMediaName!!)).encode()
)
id to archiveCdn
} else {
if (remoteLocation.isNullOrEmpty()) {
throw InvalidAttachmentException("empty content id")
}
SignalServiceAttachmentRemoteId.from(remoteLocation) to cdn.cdnNumber
}
val key = Base64.decode(remoteKey)
SignalServiceAttachmentPointer(
cdnNumber = cdnNumber,
remoteId = remoteId,
contentType = null,
key = key,
size = Optional.of(Util.toIntExact(size)),
preview = Optional.empty(),
width = 0,
height = 0,
digest = Optional.ofNullable(remoteDigest),
incrementalDigest = Optional.ofNullable(getIncrementalDigest()),
incrementalMacChunkSize = incrementalMacChunkSize,
fileName = Optional.ofNullable(fileName),
voiceNote = voiceNote,
isBorderless = borderless,
isGif = videoGif,
caption = Optional.empty(),
blurHash = Optional.ofNullable(blurHash).map { it.hash },
uploadTimestamp = uploadTimestamp,
uuid = uuid
)
} catch (e: IOException) {
throw InvalidAttachmentException(e)
} catch (e: ArithmeticException) {
throw InvalidAttachmentException(e)
}
}
/**
* Creates a [SignalServiceAttachmentPointer] for an archived thumbnail of the given [DatabaseAttachment].
*/
@Throws(InvalidAttachmentException::class)
fun DatabaseAttachment.createArchiveThumbnailPointer(): SignalServiceAttachmentPointer {
if (TextUtils.isEmpty(remoteKey)) {
throw InvalidAttachmentException("empty encrypted key")
}
val backupKey = SignalStore.svr.getOrCreateMasterKey().deriveBackupKey()
val backupDirectories = BackupRepository.getCdnBackupDirectories().successOrThrow()
return try {
val key = backupKey.deriveThumbnailTransitKey(getThumbnailMediaName())
val mediaId = backupKey.deriveMediaId(getThumbnailMediaName()).encode()
SignalServiceAttachmentPointer(
cdnNumber = archiveCdn,
remoteId = SignalServiceAttachmentRemoteId.Backup(
backupDir = backupDirectories.backupDir,
mediaDir = backupDirectories.mediaDir,
mediaId = mediaId
),
contentType = null,
key = key,
size = Optional.empty(),
preview = Optional.empty(),
width = 0,
height = 0,
digest = Optional.empty(),
incrementalDigest = Optional.empty(),
incrementalMacChunkSize = incrementalMacChunkSize,
fileName = Optional.empty(),
voiceNote = voiceNote,
isBorderless = borderless,
isGif = videoGif,
caption = Optional.empty(),
blurHash = Optional.ofNullable(blurHash).map { it.hash },
uploadTimestamp = uploadTimestamp,
uuid = uuid
)
} catch (e: IOException) {
throw InvalidAttachmentException(e)
} catch (e: ArithmeticException) {
throw InvalidAttachmentException(e)
}
}

View File

@@ -184,7 +184,8 @@ class InternalBackupPlaygroundFragment : ComposeFragment() {
deleteArchivedMedia = { viewModel.deleteArchivedMedia(it) },
batchArchiveAttachmentMedia = { viewModel.archiveAttachmentMedia(it) },
batchDeleteBackupAttachmentMedia = { viewModel.deleteArchivedMedia(it) },
restoreArchivedMedia = { viewModel.restoreArchivedMedia(it) }
restoreArchivedMedia = { viewModel.restoreArchivedMedia(it, asThumbnail = false) },
restoreArchivedMediaThumbnail = { viewModel.restoreArchivedMedia(it, asThumbnail = true) }
)
}
)
@@ -450,7 +451,8 @@ fun MediaList(
deleteArchivedMedia: (InternalBackupPlaygroundViewModel.BackupAttachment) -> Unit,
batchArchiveAttachmentMedia: (Set<AttachmentId>) -> Unit,
batchDeleteBackupAttachmentMedia: (Set<AttachmentId>) -> Unit,
restoreArchivedMedia: (InternalBackupPlaygroundViewModel.BackupAttachment) -> Unit
restoreArchivedMedia: (InternalBackupPlaygroundViewModel.BackupAttachment) -> Unit,
restoreArchivedMediaThumbnail: (InternalBackupPlaygroundViewModel.BackupAttachment) -> Unit
) {
if (!enabled) {
Text(
@@ -553,6 +555,14 @@ fun MediaList(
}
)
DropdownMenuItem(
text = { Text("Pseudo Restore Thumbnail") },
onClick = {
selectionState = selectionState.copy(expandedOption = null)
restoreArchivedMediaThumbnail(attachment)
}
)
if (attachment.dbAttachment.dataHash != null && attachment.state == InternalBackupPlaygroundViewModel.BackupAttachment.State.UPLOADED_UNDOWNLOADED) {
DropdownMenuItem(
text = { Text("Re-copy with hash") },

View File

@@ -32,11 +32,12 @@ import org.thoughtcrime.securesms.database.MessageType
import org.thoughtcrime.securesms.database.SignalDatabase
import org.thoughtcrime.securesms.dependencies.AppDependencies
import org.thoughtcrime.securesms.jobs.ArchiveAttachmentJob
import org.thoughtcrime.securesms.jobs.AttachmentDownloadJob
import org.thoughtcrime.securesms.jobs.AttachmentUploadJob
import org.thoughtcrime.securesms.jobs.BackupMessagesJob
import org.thoughtcrime.securesms.jobs.BackupRestoreJob
import org.thoughtcrime.securesms.jobs.BackupRestoreMediaJob
import org.thoughtcrime.securesms.jobs.RestoreAttachmentJob
import org.thoughtcrime.securesms.jobs.RestoreAttachmentThumbnailJob
import org.thoughtcrime.securesms.jobs.RestoreLocalAttachmentJob
import org.thoughtcrime.securesms.jobs.SyncArchivedMediaJob
import org.thoughtcrime.securesms.keyvalue.SignalStore
@@ -346,7 +347,7 @@ class InternalBackupPlaygroundViewModel : ViewModel() {
}
}
fun restoreArchivedMedia(attachment: BackupAttachment) {
fun restoreArchivedMedia(attachment: BackupAttachment, asThumbnail: Boolean) {
disposables += Completable
.fromCallable {
val recipientId = SignalStore.releaseChannel.releaseChannelRecipientId!!
@@ -365,20 +366,29 @@ class InternalBackupPlaygroundViewModel : ViewModel() {
val insertMessage = SignalDatabase.messages.insertMessageInbox(message, threadId).get()
SignalDatabase.attachments.debugCopyAttachmentForArchiveRestore(
insertMessage.messageId,
attachment.dbAttachment
mmsId = insertMessage.messageId,
attachment = attachment.dbAttachment,
forThumbnail = asThumbnail
)
val archivedAttachment = SignalDatabase.attachments.getAttachmentsForMessage(insertMessage.messageId).first()
AppDependencies.jobManager.add(
AttachmentDownloadJob(
messageId = insertMessage.messageId,
attachmentId = archivedAttachment.attachmentId,
manual = false,
forceArchiveDownload = true
if (asThumbnail) {
AppDependencies.jobManager.add(
RestoreAttachmentThumbnailJob(
messageId = insertMessage.messageId,
attachmentId = archivedAttachment.attachmentId,
highPriority = false
)
)
)
} else {
AppDependencies.jobManager.add(
RestoreAttachmentJob(
messageId = insertMessage.messageId,
attachmentId = archivedAttachment.attachmentId
)
)
}
}
.subscribeOn(Schedulers.io())
.observeOn(Schedulers.single())

View File

@@ -55,7 +55,6 @@ import org.signal.core.util.requireBlob
import org.signal.core.util.requireBoolean
import org.signal.core.util.requireInt
import org.signal.core.util.requireLong
import org.signal.core.util.requireLongOrNull
import org.signal.core.util.requireNonNullBlob
import org.signal.core.util.requireNonNullString
import org.signal.core.util.requireObject
@@ -98,10 +97,12 @@ import org.thoughtcrime.securesms.util.StorageUtil
import org.thoughtcrime.securesms.util.Util
import org.thoughtcrime.securesms.video.EncryptedMediaDataSource
import org.whispersystems.signalservice.api.attachment.AttachmentUploadResult
import org.whispersystems.signalservice.api.backup.MediaId
import org.whispersystems.signalservice.api.crypto.AttachmentCipherOutputStream
import org.whispersystems.signalservice.api.util.UuidUtil
import org.whispersystems.signalservice.internal.crypto.PaddingInputStream
import org.whispersystems.signalservice.internal.util.JsonUtil
import java.io.ByteArrayInputStream
import java.io.File
import java.io.FileNotFoundException
import java.io.IOException
@@ -163,7 +164,6 @@ class AttachmentTable(
const val ARCHIVE_MEDIA_NAME = "archive_media_name"
const val ARCHIVE_MEDIA_ID = "archive_media_id"
const val ARCHIVE_THUMBNAIL_MEDIA_ID = "archive_thumbnail_media_id"
const val ARCHIVE_THUMBNAIL_CDN = "archive_thumbnail_cdn"
const val ARCHIVE_TRANSFER_FILE = "archive_transfer_file"
const val ARCHIVE_TRANSFER_STATE = "archive_transfer_state"
const val THUMBNAIL_RESTORE_STATE = "thumbnail_restore_state"
@@ -219,7 +219,6 @@ class AttachmentTable(
DATA_HASH_START,
DATA_HASH_END,
ARCHIVE_CDN,
ARCHIVE_THUMBNAIL_CDN,
ARCHIVE_MEDIA_NAME,
ARCHIVE_MEDIA_ID,
ARCHIVE_TRANSFER_FILE,
@@ -269,7 +268,6 @@ class AttachmentTable(
$ARCHIVE_MEDIA_ID TEXT DEFAULT NULL,
$ARCHIVE_TRANSFER_FILE TEXT DEFAULT NULL,
$ARCHIVE_TRANSFER_STATE INTEGER DEFAULT ${ArchiveTransferState.NONE.value},
$ARCHIVE_THUMBNAIL_CDN INTEGER DEFAULT 0,
$ARCHIVE_THUMBNAIL_MEDIA_ID TEXT DEFAULT NULL,
$THUMBNAIL_FILE TEXT DEFAULT NULL,
$THUMBNAIL_RANDOM BLOB DEFAULT NULL,
@@ -499,29 +497,16 @@ class AttachmentTable(
}
}
fun getRestorableAttachments(batchSize: Int): List<DatabaseAttachment> {
fun getRestorableAttachments(batchSize: Int): List<RestorableAttachment> {
return readableDatabase
.select(*PROJECTION)
.select(ID, MESSAGE_ID, DATA_SIZE, REMOTE_DIGEST, REMOTE_KEY)
.from(TABLE_NAME)
.where("$TRANSFER_STATE = ?", TRANSFER_NEEDS_RESTORE)
.limit(batchSize)
.orderBy("$ID DESC")
.run()
.readToList {
it.readAttachment()
}
}
fun getLocalRestorableAttachments(batchSize: Int): List<LocalRestorableAttachment> {
return readableDatabase
.select(*PROJECTION)
.from(TABLE_NAME)
.where("$TRANSFER_STATE = ?", TRANSFER_NEEDS_RESTORE)
.limit(batchSize)
.orderBy("$ID DESC")
.run()
.readToList {
LocalRestorableAttachment(
RestorableAttachment(
attachmentId = AttachmentId(it.requireLong(ID)),
mmsId = it.requireLong(MESSAGE_ID),
size = it.requireLong(DATA_SIZE),
@@ -871,17 +856,6 @@ class AttachmentTable(
notifyConversationListeners(threadId)
}
fun setThumbnailTransferState(messageId: Long, attachmentId: AttachmentId, thumbnailRestoreState: ThumbnailRestoreState) {
writableDatabase
.update(TABLE_NAME)
.values(THUMBNAIL_RESTORE_STATE to thumbnailRestoreState.value)
.where("$ID = ?", attachmentId.id)
.run()
val threadId = messages.getThreadIdForMessage(messageId)
notifyConversationListeners(threadId)
}
fun setTransferProgressFailed(attachmentId: AttachmentId, mmsId: Long) {
writableDatabase
.update(TABLE_NAME)
@@ -912,30 +886,46 @@ class AttachmentTable(
notifyConversationListeners(messages.getThreadIdForMessage(mmsId))
}
fun setRestoreInProgressTransferState(restorableAttachments: List<LocalRestorableAttachment>) {
setRestoreTransferState(
restorableAttachments = restorableAttachments,
prefix = "$TRANSFER_STATE = $TRANSFER_NEEDS_RESTORE",
state = TRANSFER_RESTORE_IN_PROGRESS
)
}
fun setThumbnailRestoreState(thumbnailAttachmentIds: List<AttachmentId>, thumbnailRestoreState: ThumbnailRestoreState) {
val prefix: String = when (thumbnailRestoreState) {
ThumbnailRestoreState.IN_PROGRESS -> {
"($THUMBNAIL_RESTORE_STATE = ${ThumbnailRestoreState.NEEDS_RESTORE.value} OR $THUMBNAIL_RESTORE_STATE = ${ThumbnailRestoreState.IN_PROGRESS.value}) AND"
}
else -> ""
}
fun setRestoreFailedTransferState(notRestorableAttachments: List<LocalRestorableAttachment>) {
setRestoreTransferState(
restorableAttachments = notRestorableAttachments,
prefix = "$TRANSFER_STATE != $TRANSFER_PROGRESS_PERMANENT_FAILURE",
state = TRANSFER_PROGRESS_FAILED
val setQueries = SqlUtil.buildCollectionQuery(
column = ID,
values = thumbnailAttachmentIds.map { it.id },
prefix = prefix
)
}
private fun setRestoreTransferState(restorableAttachments: List<LocalRestorableAttachment>, prefix: String, state: Int) {
writableDatabase.withinTransaction {
val setQueries = SqlUtil.buildCollectionQuery(
column = ID,
values = restorableAttachments.map { it.attachmentId.id },
prefix = "$prefix AND"
)
setQueries.forEach { query ->
writableDatabase
.update(TABLE_NAME)
.values(THUMBNAIL_RESTORE_STATE to thumbnailRestoreState.value)
.where(query.where, query.whereArgs)
.run()
}
}
}
fun setRestoreTransferState(restorableAttachments: Collection<RestorableAttachment>, state: Int) {
val prefix = when (state) {
TRANSFER_RESTORE_OFFLOADED -> "$TRANSFER_STATE != $TRANSFER_PROGRESS_PERMANENT_FAILURE AND"
TRANSFER_RESTORE_IN_PROGRESS -> "($TRANSFER_STATE = $TRANSFER_NEEDS_RESTORE OR $TRANSFER_STATE = $TRANSFER_RESTORE_OFFLOADED) AND"
TRANSFER_PROGRESS_FAILED -> "$TRANSFER_STATE != $TRANSFER_PROGRESS_PERMANENT_FAILURE AND"
else -> ""
}
val setQueries = SqlUtil.buildCollectionQuery(
column = ID,
values = restorableAttachments.map { it.attachmentId.id },
prefix = prefix
)
writableDatabase.withinTransaction {
setQueries.forEach { query ->
writableDatabase
.update(TABLE_NAME)
@@ -943,23 +933,6 @@ class AttachmentTable(
.where(query.where, query.whereArgs)
.run()
}
val threadQueries = SqlUtil.buildCollectionQuery(
column = MessageTable.ID,
values = restorableAttachments.map { it.mmsId }
)
val threads = mutableSetOf<Long>()
threadQueries.forEach { query ->
threads += readableDatabase
.select("DISTINCT ${MessageTable.THREAD_ID}")
.from(MessageTable.TABLE_NAME)
.where(query.where, query.whereArgs)
.run()
.readToList { it.requireLongOrNull(MessageTable.THREAD_ID) ?: -1 }
}
notifyConversationListeners(threads)
}
}
@@ -1103,6 +1076,30 @@ class AttachmentTable(
}
}
fun finalizeAttachmentThumbnailAfterUpload(
attachmentId: AttachmentId,
archiveMediaId: String,
archiveThumbnailMediaId: MediaId,
data: ByteArray
) {
Log.i(TAG, "[finalizeAttachmentThumbnailAfterUpload] Finalizing archive data for $attachmentId thumbnail.")
val fileWriteResult: DataFileWriteResult = writeToDataFile(newDataFile(context), ByteArrayInputStream(data), TransformProperties.empty())
writableDatabase.withinTransaction { db ->
val values = contentValuesOf(
THUMBNAIL_FILE to fileWriteResult.file.absolutePath,
THUMBNAIL_RANDOM to fileWriteResult.random,
THUMBNAIL_RESTORE_STATE to ThumbnailRestoreState.FINISHED.value,
ARCHIVE_THUMBNAIL_MEDIA_ID to archiveThumbnailMediaId.encode()
)
db.update(TABLE_NAME)
.values(values)
.where("$ARCHIVE_MEDIA_ID = ? OR $ID = ?", archiveMediaId, attachmentId)
.run()
}
}
/**
* Needs to be called after an attachment is successfully uploaded. Writes metadata around it's final remote location, as well as calculates
* it's ending hash, which is critical for backups.
@@ -1334,7 +1331,8 @@ class AttachmentTable(
fun debugCopyAttachmentForArchiveRestore(
mmsId: Long,
attachment: DatabaseAttachment
attachment: DatabaseAttachment,
forThumbnail: Boolean
) {
val copy =
"""
@@ -1365,7 +1363,9 @@ class AttachmentTable(
$DATA_HASH_END,
$ARCHIVE_MEDIA_ID,
$ARCHIVE_MEDIA_NAME,
$ARCHIVE_CDN
$ARCHIVE_CDN,
$ARCHIVE_THUMBNAIL_MEDIA_ID,
$THUMBNAIL_RESTORE_STATE
)
SELECT
$mmsId,
@@ -1393,7 +1393,9 @@ class AttachmentTable(
$DATA_HASH_END,
"${attachment.archiveMediaId}",
"${attachment.archiveMediaName}",
${attachment.archiveCdn}
${attachment.archiveCdn},
$ARCHIVE_THUMBNAIL_MEDIA_ID,
${if (forThumbnail) ThumbnailRestoreState.NEEDS_RESTORE.value else ThumbnailRestoreState.NONE.value}
FROM $TABLE_NAME
WHERE $ID = ${attachment.attachmentId.id}
"""
@@ -1667,7 +1669,6 @@ class AttachmentTable(
uploadTimestamp = jsonObject.getLong(UPLOAD_TIMESTAMP),
dataHash = jsonObject.getString(DATA_HASH_END),
archiveCdn = jsonObject.getInt(ARCHIVE_CDN),
archiveThumbnailCdn = jsonObject.getInt(ARCHIVE_THUMBNAIL_CDN),
archiveMediaName = jsonObject.getString(ARCHIVE_MEDIA_NAME),
archiveMediaId = jsonObject.getString(ARCHIVE_MEDIA_ID),
hasArchiveThumbnail = !TextUtils.isEmpty(jsonObject.getString(THUMBNAIL_FILE)),
@@ -1745,11 +1746,10 @@ class AttachmentTable(
fun updateArchiveCdnByMediaId(archiveMediaId: String, archiveCdn: Int): Int {
return writableDatabase.rawQuery(
"UPDATE $TABLE_NAME SET " +
"$ARCHIVE_THUMBNAIL_CDN = CASE WHEN $ARCHIVE_THUMBNAIL_MEDIA_ID = ? THEN ? ELSE $ARCHIVE_THUMBNAIL_CDN END," +
"$ARCHIVE_CDN = CASE WHEN $ARCHIVE_MEDIA_ID = ? THEN ? ELSE $ARCHIVE_CDN END " +
"WHERE $ARCHIVE_MEDIA_ID = ? OR $ARCHIVE_THUMBNAIL_MEDIA_ID = ? " +
"RETURNING $ARCHIVE_CDN, $ARCHIVE_THUMBNAIL_CDN",
SqlUtil.buildArgs(archiveMediaId, archiveCdn, archiveMediaId, archiveCdn, archiveMediaId, archiveMediaId)
"RETURNING $ARCHIVE_CDN",
SqlUtil.buildArgs(archiveMediaId, archiveCdn, archiveMediaId, archiveMediaId)
).count
}
@@ -2269,7 +2269,6 @@ class AttachmentTable(
uploadTimestamp = cursor.requireLong(UPLOAD_TIMESTAMP),
dataHash = cursor.requireString(DATA_HASH_END),
archiveCdn = cursor.requireInt(ARCHIVE_CDN),
archiveThumbnailCdn = cursor.requireInt(ARCHIVE_THUMBNAIL_CDN),
archiveMediaName = cursor.requireString(ARCHIVE_MEDIA_NAME),
archiveMediaId = cursor.requireString(ARCHIVE_MEDIA_ID),
hasArchiveThumbnail = !cursor.isNull(THUMBNAIL_FILE),
@@ -2562,11 +2561,19 @@ class AttachmentTable(
val remoteIv: ByteArray
)
class LocalRestorableAttachment(
class RestorableAttachment(
val attachmentId: AttachmentId,
val mmsId: Long,
val size: Long,
val remoteDigest: ByteArray?,
val remoteKey: ByteArray?
)
) {
override fun equals(other: Any?): Boolean {
return this === other || attachmentId == (other as? RestorableAttachment)?.attachmentId
}
override fun hashCode(): Int {
return attachmentId.hashCode()
}
}
}

View File

@@ -56,7 +56,6 @@ class MediaTable internal constructor(context: Context?, databaseHelper: SignalD
${AttachmentTable.TABLE_NAME}.${AttachmentTable.ARCHIVE_CDN},
${AttachmentTable.TABLE_NAME}.${AttachmentTable.ARCHIVE_MEDIA_NAME},
${AttachmentTable.TABLE_NAME}.${AttachmentTable.ARCHIVE_MEDIA_ID},
${AttachmentTable.TABLE_NAME}.${AttachmentTable.ARCHIVE_THUMBNAIL_CDN},
${AttachmentTable.TABLE_NAME}.${AttachmentTable.THUMBNAIL_RESTORE_STATE},
${AttachmentTable.TABLE_NAME}.${AttachmentTable.ATTACHMENT_UUID},
${MessageTable.TABLE_NAME}.${MessageTable.TYPE},

View File

@@ -388,7 +388,6 @@ open class MessageTable(context: Context?, databaseHelper: SignalDatabase) : Dat
'${AttachmentTable.UPLOAD_TIMESTAMP}', ${AttachmentTable.TABLE_NAME}.${AttachmentTable.UPLOAD_TIMESTAMP},
'${AttachmentTable.DATA_HASH_END}', ${AttachmentTable.TABLE_NAME}.${AttachmentTable.DATA_HASH_END},
'${AttachmentTable.ARCHIVE_CDN}', ${AttachmentTable.TABLE_NAME}.${AttachmentTable.ARCHIVE_CDN},
'${AttachmentTable.ARCHIVE_THUMBNAIL_CDN}', ${AttachmentTable.TABLE_NAME}.${AttachmentTable.ARCHIVE_THUMBNAIL_CDN},
'${AttachmentTable.ARCHIVE_MEDIA_NAME}', ${AttachmentTable.TABLE_NAME}.${AttachmentTable.ARCHIVE_MEDIA_NAME},
'${AttachmentTable.ARCHIVE_MEDIA_ID}', ${AttachmentTable.TABLE_NAME}.${AttachmentTable.ARCHIVE_MEDIA_ID},
'${AttachmentTable.THUMBNAIL_RESTORE_STATE}', ${AttachmentTable.TABLE_NAME}.${AttachmentTable.THUMBNAIL_RESTORE_STATE},

View File

@@ -103,6 +103,7 @@ import org.thoughtcrime.securesms.database.helpers.migration.V242_MessageFullTex
import org.thoughtcrime.securesms.database.helpers.migration.V243_MessageFullTextSearchDisableSecureDelete
import org.thoughtcrime.securesms.database.helpers.migration.V244_AttachmentRemoteIv
import org.thoughtcrime.securesms.database.helpers.migration.V245_DeletionTimestampOnCallLinks
import org.thoughtcrime.securesms.database.helpers.migration.V246_DropThumbnailCdnFromAttachments
/**
* Contains all of the database migrations for [SignalDatabase]. Broken into a separate file for cleanliness.
@@ -208,10 +209,11 @@ object SignalDatabaseMigrations {
242 to V242_MessageFullTextSearchEmojiSupportV2,
243 to V243_MessageFullTextSearchDisableSecureDelete,
244 to V244_AttachmentRemoteIv,
245 to V245_DeletionTimestampOnCallLinks
245 to V245_DeletionTimestampOnCallLinks,
246 to V246_DropThumbnailCdnFromAttachments
)
const val DATABASE_VERSION = 245
const val DATABASE_VERSION = 246
@JvmStatic
fun migrate(context: Application, db: SQLiteDatabase, oldVersion: Int, newVersion: Int) {

View File

@@ -0,0 +1,19 @@
/*
* Copyright 2023 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.thoughtcrime.securesms.database.helpers.migration
import android.app.Application
import net.zetetic.database.sqlcipher.SQLiteDatabase
/**
* Thumbnails are best effort and assumed to have the same CDN as the full attachment, there is no need to store it in the database.
*/
@Suppress("ClassName")
object V246_DropThumbnailCdnFromAttachments : SignalDatabaseMigration {
override fun migrate(context: Application, db: SQLiteDatabase, oldVersion: Int, newVersion: Int) {
db.execSQL("ALTER TABLE attachment DROP COLUMN archive_thumbnail_cdn")
}
}

View File

@@ -164,7 +164,7 @@ class JobController {
}
@WorkerThread
void submitJobs(@NonNull List<Job> jobs) {
<T extends Job> void submitJobs(@NonNull List<T> jobs) {
List<Job> canRun = new ArrayList<>(jobs.size());
synchronized (this) {

View File

@@ -198,7 +198,7 @@ public class JobManager implements ConstraintObserver.Notifier {
});
}
public void addAll(@NonNull List<Job> jobs) {
public <T extends Job> void addAll(@NonNull List<T> jobs) {
if (jobs.isEmpty()) {
return;
}

View File

@@ -13,6 +13,7 @@ import org.thoughtcrime.securesms.attachments.AttachmentId
import org.thoughtcrime.securesms.attachments.DatabaseAttachment
import org.thoughtcrime.securesms.attachments.PointerAttachment
import org.thoughtcrime.securesms.backup.v2.BackupRepository
import org.thoughtcrime.securesms.backup.v2.BackupRepository.getMediaName
import org.thoughtcrime.securesms.backup.v2.BackupRepository.getThumbnailMediaName
import org.thoughtcrime.securesms.database.SignalDatabase
import org.thoughtcrime.securesms.dependencies.AppDependencies
@@ -29,7 +30,6 @@ import org.whispersystems.signalservice.api.messages.SignalServiceAttachmentStre
import org.whispersystems.signalservice.internal.push.http.ResumableUploadSpec
import java.io.ByteArrayInputStream
import java.io.IOException
import java.lang.RuntimeException
import java.util.Optional
import kotlin.time.Duration.Companion.days
@@ -52,7 +52,7 @@ class ArchiveThumbnailUploadJob private constructor(
}
}
constructor(attachmentId: AttachmentId) : this(
private constructor(attachmentId: AttachmentId) : this(
Parameters.Builder()
.setQueue("ArchiveThumbnailUploadJob")
.addConstraint(NetworkConstraint.KEY)
@@ -82,11 +82,15 @@ class ArchiveThumbnailUploadJob private constructor(
return Result.success()
}
// TODO [backups] Decide if we fail a job when associated attachment not already backed up
// TODO [backups] Determine if we actually need to upload or are reusing a thumbnail from another attachment
val thumbnailResult = generateThumbnailIfPossible(attachment)
if (thumbnailResult == null) {
Log.w(TAG, "Unable to generate a thumbnail result for $attachmentId")
return Result.success()
}
val backupKey = SignalStore.svr.getOrCreateMasterKey().deriveBackupKey()
val resumableUpload = when (val result = BackupRepository.getMediaUploadSpec(secretKey = backupKey.deriveThumbnailTransitKey(attachment.getThumbnailMediaName()))) {
@@ -123,6 +127,10 @@ class ArchiveThumbnailUploadJob private constructor(
return when (val result = BackupRepository.archiveThumbnail(attachmentPointer, attachment)) {
is NetworkResult.Success -> {
// save attachment thumbnail
val archiveMediaId = attachment.archiveMediaId ?: backupKey.deriveMediaId(attachment.getMediaName()).encode()
SignalDatabase.attachments.finalizeAttachmentThumbnailAfterUpload(attachmentId, archiveMediaId, mediaSecrets.id, thumbnailResult.data)
Log.i(RestoreAttachmentJob.TAG, "Restore: Thumbnail mediaId=${mediaSecrets.id.encode()} backupDir=${backupDirectories.backupDir} mediaDir=${backupDirectories.mediaDir}")
Log.d(TAG, "Successfully archived thumbnail for $attachmentId mediaName=${attachment.getThumbnailMediaName()}")
Result.success()

View File

@@ -4,8 +4,6 @@
*/
package org.thoughtcrime.securesms.jobs
import android.text.TextUtils
import androidx.annotation.VisibleForTesting
import okio.Source
import okio.buffer
import org.greenrobot.eventbus.EventBus
@@ -19,7 +17,7 @@ import org.thoughtcrime.securesms.attachments.Attachment
import org.thoughtcrime.securesms.attachments.AttachmentId
import org.thoughtcrime.securesms.attachments.Cdn
import org.thoughtcrime.securesms.attachments.DatabaseAttachment
import org.thoughtcrime.securesms.backup.v2.BackupRepository
import org.thoughtcrime.securesms.attachments.InvalidAttachmentException
import org.thoughtcrime.securesms.database.AttachmentTable
import org.thoughtcrime.securesms.database.SignalDatabase
import org.thoughtcrime.securesms.dependencies.AppDependencies
@@ -37,7 +35,6 @@ import org.thoughtcrime.securesms.transport.RetryLaterException
import org.thoughtcrime.securesms.util.AttachmentUtil
import org.thoughtcrime.securesms.util.RemoteConfig
import org.thoughtcrime.securesms.util.Util
import org.whispersystems.signalservice.api.backup.MediaName
import org.whispersystems.signalservice.api.messages.SignalServiceAttachment
import org.whispersystems.signalservice.api.messages.SignalServiceAttachmentPointer
import org.whispersystems.signalservice.api.messages.SignalServiceAttachmentRemoteId
@@ -56,9 +53,8 @@ import java.util.concurrent.TimeUnit
class AttachmentDownloadJob private constructor(
parameters: Parameters,
private val messageId: Long,
attachmentId: AttachmentId,
private val manual: Boolean,
private var forceArchiveDownload: Boolean
private val attachmentId: AttachmentId,
private val manual: Boolean
) : BaseJob(parameters) {
companion object {
@@ -68,7 +64,6 @@ class AttachmentDownloadJob private constructor(
private const val KEY_MESSAGE_ID = "message_id"
private const val KEY_ATTACHMENT_ID = "part_row_id"
private const val KEY_MANUAL = "part_manual"
private const val KEY_FORCE_ARCHIVE = "force_archive"
@JvmStatic
fun constructQueueString(attachmentId: AttachmentId): String {
@@ -89,25 +84,37 @@ class AttachmentDownloadJob private constructor(
@JvmStatic
fun downloadAttachmentIfNeeded(databaseAttachment: DatabaseAttachment): String? {
return when (val transferState = databaseAttachment.transferState) {
AttachmentTable.TRANSFER_RESTORE_OFFLOADED -> RestoreAttachmentJob.restoreOffloadedAttachment(databaseAttachment)
AttachmentTable.TRANSFER_RESTORE_OFFLOADED,
AttachmentTable.TRANSFER_NEEDS_RESTORE -> RestoreAttachmentJob.restoreAttachment(databaseAttachment)
AttachmentTable.TRANSFER_PROGRESS_PENDING,
AttachmentTable.TRANSFER_PROGRESS_FAILED,
AttachmentTable.TRANSFER_NEEDS_RESTORE -> {
val downloadJob = AttachmentDownloadJob(
messageId = databaseAttachment.mmsId,
attachmentId = databaseAttachment.attachmentId,
manual = true,
forceArchiveDownload = false
)
AppDependencies.jobManager.add(downloadJob)
downloadJob.id
AttachmentTable.TRANSFER_PROGRESS_FAILED -> {
if (SignalStore.backup.backsUpMedia && databaseAttachment.remoteLocation == null) {
if (databaseAttachment.archiveMediaName.isNullOrEmpty()) {
Log.w(TAG, "No remote location or archive media name, can't download")
null
} else {
Log.i(TAG, "Trying to restore attachment from archive cdn")
RestoreAttachmentJob.restoreAttachment(databaseAttachment)
}
} else {
val downloadJob = AttachmentDownloadJob(
messageId = databaseAttachment.mmsId,
attachmentId = databaseAttachment.attachmentId,
manual = true
)
AppDependencies.jobManager.add(downloadJob)
downloadJob.id
}
}
AttachmentTable.TRANSFER_RESTORE_IN_PROGRESS,
AttachmentTable.TRANSFER_PROGRESS_DONE,
AttachmentTable.TRANSFER_PROGRESS_STARTED,
AttachmentTable.TRANSFER_PROGRESS_PERMANENT_FAILURE -> null
AttachmentTable.TRANSFER_PROGRESS_PERMANENT_FAILURE -> {
Log.d(TAG, "$databaseAttachment is downloading, downloaded already or permanently failed, transferState: $transferState")
null
}
else -> {
Log.w(TAG, "Attempted to download attachment with unknown transfer state: $transferState")
@@ -117,9 +124,7 @@ class AttachmentDownloadJob private constructor(
}
}
private val attachmentId: Long
constructor(messageId: Long, attachmentId: AttachmentId, manual: Boolean, forceArchiveDownload: Boolean = false) : this(
constructor(messageId: Long, attachmentId: AttachmentId, manual: Boolean) : this(
Parameters.Builder()
.setQueue(constructQueueString(attachmentId))
.addConstraint(NetworkConstraint.KEY)
@@ -128,20 +133,14 @@ class AttachmentDownloadJob private constructor(
.build(),
messageId,
attachmentId,
manual,
forceArchiveDownload
manual
)
init {
this.attachmentId = attachmentId.id
}
override fun serialize(): ByteArray? {
return JsonJobData.Builder()
.putLong(KEY_MESSAGE_ID, messageId)
.putLong(KEY_ATTACHMENT_ID, attachmentId)
.putLong(KEY_ATTACHMENT_ID, attachmentId.id)
.putBoolean(KEY_MANUAL, manual)
.putBoolean(KEY_FORCE_ARCHIVE, forceArchiveDownload)
.serialize()
}
@@ -152,7 +151,6 @@ class AttachmentDownloadJob private constructor(
override fun onAdded() {
Log.i(TAG, "onAdded() messageId: $messageId attachmentId: $attachmentId manual: $manual")
val attachmentId = AttachmentId(attachmentId)
val attachment = SignalDatabase.attachments.getAttachment(attachmentId)
val pending = attachment != null && attachment.transferState != AttachmentTable.TRANSFER_PROGRESS_DONE && attachment.transferState != AttachmentTable.TRANSFER_PROGRESS_PERMANENT_FAILURE
@@ -175,7 +173,6 @@ class AttachmentDownloadJob private constructor(
fun doWork() {
Log.i(TAG, "onRun() messageId: $messageId attachmentId: $attachmentId manual: $manual")
val attachmentId = AttachmentId(attachmentId)
val attachment = SignalDatabase.attachments.getAttachment(attachmentId)
if (attachment == null) {
@@ -199,6 +196,17 @@ class AttachmentDownloadJob private constructor(
return
}
if (SignalStore.backup.backsUpMedia && attachment.remoteLocation == null) {
if (attachment.archiveMediaName.isNullOrEmpty()) {
throw InvalidAttachmentException("No remote location or archive media name")
}
Log.i(TAG, "Trying to restore attachment from archive cdn instead")
RestoreAttachmentJob.restoreAttachment(attachment)
return
}
Log.i(TAG, "Downloading push part $attachmentId")
SignalDatabase.attachments.setTransferState(messageId, attachmentId, AttachmentTable.TRANSFER_PROGRESS_STARTED)
@@ -218,7 +226,6 @@ class AttachmentDownloadJob private constructor(
override fun onFailure() {
Log.w(TAG, format(this, "onFailure() messageId: $messageId attachmentId: $attachmentId manual: $manual"))
val attachmentId = AttachmentId(attachmentId)
markFailed(messageId, attachmentId)
}
@@ -235,25 +242,13 @@ class AttachmentDownloadJob private constructor(
) {
val maxReceiveSize: Long = RemoteConfig.maxAttachmentReceiveSizeBytes
val attachmentFile: File = SignalDatabase.attachments.getOrCreateTransferFile(attachmentId)
var archiveFile: File? = null
var useArchiveCdn = false
try {
if (attachment.size > maxReceiveSize) {
throw MmsException("Attachment too large, failing download")
}
useArchiveCdn = if (SignalStore.backup.backsUpMedia && (forceArchiveDownload || attachment.remoteLocation == null)) {
if (attachment.archiveMediaName.isNullOrEmpty()) {
throw InvalidPartException("Invalid attachment configuration")
}
true
} else {
false
}
val messageReceiver = AppDependencies.signalServiceMessageReceiver
val pointer = createAttachmentPointer(attachment, useArchiveCdn)
val pointer = createAttachmentPointer(attachment)
val progressListener = object : SignalServiceAttachment.ProgressListener {
override fun onAttachmentProgress(total: Long, progress: Long) {
@@ -265,55 +260,32 @@ class AttachmentDownloadJob private constructor(
}
}
val downloadResult = if (useArchiveCdn) {
archiveFile = SignalDatabase.attachments.getOrCreateArchiveTransferFile(attachmentId)
val cdnCredentials = BackupRepository.getCdnReadCredentials(attachment.archiveCdn).successOrThrow().headers
messageReceiver
.retrieveArchivedAttachment(
SignalStore.svr.getOrCreateMasterKey().deriveBackupKey().deriveMediaSecrets(MediaName(attachment.archiveMediaName!!)),
cdnCredentials,
archiveFile,
pointer,
attachmentFile,
maxReceiveSize,
false,
progressListener
)
} else {
messageReceiver
.retrieveAttachment(
pointer,
attachmentFile,
maxReceiveSize,
progressListener
)
}
val downloadResult = AppDependencies
.signalServiceMessageReceiver
.retrieveAttachment(
pointer,
attachmentFile,
maxReceiveSize,
progressListener
)
SignalDatabase.attachments.finalizeAttachmentAfterDownload(messageId, attachmentId, downloadResult.dataStream, downloadResult.iv)
} catch (e: RangeException) {
val transferFile = archiveFile ?: attachmentFile
Log.w(TAG, "Range exception, file size " + transferFile.length(), e)
if (transferFile.delete()) {
Log.w(TAG, "Range exception, file size " + attachmentFile.length(), e)
if (attachmentFile.delete()) {
Log.i(TAG, "Deleted temp download file to recover")
throw RetryLaterException(e)
} else {
throw IOException("Failed to delete temp download file following range exception")
}
} catch (e: InvalidPartException) {
} catch (e: InvalidAttachmentException) {
Log.w(TAG, "Experienced exception while trying to download an attachment.", e)
markFailed(messageId, attachmentId)
} catch (e: NonSuccessfulResponseCodeException) {
if (SignalStore.backup.backsUpMedia) {
if (e.code == 404 && !useArchiveCdn && attachment.archiveMediaName?.isNotEmpty() == true) {
Log.i(TAG, "Retrying download from archive CDN")
forceArchiveDownload = true
retrieveAttachment(messageId, attachmentId, attachment)
return
} else if (e.code == 401 && useArchiveCdn) {
SignalStore.backup.cdnReadCredentials = null
throw RetryLaterException(e)
}
if (SignalStore.backup.backsUpMedia && e.code == 404 && attachment.archiveMediaName?.isNotEmpty() == true) {
Log.i(TAG, "Retrying download from archive CDN")
RestoreAttachmentJob.restoreAttachment(attachment)
return
}
Log.w(TAG, "Experienced exception while trying to download an attachment.", e)
@@ -335,47 +307,31 @@ class AttachmentDownloadJob private constructor(
}
}
@Throws(InvalidPartException::class)
private fun createAttachmentPointer(attachment: DatabaseAttachment, useArchiveCdn: Boolean): SignalServiceAttachmentPointer {
if (TextUtils.isEmpty(attachment.remoteKey)) {
throw InvalidPartException("empty encrypted key")
@Throws(InvalidAttachmentException::class)
private fun createAttachmentPointer(attachment: DatabaseAttachment): SignalServiceAttachmentPointer {
if (attachment.remoteKey.isNullOrEmpty()) {
throw InvalidAttachmentException("empty encrypted key")
}
if (attachment.remoteLocation.isNullOrEmpty()) {
throw InvalidAttachmentException("empty content id")
}
return try {
val remoteData: RemoteData = if (useArchiveCdn) {
val backupKey = SignalStore.svr.getOrCreateMasterKey().deriveBackupKey()
val backupDirectories = BackupRepository.getCdnBackupDirectories().successOrThrow()
val remoteId = SignalServiceAttachmentRemoteId.from(attachment.remoteLocation)
val cdnNumber = attachment.cdn.cdnNumber
RemoteData(
remoteId = SignalServiceAttachmentRemoteId.Backup(
backupDir = backupDirectories.backupDir,
mediaDir = backupDirectories.mediaDir,
mediaId = backupKey.deriveMediaId(MediaName(attachment.archiveMediaName!!)).encode()
),
cdnNumber = attachment.archiveCdn
)
} else {
if (attachment.remoteLocation.isNullOrEmpty()) {
throw InvalidPartException("empty content id")
}
RemoteData(
remoteId = SignalServiceAttachmentRemoteId.from(attachment.remoteLocation),
cdnNumber = attachment.cdn.cdnNumber
)
}
val key = Base64.decode(attachment.remoteKey!!)
val key = Base64.decode(attachment.remoteKey)
if (attachment.remoteDigest != null) {
Log.i(TAG, "Downloading attachment with digest: " + Hex.toString(attachment.remoteDigest))
} else {
throw InvalidPartException("Null remote digest for $attachmentId")
throw InvalidAttachmentException("Null remote digest for $attachmentId")
}
SignalServiceAttachmentPointer(
remoteData.cdnNumber,
remoteData.remoteId,
cdnNumber,
remoteId,
null,
key,
Optional.of(Util.toIntExact(attachment.size)),
@@ -396,10 +352,10 @@ class AttachmentDownloadJob private constructor(
)
} catch (e: IOException) {
Log.w(TAG, e)
throw InvalidPartException(e)
throw InvalidAttachmentException(e)
} catch (e: ArithmeticException) {
Log.w(TAG, e)
throw InvalidPartException(e)
throw InvalidAttachmentException(e)
}
}
@@ -442,14 +398,6 @@ class AttachmentDownloadJob private constructor(
SignalDatabase.attachments.setTransferProgressPermanentFailure(attachmentId, messageId)
}
@VisibleForTesting
internal class InvalidPartException : Exception {
constructor(s: String?) : super(s)
constructor(e: Exception?) : super(e)
}
private data class RemoteData(val remoteId: SignalServiceAttachmentRemoteId, val cdnNumber: Int)
class Factory : Job.Factory<AttachmentDownloadJob?> {
override fun create(parameters: Parameters, serializedData: ByteArray?): AttachmentDownloadJob {
val data = JsonJobData.deserialize(serializedData)
@@ -457,8 +405,7 @@ class AttachmentDownloadJob private constructor(
parameters = parameters,
messageId = data.getLong(KEY_MESSAGE_ID),
attachmentId = AttachmentId(data.getLong(KEY_ATTACHMENT_ID)),
manual = data.getBoolean(KEY_MANUAL),
forceArchiveDownload = data.getBooleanOrDefault(KEY_FORCE_ARCHIVE, false)
manual = data.getBoolean(KEY_MANUAL)
)
}
}

View File

@@ -6,7 +6,9 @@
package org.thoughtcrime.securesms.jobs
import org.signal.core.util.logging.Log
import org.signal.core.util.withinTransaction
import org.thoughtcrime.securesms.database.AttachmentTable
import org.thoughtcrime.securesms.database.AttachmentTable.RestorableAttachment
import org.thoughtcrime.securesms.database.SignalDatabase
import org.thoughtcrime.securesms.database.model.MmsMessageRecord
import org.thoughtcrime.securesms.dependencies.AppDependencies
@@ -48,40 +50,64 @@ class BackupRestoreMediaJob private constructor(parameters: Parameters) : BaseJo
throw NotPushRegisteredException()
}
SignalStore.backup.totalRestorableAttachmentSize = SignalDatabase.attachments.getRemainingRestorableAttachmentSize()
val jobManager = AppDependencies.jobManager
val batchSize = 100
val batchSize = 500
val restoreTime = System.currentTimeMillis()
var restoreJobBatch: List<Job>
do {
val restoreThumbnailJobs: MutableList<RestoreAttachmentThumbnailJob> = mutableListOf()
val restoreFullAttachmentJobs: MutableMap<RestorableAttachment, RestoreAttachmentJob> = mutableMapOf()
val restoreThumbnailOnlyAttachments: MutableList<RestorableAttachment> = mutableListOf()
val notRestorable: MutableList<RestorableAttachment> = mutableListOf()
val attachmentBatch = SignalDatabase.attachments.getRestorableAttachments(batchSize)
val messageIds = attachmentBatch.map { it.mmsId }.toSet()
val messageMap = SignalDatabase.messages.getMessages(messageIds).associate { it.id to (it as MmsMessageRecord) }
restoreJobBatch = SignalDatabase.attachments.getRestorableAttachments(batchSize).map { attachment ->
val message = messageMap[attachment.mmsId]!!
for (attachment in attachmentBatch) {
val message = messageMap[attachment.mmsId]
if (message == null) {
Log.w(TAG, "Unable to find message for ${attachment.attachmentId}")
notRestorable += attachment
continue
}
restoreThumbnailJobs += RestoreAttachmentThumbnailJob(
messageId = attachment.mmsId,
attachmentId = attachment.attachmentId,
highPriority = false
)
if (shouldRestoreFullSize(message, restoreTime, SignalStore.backup.optimizeStorage)) {
RestoreAttachmentJob(
restoreFullAttachmentJobs += attachment to RestoreAttachmentJob(
messageId = attachment.mmsId,
attachmentId = attachment.attachmentId,
manual = false,
forceArchiveDownload = true,
restoreMode = RestoreAttachmentJob.RestoreMode.ORIGINAL
attachmentId = attachment.attachmentId
)
} else {
SignalDatabase.attachments.setTransferState(
messageId = attachment.mmsId,
attachmentId = attachment.attachmentId,
transferState = AttachmentTable.TRANSFER_RESTORE_OFFLOADED
)
RestoreAttachmentThumbnailJob(
messageId = attachment.mmsId,
attachmentId = attachment.attachmentId,
highPriority = false
)
restoreThumbnailOnlyAttachments += attachment
}
}
jobManager.addAll(restoreJobBatch)
} while (restoreJobBatch.isNotEmpty())
SignalDatabase.rawDatabase.withinTransaction {
// Mark not restorable thumbnails and attachments as failed
SignalDatabase.attachments.setThumbnailRestoreState(notRestorable.map { it.attachmentId }, AttachmentTable.ThumbnailRestoreState.PERMANENT_FAILURE)
SignalDatabase.attachments.setRestoreTransferState(notRestorable, AttachmentTable.TRANSFER_PROGRESS_FAILED)
// Mark restorable thumbnails and attachments as in progress
SignalDatabase.attachments.setThumbnailRestoreState(restoreThumbnailJobs.map { it.attachmentId }, AttachmentTable.ThumbnailRestoreState.IN_PROGRESS)
SignalDatabase.attachments.setRestoreTransferState(restoreFullAttachmentJobs.keys, AttachmentTable.TRANSFER_RESTORE_IN_PROGRESS)
// Set thumbnail only attachments as offloaded
SignalDatabase.attachments.setRestoreTransferState(restoreThumbnailOnlyAttachments, AttachmentTable.TRANSFER_RESTORE_OFFLOADED)
jobManager.addAll(restoreThumbnailJobs + restoreFullAttachmentJobs.values)
}
} while (restoreThumbnailJobs.isNotEmpty() && restoreFullAttachmentJobs.isNotEmpty() && notRestorable.isNotEmpty())
SignalStore.backup.totalRestorableAttachmentSize = SignalDatabase.attachments.getRemainingRestorableAttachmentSize()
jobManager.add(CheckRestoreMediaLeftJob(RestoreAttachmentJob.constructQueueString()))
}
private fun shouldRestoreFullSize(message: MmsMessageRecord, restoreTime: Long, optimizeStorage: Boolean): Boolean {

View File

@@ -4,44 +4,36 @@
*/
package org.thoughtcrime.securesms.jobs
import android.text.TextUtils
import androidx.annotation.VisibleForTesting
import org.greenrobot.eventbus.EventBus
import org.signal.core.util.Base64
import org.signal.core.util.Hex
import org.signal.core.util.logging.Log
import org.signal.libsignal.protocol.InvalidMacException
import org.signal.libsignal.protocol.InvalidMessageException
import org.thoughtcrime.securesms.attachments.AttachmentId
import org.thoughtcrime.securesms.attachments.DatabaseAttachment
import org.thoughtcrime.securesms.attachments.InvalidAttachmentException
import org.thoughtcrime.securesms.backup.v2.BackupRepository
import org.thoughtcrime.securesms.backup.v2.BackupRepository.getThumbnailMediaName
import org.thoughtcrime.securesms.backup.v2.database.createArchiveAttachmentPointer
import org.thoughtcrime.securesms.database.AttachmentTable
import org.thoughtcrime.securesms.database.SignalDatabase
import org.thoughtcrime.securesms.dependencies.AppDependencies
import org.thoughtcrime.securesms.events.PartProgressEvent
import org.thoughtcrime.securesms.jobmanager.Job
import org.thoughtcrime.securesms.jobmanager.JobLogger.format
import org.thoughtcrime.securesms.jobmanager.JsonJobData
import org.thoughtcrime.securesms.jobmanager.impl.NetworkConstraint
import org.thoughtcrime.securesms.jobmanager.persistence.JobSpec
import org.thoughtcrime.securesms.jobs.protos.RestoreAttachmentJobData
import org.thoughtcrime.securesms.keyvalue.SignalStore
import org.thoughtcrime.securesms.mms.MmsException
import org.thoughtcrime.securesms.notifications.v2.ConversationId.Companion.forConversation
import org.thoughtcrime.securesms.transport.RetryLaterException
import org.thoughtcrime.securesms.util.RemoteConfig
import org.thoughtcrime.securesms.util.Util
import org.whispersystems.signalservice.api.backup.MediaName
import org.whispersystems.signalservice.api.messages.SignalServiceAttachment
import org.whispersystems.signalservice.api.messages.SignalServiceAttachmentPointer
import org.whispersystems.signalservice.api.messages.SignalServiceAttachmentRemoteId
import org.whispersystems.signalservice.api.push.exceptions.MissingConfigurationException
import org.whispersystems.signalservice.api.push.exceptions.NonSuccessfulResponseCodeException
import org.whispersystems.signalservice.api.push.exceptions.PushNetworkException
import org.whispersystems.signalservice.api.push.exceptions.RangeException
import java.io.File
import java.io.IOException
import java.util.Optional
import java.util.concurrent.TimeUnit
/**
@@ -50,83 +42,46 @@ import java.util.concurrent.TimeUnit
class RestoreAttachmentJob private constructor(
parameters: Parameters,
private val messageId: Long,
attachmentId: AttachmentId,
private val manual: Boolean,
private var forceArchiveDownload: Boolean,
private val restoreMode: RestoreMode
private val attachmentId: AttachmentId,
private val offloaded: Boolean
) : BaseJob(parameters) {
companion object {
const val KEY = "RestoreAttachmentJob"
val TAG = Log.tag(RestoreAttachmentJob::class.java)
private const val KEY_MESSAGE_ID = "message_id"
private const val KEY_ATTACHMENT_ID = "part_row_id"
private const val KEY_MANUAL = "part_manual"
private const val KEY_FORCE_ARCHIVE = "force_archive"
private const val KEY_RESTORE_MODE = "restore_mode"
@JvmStatic
fun constructQueueString(attachmentId: AttachmentId): String {
fun constructQueueString(): String {
// TODO: decide how many queues
return "RestoreAttachmentJob"
}
private fun getJsonJobData(jobSpec: JobSpec): JsonJobData? {
if (KEY != jobSpec.factoryKey) {
return null
}
val serializedData = jobSpec.serializedData ?: return null
return JsonJobData.deserialize(serializedData)
}
fun jobSpecMatchesAnyAttachmentId(data: JsonJobData?, ids: Set<AttachmentId>): Boolean {
if (data == null) {
return false
}
val parsed = AttachmentId(data.getLong(KEY_ATTACHMENT_ID))
return ids.contains(parsed)
}
@JvmStatic
fun restoreOffloadedAttachment(attachment: DatabaseAttachment): String {
fun restoreAttachment(attachment: DatabaseAttachment): String {
val restoreJob = RestoreAttachmentJob(
messageId = attachment.mmsId,
attachmentId = attachment.attachmentId,
manual = false,
forceArchiveDownload = true,
restoreMode = RestoreAttachmentJob.RestoreMode.ORIGINAL
offloaded = true
)
AppDependencies.jobManager.add(restoreJob)
return restoreJob.id
}
}
private val attachmentId: Long = attachmentId.id
constructor(messageId: Long, attachmentId: AttachmentId, manual: Boolean, forceArchiveDownload: Boolean = false, restoreMode: RestoreMode = RestoreMode.ORIGINAL) : this(
constructor(messageId: Long, attachmentId: AttachmentId, offloaded: Boolean = false) : this(
Parameters.Builder()
.setQueue(constructQueueString(attachmentId))
.setQueue(constructQueueString())
.addConstraint(NetworkConstraint.KEY)
.setLifespan(TimeUnit.DAYS.toMillis(1))
.setMaxAttempts(Parameters.UNLIMITED)
.build(),
messageId,
attachmentId,
manual,
forceArchiveDownload,
restoreMode
offloaded
)
override fun serialize(): ByteArray? {
return JsonJobData.Builder()
.putLong(KEY_MESSAGE_ID, messageId)
.putLong(KEY_ATTACHMENT_ID, attachmentId)
.putBoolean(KEY_MANUAL, manual)
.putBoolean(KEY_FORCE_ARCHIVE, forceArchiveDownload)
.putInt(KEY_RESTORE_MODE, restoreMode.value)
.serialize()
return RestoreAttachmentJobData(messageId = messageId, attachmentId = attachmentId.id, offloaded = offloaded).encode()
}
override fun getFactoryKey(): String {
@@ -134,14 +89,14 @@ class RestoreAttachmentJob private constructor(
}
override fun onAdded() {
Log.i(TAG, "onAdded() messageId: $messageId attachmentId: $attachmentId manual: $manual")
if (offloaded) {
Log.i(TAG, "onAdded() messageId: $messageId attachmentId: $attachmentId")
val attachmentId = AttachmentId(attachmentId)
val attachment = SignalDatabase.attachments.getAttachment(attachmentId)
val pending = attachment != null && attachment.transferState != AttachmentTable.TRANSFER_PROGRESS_DONE && attachment.transferState != AttachmentTable.TRANSFER_PROGRESS_PERMANENT_FAILURE
if (attachment?.transferState == AttachmentTable.TRANSFER_NEEDS_RESTORE || attachment?.transferState == AttachmentTable.TRANSFER_RESTORE_OFFLOADED) {
Log.i(TAG, "onAdded() Marking attachment restore progress as 'started'")
SignalDatabase.attachments.setTransferState(messageId, attachmentId, AttachmentTable.TRANSFER_RESTORE_IN_PROGRESS)
val attachment = SignalDatabase.attachments.getAttachment(attachmentId)
if (attachment?.transferState == AttachmentTable.TRANSFER_NEEDS_RESTORE || attachment?.transferState == AttachmentTable.TRANSFER_RESTORE_OFFLOADED) {
Log.i(TAG, "onAdded() Marking attachment restore progress as 'started'")
SignalDatabase.attachments.setTransferState(messageId, attachmentId, AttachmentTable.TRANSFER_RESTORE_IN_PROGRESS)
}
}
}
@@ -152,17 +107,12 @@ class RestoreAttachmentJob private constructor(
if (!SignalDatabase.messages.isStory(messageId)) {
AppDependencies.messageNotifier.updateNotification(context, forConversation(0))
}
if (SignalDatabase.attachments.getRemainingRestorableAttachmentSize() == 0L) {
SignalStore.backup.totalRestorableAttachmentSize = 0L
}
}
@Throws(IOException::class, RetryLaterException::class)
fun doWork() {
Log.i(TAG, "onRun() messageId: $messageId attachmentId: $attachmentId manual: $manual")
Log.i(TAG, "onRun() messageId: $messageId attachmentId: $attachmentId")
val attachmentId = AttachmentId(attachmentId)
val attachment = SignalDatabase.attachments.getAttachment(attachmentId)
if (attachment == null) {
@@ -177,23 +127,18 @@ class RestoreAttachmentJob private constructor(
if (attachment.transferState != AttachmentTable.TRANSFER_NEEDS_RESTORE &&
attachment.transferState != AttachmentTable.TRANSFER_RESTORE_IN_PROGRESS &&
(attachment.transferState != AttachmentTable.TRANSFER_RESTORE_OFFLOADED || restoreMode == RestoreMode.THUMBNAIL)
(attachment.transferState != AttachmentTable.TRANSFER_RESTORE_OFFLOADED)
) {
Log.w(TAG, "Attachment does not need to be restored.")
return
}
if (attachment.thumbnailUri == null && (restoreMode == RestoreMode.THUMBNAIL || restoreMode == RestoreMode.BOTH)) {
downloadThumbnail(attachmentId, attachment)
}
if (restoreMode == RestoreMode.ORIGINAL || restoreMode == RestoreMode.BOTH) {
retrieveAttachment(messageId, attachmentId, attachment)
}
retrieveAttachment(messageId, attachmentId, attachment)
}
override fun onFailure() {
Log.w(TAG, format(this, "onFailure() messageId: $messageId attachmentId: $attachmentId manual: $manual"))
Log.w(TAG, format(this, "onFailure() messageId: $messageId attachmentId: $attachmentId"))
val attachmentId = AttachmentId(attachmentId)
markFailed(messageId, attachmentId)
}
@@ -206,7 +151,8 @@ class RestoreAttachmentJob private constructor(
private fun retrieveAttachment(
messageId: Long,
attachmentId: AttachmentId,
attachment: DatabaseAttachment
attachment: DatabaseAttachment,
forceTransitTier: Boolean = false
) {
val maxReceiveSize: Long = RemoteConfig.maxAttachmentReceiveSizeBytes
val attachmentFile: File = SignalDatabase.attachments.getOrCreateTransferFile(attachmentId)
@@ -218,9 +164,9 @@ class RestoreAttachmentJob private constructor(
throw MmsException("Attachment too large, failing download")
}
useArchiveCdn = if (SignalStore.backup.backsUpMedia && (forceArchiveDownload || attachment.remoteLocation == null)) {
useArchiveCdn = if (SignalStore.backup.backsUpMedia && !forceTransitTier) {
if (attachment.archiveMediaName.isNullOrEmpty()) {
throw InvalidPartException("Invalid attachment configuration")
throw InvalidAttachmentException("Invalid attachment configuration")
}
true
} else {
@@ -228,7 +174,7 @@ class RestoreAttachmentJob private constructor(
}
val messageReceiver = AppDependencies.signalServiceMessageReceiver
val pointer = createAttachmentPointer(attachment, useArchiveCdn)
val pointer = attachment.createArchiveAttachmentPointer(useArchiveCdn)
val progressListener = object : SignalServiceAttachment.ProgressListener {
override fun onAttachmentProgress(total: Long, progress: Long) {
@@ -275,15 +221,14 @@ class RestoreAttachmentJob private constructor(
} else {
throw IOException("Failed to delete temp download file following range exception")
}
} catch (e: InvalidPartException) {
} catch (e: InvalidAttachmentException) {
Log.w(TAG, "Experienced exception while trying to download an attachment.", e)
markFailed(messageId, attachmentId)
} catch (e: NonSuccessfulResponseCodeException) {
if (SignalStore.backup.backsUpMedia) {
if (e.code == 404 && !useArchiveCdn && attachment.archiveMediaName?.isNotEmpty() == true) {
Log.i(TAG, "Retrying download from archive CDN")
forceArchiveDownload = true
retrieveAttachment(messageId, attachmentId, attachment)
if (e.code == 404 && !forceTransitTier && attachment.remoteLocation?.isNotBlank() == true) {
Log.i(TAG, "Retrying download from transit CDN")
retrieveAttachment(messageId, attachmentId, attachment, true)
return
} else if (e.code == 401 && useArchiveCdn) {
SignalStore.backup.cdnReadCredentials = null
@@ -310,212 +255,22 @@ class RestoreAttachmentJob private constructor(
}
}
@Throws(InvalidPartException::class)
private fun createAttachmentPointer(attachment: DatabaseAttachment, useArchiveCdn: Boolean): SignalServiceAttachmentPointer {
if (TextUtils.isEmpty(attachment.remoteKey)) {
throw InvalidPartException("empty encrypted key")
}
return try {
val remoteData: RemoteData = if (useArchiveCdn) {
val backupKey = SignalStore.svr.getOrCreateMasterKey().deriveBackupKey()
val backupDirectories = BackupRepository.getCdnBackupDirectories().successOrThrow()
RemoteData(
remoteId = SignalServiceAttachmentRemoteId.Backup(
backupDir = backupDirectories.backupDir,
mediaDir = backupDirectories.mediaDir,
mediaId = backupKey.deriveMediaId(MediaName(attachment.archiveMediaName!!)).encode()
),
cdnNumber = attachment.archiveCdn
)
} else {
if (attachment.remoteLocation.isNullOrEmpty()) {
throw InvalidPartException("empty content id")
}
RemoteData(
remoteId = SignalServiceAttachmentRemoteId.from(attachment.remoteLocation),
cdnNumber = attachment.cdn.cdnNumber
)
}
val key = Base64.decode(attachment.remoteKey!!)
if (attachment.remoteDigest != null) {
Log.i(TAG, "Downloading attachment with digest: " + Hex.toString(attachment.remoteDigest))
} else {
Log.i(TAG, "Downloading attachment with no digest...")
}
SignalServiceAttachmentPointer(
remoteData.cdnNumber,
remoteData.remoteId,
null,
key,
Optional.of(Util.toIntExact(attachment.size)),
Optional.empty(),
0,
0,
Optional.ofNullable(attachment.remoteDigest),
Optional.ofNullable(attachment.getIncrementalDigest()),
attachment.incrementalMacChunkSize,
Optional.ofNullable(attachment.fileName),
attachment.voiceNote,
attachment.borderless,
attachment.videoGif,
Optional.empty(),
Optional.ofNullable(attachment.blurHash).map { it.hash },
attachment.uploadTimestamp,
attachment.uuid
)
} catch (e: IOException) {
Log.w(TAG, e)
throw InvalidPartException(e)
} catch (e: ArithmeticException) {
Log.w(TAG, e)
throw InvalidPartException(e)
}
}
@Throws(InvalidPartException::class)
private fun createThumbnailPointer(attachment: DatabaseAttachment): SignalServiceAttachmentPointer {
if (TextUtils.isEmpty(attachment.remoteKey)) {
throw InvalidPartException("empty encrypted key")
}
val backupKey = SignalStore.svr.getOrCreateMasterKey().deriveBackupKey()
val backupDirectories = BackupRepository.getCdnBackupDirectories().successOrThrow()
return try {
val key = backupKey.deriveThumbnailTransitKey(attachment.getThumbnailMediaName())
val mediaId = backupKey.deriveMediaId(attachment.getThumbnailMediaName()).encode()
SignalServiceAttachmentPointer(
attachment.archiveThumbnailCdn,
SignalServiceAttachmentRemoteId.Backup(
backupDir = backupDirectories.backupDir,
mediaDir = backupDirectories.mediaDir,
mediaId = mediaId
),
null,
key,
Optional.empty(),
Optional.empty(),
0,
0,
Optional.empty(),
Optional.empty(),
attachment.incrementalMacChunkSize,
Optional.empty(),
attachment.voiceNote,
attachment.borderless,
attachment.videoGif,
Optional.empty(),
Optional.ofNullable(attachment.blurHash).map { it.hash },
attachment.uploadTimestamp,
attachment.uuid
)
} catch (e: IOException) {
Log.w(TAG, e)
throw InvalidPartException(e)
} catch (e: ArithmeticException) {
Log.w(TAG, e)
throw InvalidPartException(e)
}
}
private fun downloadThumbnail(attachmentId: AttachmentId, attachment: DatabaseAttachment) {
if (attachment.thumbnailRestoreState == AttachmentTable.ThumbnailRestoreState.FINISHED) {
Log.w(TAG, "$attachmentId already has thumbnail downloaded")
return
}
if (attachment.thumbnailRestoreState == AttachmentTable.ThumbnailRestoreState.NONE) {
Log.w(TAG, "$attachmentId has no thumbnail state")
return
}
if (attachment.thumbnailRestoreState == AttachmentTable.ThumbnailRestoreState.PERMANENT_FAILURE) {
Log.w(TAG, "$attachmentId thumbnail permanently failed")
return
}
if (attachment.archiveMediaName == null) {
Log.w(TAG, "$attachmentId was never archived! Cannot proceed.")
return
}
val maxThumbnailSize: Long = RemoteConfig.maxAttachmentReceiveSizeBytes
val thumbnailTransferFile: File = SignalDatabase.attachments.createArchiveThumbnailTransferFile()
val thumbnailFile: File = SignalDatabase.attachments.createArchiveThumbnailTransferFile()
val progressListener = object : SignalServiceAttachment.ProgressListener {
override fun onAttachmentProgress(total: Long, progress: Long) {
}
override fun shouldCancel(): Boolean {
return this@RestoreAttachmentJob.isCanceled
}
}
val cdnCredentials = BackupRepository.getCdnReadCredentials(attachment.archiveCdn).successOrThrow().headers
val messageReceiver = AppDependencies.signalServiceMessageReceiver
val pointer = createThumbnailPointer(attachment)
Log.w(TAG, "Downloading thumbnail for $attachmentId mediaName=${attachment.getThumbnailMediaName()}")
val downloadResult = messageReceiver
.retrieveArchivedAttachment(
SignalStore.svr.getOrCreateMasterKey().deriveBackupKey().deriveMediaSecrets(attachment.getThumbnailMediaName()),
cdnCredentials,
thumbnailTransferFile,
pointer,
thumbnailFile,
maxThumbnailSize,
true,
progressListener
)
SignalDatabase.attachments.finalizeAttachmentThumbnailAfterDownload(attachmentId, attachment.archiveMediaId!!, downloadResult.dataStream, thumbnailTransferFile)
}
private fun markFailed(messageId: Long, attachmentId: AttachmentId) {
try {
SignalDatabase.attachments.setTransferProgressFailed(attachmentId, messageId)
} catch (e: MmsException) {
Log.w(TAG, e)
}
SignalDatabase.attachments.setTransferProgressFailed(attachmentId, messageId)
}
private fun markPermanentlyFailed(messageId: Long, attachmentId: AttachmentId) {
SignalDatabase.attachments.setTransferProgressPermanentFailure(attachmentId, messageId)
}
@VisibleForTesting
internal class InvalidPartException : Exception {
constructor(s: String?) : super(s)
constructor(e: Exception?) : super(e)
}
enum class RestoreMode(val value: Int) {
THUMBNAIL(0),
ORIGINAL(1),
BOTH(2);
companion object {
fun deserialize(value: Int): RestoreMode {
return values().firstOrNull { it.value == value } ?: ORIGINAL
}
}
}
private data class RemoteData(val remoteId: SignalServiceAttachmentRemoteId, val cdnNumber: Int)
class Factory : Job.Factory<RestoreAttachmentJob?> {
override fun create(parameters: Parameters, serializedData: ByteArray?): RestoreAttachmentJob {
val data = JsonJobData.deserialize(serializedData)
val data = RestoreAttachmentJobData.ADAPTER.decode(serializedData!!)
return RestoreAttachmentJob(
parameters = parameters,
messageId = data.getLong(KEY_MESSAGE_ID),
attachmentId = AttachmentId(data.getLong(KEY_ATTACHMENT_ID)),
manual = data.getBoolean(KEY_MANUAL),
forceArchiveDownload = data.getBooleanOrDefault(KEY_FORCE_ARCHIVE, false),
restoreMode = RestoreMode.deserialize(data.getIntOrDefault(KEY_RESTORE_MODE, RestoreMode.ORIGINAL.value))
messageId = data.messageId,
attachmentId = AttachmentId(data.attachmentId),
offloaded = data.offloaded
)
}
}

View File

@@ -4,13 +4,13 @@
*/
package org.thoughtcrime.securesms.jobs
import android.text.TextUtils
import androidx.annotation.VisibleForTesting
import org.signal.core.util.logging.Log
import org.signal.libsignal.protocol.InvalidMessageException
import org.thoughtcrime.securesms.attachments.AttachmentId
import org.thoughtcrime.securesms.attachments.DatabaseAttachment
import org.thoughtcrime.securesms.attachments.InvalidAttachmentException
import org.thoughtcrime.securesms.backup.v2.BackupRepository
import org.thoughtcrime.securesms.backup.v2.BackupRepository.getThumbnailMediaName
import org.thoughtcrime.securesms.backup.v2.database.createArchiveThumbnailPointer
import org.thoughtcrime.securesms.database.AttachmentTable
import org.thoughtcrime.securesms.database.SignalDatabase
import org.thoughtcrime.securesms.dependencies.AppDependencies
@@ -19,16 +19,11 @@ import org.thoughtcrime.securesms.jobmanager.JobLogger.format
import org.thoughtcrime.securesms.jobmanager.JsonJobData
import org.thoughtcrime.securesms.jobmanager.impl.NetworkConstraint
import org.thoughtcrime.securesms.keyvalue.SignalStore
import org.thoughtcrime.securesms.notifications.v2.ConversationId.Companion.forConversation
import org.thoughtcrime.securesms.transport.RetryLaterException
import org.thoughtcrime.securesms.util.RemoteConfig
import org.whispersystems.signalservice.api.messages.SignalServiceAttachment
import org.whispersystems.signalservice.api.messages.SignalServiceAttachmentPointer
import org.whispersystems.signalservice.api.messages.SignalServiceAttachmentRemoteId
import org.whispersystems.signalservice.api.push.exceptions.PushNetworkException
import org.whispersystems.signalservice.api.push.exceptions.MissingConfigurationException
import java.io.File
import java.io.IOException
import java.util.Optional
import java.util.concurrent.TimeUnit
/**
@@ -37,7 +32,7 @@ import java.util.concurrent.TimeUnit
class RestoreAttachmentThumbnailJob private constructor(
parameters: Parameters,
private val messageId: Long,
attachmentId: AttachmentId
val attachmentId: AttachmentId
) : BaseJob(parameters) {
companion object {
@@ -54,8 +49,6 @@ class RestoreAttachmentThumbnailJob private constructor(
}
}
private val attachmentId: Long
constructor(messageId: Long, attachmentId: AttachmentId, highPriority: Boolean = false) : this(
Parameters.Builder()
.setQueue(constructQueueString(attachmentId))
@@ -68,14 +61,10 @@ class RestoreAttachmentThumbnailJob private constructor(
attachmentId
)
init {
this.attachmentId = attachmentId.id
}
override fun serialize(): ByteArray? {
return JsonJobData.Builder()
.putLong(KEY_MESSAGE_ID, messageId)
.putLong(KEY_ATTACHMENT_ID, attachmentId)
.putLong(KEY_ATTACHMENT_ID, attachmentId.id)
.serialize()
}
@@ -83,35 +72,10 @@ class RestoreAttachmentThumbnailJob private constructor(
return KEY
}
override fun onAdded() {
Log.i(TAG, "onAdded() messageId: $messageId attachmentId: $attachmentId")
val attachmentId = AttachmentId(attachmentId)
val attachment = SignalDatabase.attachments.getAttachment(attachmentId)
val pending = attachment != null &&
attachment.thumbnailRestoreState != AttachmentTable.ThumbnailRestoreState.FINISHED &&
attachment.thumbnailRestoreState != AttachmentTable.ThumbnailRestoreState.PERMANENT_FAILURE &&
attachment.thumbnailRestoreState != AttachmentTable.ThumbnailRestoreState.NONE
if (pending) {
Log.i(TAG, "onAdded() Marking thumbnail restore progress as 'started'")
SignalDatabase.attachments.setThumbnailTransferState(messageId, attachmentId, AttachmentTable.ThumbnailRestoreState.IN_PROGRESS)
}
}
@Throws(Exception::class)
@Throws(Exception::class, IOException::class, InvalidAttachmentException::class, InvalidMessageException::class, MissingConfigurationException::class)
public override fun onRun() {
doWork()
if (!SignalDatabase.messages.isStory(messageId)) {
AppDependencies.messageNotifier.updateNotification(context, forConversation(0))
}
}
@Throws(IOException::class, RetryLaterException::class)
fun doWork() {
Log.i(TAG, "onRun() messageId: $messageId attachmentId: $attachmentId")
val attachmentId = AttachmentId(attachmentId)
val attachment = SignalDatabase.attachments.getAttachment(attachmentId)
if (attachment == null) {
@@ -119,79 +83,21 @@ class RestoreAttachmentThumbnailJob private constructor(
return
}
downloadThumbnail(attachmentId, attachment)
}
override fun onFailure() {
Log.w(TAG, format(this, "onFailure() thumbnail messageId: $messageId attachmentId: $attachmentId "))
val attachmentId = AttachmentId(attachmentId)
markFailed(messageId, attachmentId)
}
override fun onShouldRetry(exception: Exception): Boolean {
return exception is PushNetworkException ||
exception is RetryLaterException
}
@Throws(InvalidPartException::class)
private fun createThumbnailPointer(attachment: DatabaseAttachment): SignalServiceAttachmentPointer {
if (TextUtils.isEmpty(attachment.remoteKey)) {
throw InvalidPartException("empty encrypted key")
}
val backupKey = SignalStore.svr.getOrCreateMasterKey().deriveBackupKey()
val backupDirectories = BackupRepository.getCdnBackupDirectories().successOrThrow()
return try {
val key = backupKey.deriveThumbnailTransitKey(attachment.getThumbnailMediaName())
val mediaId = backupKey.deriveMediaId(attachment.getThumbnailMediaName()).encode()
SignalServiceAttachmentPointer(
attachment.archiveThumbnailCdn,
SignalServiceAttachmentRemoteId.Backup(
backupDir = backupDirectories.backupDir,
mediaDir = backupDirectories.mediaDir,
mediaId = mediaId
),
null,
key,
Optional.empty(),
Optional.empty(),
0,
0,
Optional.empty(),
Optional.empty(),
attachment.incrementalMacChunkSize,
Optional.empty(),
attachment.voiceNote,
attachment.borderless,
attachment.videoGif,
Optional.empty(),
Optional.ofNullable(attachment.blurHash).map { it.hash },
attachment.uploadTimestamp,
attachment.uuid
)
} catch (e: IOException) {
Log.w(TAG, e)
throw InvalidPartException(e)
} catch (e: ArithmeticException) {
Log.w(TAG, e)
throw InvalidPartException(e)
}
}
private fun downloadThumbnail(attachmentId: AttachmentId, attachment: DatabaseAttachment) {
if (attachment.thumbnailRestoreState == AttachmentTable.ThumbnailRestoreState.FINISHED) {
Log.w(TAG, "$attachmentId already has thumbnail downloaded")
return
}
if (attachment.thumbnailRestoreState == AttachmentTable.ThumbnailRestoreState.NONE) {
Log.w(TAG, "$attachmentId has no thumbnail state")
return
}
if (attachment.thumbnailRestoreState == AttachmentTable.ThumbnailRestoreState.PERMANENT_FAILURE) {
Log.w(TAG, "$attachmentId thumbnail permanently failed")
return
}
if (attachment.archiveMediaName == null) {
Log.w(TAG, "$attachmentId was never archived! Cannot proceed.")
return
@@ -202,20 +108,15 @@ class RestoreAttachmentThumbnailJob private constructor(
val thumbnailFile: File = SignalDatabase.attachments.createArchiveThumbnailTransferFile()
val progressListener = object : SignalServiceAttachment.ProgressListener {
override fun onAttachmentProgress(total: Long, progress: Long) {
}
override fun shouldCancel(): Boolean {
return this@RestoreAttachmentThumbnailJob.isCanceled
}
override fun onAttachmentProgress(total: Long, progress: Long) = Unit
override fun shouldCancel(): Boolean = this@RestoreAttachmentThumbnailJob.isCanceled
}
val cdnCredentials = BackupRepository.getCdnReadCredentials(attachment.archiveCdn).successOrThrow().headers
val messageReceiver = AppDependencies.signalServiceMessageReceiver
val pointer = createThumbnailPointer(attachment)
val pointer = attachment.createArchiveThumbnailPointer()
Log.w(TAG, "Downloading thumbnail for $attachmentId")
val downloadResult = messageReceiver
Log.i(TAG, "Downloading thumbnail for $attachmentId")
val downloadResult = AppDependencies.signalServiceMessageReceiver
.retrieveArchivedAttachment(
SignalStore.svr.getOrCreateMasterKey().deriveBackupKey().deriveMediaSecrets(attachment.getThumbnailMediaName()),
cdnCredentials,
@@ -228,16 +129,20 @@ class RestoreAttachmentThumbnailJob private constructor(
)
SignalDatabase.attachments.finalizeAttachmentThumbnailAfterDownload(attachmentId, attachment.archiveMediaId!!, downloadResult.dataStream, thumbnailTransferFile)
if (!SignalDatabase.messages.isStory(messageId)) {
AppDependencies.messageNotifier.updateNotification(context)
}
}
private fun markFailed(messageId: Long, attachmentId: AttachmentId) {
override fun onFailure() {
Log.w(TAG, format(this, "onFailure() thumbnail messageId: $messageId attachmentId: $attachmentId "))
SignalDatabase.attachments.setThumbnailRestoreProgressFailed(attachmentId, messageId)
}
@VisibleForTesting
internal class InvalidPartException : Exception {
constructor(s: String?) : super(s)
constructor(e: Exception?) : super(e)
override fun onShouldRetry(exception: Exception): Boolean {
return exception is IOException
}
class Factory : Job.Factory<RestoreAttachmentThumbnailJob?> {

View File

@@ -15,7 +15,7 @@ import org.signal.libsignal.protocol.InvalidMessageException
import org.thoughtcrime.securesms.attachments.AttachmentId
import org.thoughtcrime.securesms.backup.v2.local.ArchiveFileSystem
import org.thoughtcrime.securesms.database.AttachmentTable
import org.thoughtcrime.securesms.database.AttachmentTable.LocalRestorableAttachment
import org.thoughtcrime.securesms.database.AttachmentTable.RestorableAttachment
import org.thoughtcrime.securesms.database.SignalDatabase
import org.thoughtcrime.securesms.dependencies.AppDependencies
import org.thoughtcrime.securesms.jobmanager.Job
@@ -47,9 +47,9 @@ class RestoreLocalAttachmentJob private constructor(
var restoreAttachmentJobs: MutableList<Job>
do {
val possibleRestorableAttachments: List<LocalRestorableAttachment> = SignalDatabase.attachments.getLocalRestorableAttachments(500)
val restorableAttachments = ArrayList<LocalRestorableAttachment>(possibleRestorableAttachments.size)
val notRestorableAttachments = ArrayList<LocalRestorableAttachment>(possibleRestorableAttachments.size)
val possibleRestorableAttachments: List<RestorableAttachment> = SignalDatabase.attachments.getRestorableAttachments(500)
val restorableAttachments = ArrayList<RestorableAttachment>(possibleRestorableAttachments.size)
val notRestorableAttachments = ArrayList<RestorableAttachment>(possibleRestorableAttachments.size)
restoreAttachmentJobs = ArrayList(possibleRestorableAttachments.size)
@@ -71,8 +71,8 @@ class RestoreLocalAttachmentJob private constructor(
}
SignalDatabase.rawDatabase.withinTransaction {
SignalDatabase.attachments.setRestoreInProgressTransferState(restorableAttachments)
SignalDatabase.attachments.setRestoreFailedTransferState(notRestorableAttachments)
SignalDatabase.attachments.setRestoreTransferState(restorableAttachments, AttachmentTable.TRANSFER_RESTORE_IN_PROGRESS)
SignalDatabase.attachments.setRestoreTransferState(notRestorableAttachments, AttachmentTable.TRANSFER_PROGRESS_FAILED)
SignalStore.backup.totalRestorableAttachmentSize = SignalDatabase.attachments.getRemainingRestorableAttachmentSize()
AppDependencies.jobManager.addAll(restoreAttachmentJobs)
@@ -92,7 +92,7 @@ class RestoreLocalAttachmentJob private constructor(
}
}
private constructor(queue: String, attachment: LocalRestorableAttachment, info: DocumentFileInfo) : this(
private constructor(queue: String, attachment: RestorableAttachment, info: DocumentFileInfo) : this(
Parameters.Builder()
.setQueue(queue)
.setLifespan(Parameters.IMMORTAL)

View File

@@ -255,7 +255,7 @@ public class LegacyMigrationJob extends MigrationJob {
attachmentDb.setTransferState(attachment.mmsId, attachment.attachmentId, AttachmentTable.TRANSFER_PROGRESS_DONE);
} else if (record != null && !record.isOutgoing() && record.isPush()) {
Log.i(TAG, "queuing new attachment download job for incoming push part " + attachment.attachmentId + ".");
AppDependencies.getJobManager().add(new AttachmentDownloadJob(attachment.mmsId, attachment.attachmentId, false, false));
AppDependencies.getJobManager().add(new AttachmentDownloadJob(attachment.mmsId, attachment.attachmentId, false));
}
reader.close();
}

View File

@@ -13,4 +13,4 @@ import org.thoughtcrime.securesms.service.webrtc.PendingParticipantCollection
data class PendingParticipantsState(
val pendingParticipantCollection: PendingParticipantCollection,
val isInPipMode: Boolean
)
)