Improve db performance when dealing with many missing thumbnails.

This commit is contained in:
Cody Henthorne
2026-08-21 16:52:57 -04:00
committed by GitHub
parent a27352008a
commit 6136f50977
16 changed files with 720 additions and 134 deletions
@@ -6,6 +6,7 @@ import androidx.test.ext.junit.runners.AndroidJUnit4
import androidx.test.filters.FlakyTest
import androidx.test.platform.app.InstrumentationRegistry
import assertk.assertThat
import assertk.assertions.containsExactly
import assertk.assertions.hasSize
import assertk.assertions.isEmpty
import assertk.assertions.isEqualTo
@@ -218,14 +219,41 @@ class AttachmentTableTest {
SignalDatabase.attachments.setArchiveTransferState(attachmentId, AttachmentTable.ArchiveTransferState.FINISHED)
// Reset the transfer state by plaintextHash+remoteKey
val plaintextHash = SignalDatabase.attachments.getAttachment(attachmentId)!!.dataHash!!.decodeBase64OrThrow()
val remoteKey = SignalDatabase.attachments.getAttachment(attachmentId)!!.remoteKey!!.decodeBase64OrThrow()
SignalDatabase.attachments.resetArchiveTransferStateByPlaintextHashAndRemoteKeyIfNecessary(plaintextHash, remoteKey)
val inserted = SignalDatabase.attachments.getAttachment(attachmentId)!!
SignalDatabase.attachments.resetArchiveTransferStateByPlaintextHashAndRemoteKeyIfNecessary(
listOf(AttachmentTable.MediaNameParts(plaintextHash = inserted.dataHash!!, remoteKey = inserted.remoteKey!!))
)
// Verify it's been reset
assertThat(SignalDatabase.attachments.getAttachment(attachmentId)!!.archiveTransferState).isEqualTo(AttachmentTable.ArchiveTransferState.NONE)
}
@Test
fun resetArchiveTransferStateByPlaintextHashAndRemoteKey_batchedMatch() {
// Given two archive-finished attachments that still have their local data
val attachmentIds = listOf(byteArrayOf(1, 2, 3, 4, 5), byteArrayOf(6, 7, 8, 9, 10)).mapIndexed { index, data ->
val blob = AppDependencies.blobs.forData(data).createForSingleSessionInMemory()
val attachment = createAttachment(index + 1L, blob, TransformProperties.empty())
val attachmentId = SignalDatabase.attachments.insertAttachmentsForMessage(-1L, listOf(attachment), emptyList()).values.first()
SignalDatabase.attachments.finalizeAttachmentAfterUpload(attachmentId, AttachmentTableTestUtil.createUploadResult(attachmentId))
SignalDatabase.attachments.setArchiveTransferState(attachmentId, AttachmentTable.ArchiveTransferState.FINISHED)
attachmentId
}
val mediaNames = attachmentIds.map { attachmentId ->
val attachment = SignalDatabase.attachments.getAttachment(attachmentId)!!
AttachmentTable.MediaNameParts(plaintextHash = attachment.dataHash!!, remoteKey = attachment.remoteKey!!)
}
val results = SignalDatabase.attachments.resetArchiveTransferStateByPlaintextHashAndRemoteKeyIfNecessary(mediaNames)
// Both are reported as reset, and both actually are
assertThat(results).containsExactly(AttachmentTable.ArchiveTransferStateResetResult.RESET, AttachmentTable.ArchiveTransferStateResetResult.RESET)
attachmentIds.forEach { attachmentId ->
assertThat(SignalDatabase.attachments.getAttachment(attachmentId)!!.archiveTransferState).isEqualTo(AttachmentTable.ArchiveTransferState.NONE)
}
}
@Test
fun resetArchiveTransferStateForLocalBackupMedia_onlyResetsLocalBackupMedia() {
// Given one archive-finished attachment restored from a local backup, and one that wasn't
@@ -462,6 +490,55 @@ class AttachmentTableTest {
assertThat(result.archiveCdn).isNull()
}
@Test
fun givenPermanentlyFailedThumbnail_whenIFinalizeAttachment_thenIExpectThumbnailStateNoneSoItCanBeArchivedAgain() {
val data = byteArrayOf(1, 2, 3, 4, 5)
val attachment = createAttachmentPointer("remote-key-1".toByteArray(), data.size)
val messageResult = SignalDatabase.messages.insertMessageInbox(createIncomingMessage(serverTime = 0.days, attachment = attachment)).get()
val attachmentId = messageResult.insertedAttachments!![attachment]!!
SignalDatabase.attachments.setTransferState(messageResult.messageId, attachmentId, AttachmentTable.TRANSFER_PROGRESS_STARTED)
setDataHashEnd(attachmentId, "kMDoNIQjPGiuKzZHUCcLBiIF5wsvE7dqLKvvKG1JLmc=")
SignalDatabase.attachments.setArchiveThumbnailTransferState(attachmentId, AttachmentTable.ArchiveTransferState.PERMANENT_FAILURE)
SignalDatabase.attachments.finalizeAttachmentAfterDownload(messageResult.messageId, attachmentId, ByteArrayInputStream(data), archiveRestore = true, restoredFromArchiveCdn = true)
// Local bytes are back, so the thumbnail is eligible for archiving again
assertThat(SignalDatabase.attachments.getArchiveThumbnailTransferState(attachmentId)).isEqualTo(AttachmentTable.ArchiveTransferState.NONE)
}
@Test
fun givenPermanentlyFailedThumbnail_whenIFinalizeAnOrdinaryDownload_thenIExpectTheMarkToSurvive() {
val data = byteArrayOf(1, 2, 3, 4, 5)
val attachment = createAttachmentPointer("remote-key-1".toByteArray(), data.size)
val messageResult = SignalDatabase.messages.insertMessageInbox(createIncomingMessage(serverTime = 0.days, attachment = attachment)).get()
val attachmentId = messageResult.insertedAttachments!![attachment]!!
SignalDatabase.attachments.setTransferState(messageResult.messageId, attachmentId, AttachmentTable.TRANSFER_PROGRESS_STARTED)
SignalDatabase.attachments.setArchiveThumbnailTransferState(attachmentId, AttachmentTable.ArchiveTransferState.PERMANENT_FAILURE)
SignalDatabase.attachments.finalizeAttachmentAfterDownload(messageResult.messageId, attachmentId, ByteArrayInputStream(data))
assertThat(SignalDatabase.attachments.getArchiveThumbnailTransferState(attachmentId)).isEqualTo(AttachmentTable.ArchiveTransferState.PERMANENT_FAILURE)
}
@Test
fun givenFinishedThumbnail_whenIFinalizeAttachment_thenIExpectThumbnailStateUntouched() {
val data = byteArrayOf(1, 2, 3, 4, 5)
val attachment = createAttachmentPointer("remote-key-1".toByteArray(), data.size)
val messageResult = SignalDatabase.messages.insertMessageInbox(createIncomingMessage(serverTime = 0.days, attachment = attachment)).get()
val attachmentId = messageResult.insertedAttachments!![attachment]!!
SignalDatabase.attachments.setTransferState(messageResult.messageId, attachmentId, AttachmentTable.TRANSFER_PROGRESS_STARTED)
SignalDatabase.attachments.setArchiveThumbnailTransferState(attachmentId, AttachmentTable.ArchiveTransferState.FINISHED)
SignalDatabase.attachments.finalizeAttachmentAfterDownload(messageResult.messageId, attachmentId, ByteArrayInputStream(data), archiveRestore = true, restoredFromArchiveCdn = true)
// Only PERMANENT_FAILURE is lifted, an already-archived thumbnail keeps its state
assertThat(SignalDatabase.attachments.getArchiveThumbnailTransferState(attachmentId)).isEqualTo(AttachmentTable.ArchiveTransferState.FINISHED)
}
@Test
fun givenArchiveCdnRestore_whenIFinalizeAttachment_thenIExpectArchiveStateFinished() {
val data = byteArrayOf(1, 2, 3, 4, 5)
@@ -738,6 +815,14 @@ class AttachmentTableTest {
.run()
}
private fun setDataHashEnd(attachmentId: AttachmentId, hashEnd: String) {
SignalDatabase.rawDatabase
.update(AttachmentTable.TABLE_NAME)
.values(AttachmentTable.DATA_HASH_END to hashEnd)
.where("${AttachmentTable.ID} = ?", attachmentId.id)
.run()
}
/** Defaults to a non-media content type, which keeps the candidate eligible without also having to generate a thumbnail file. */
private fun seedOffloadCandidate(contentType: String = "application/pdf"): AttachmentId {
val data = byteArrayOf(1, 2, 3, 4, 5)
@@ -704,6 +704,7 @@ class SyncMessageProcessorTest_synchronizeDeleteForMe {
archiveCdn = this.archiveCdn,
thumbnailRestoreState = this.thumbnailRestoreState,
archiveTransferState = this.archiveTransferState,
archiveThumbnailTransferState = this.archiveThumbnailTransferState,
uuid = uuid,
quoteTargetContentType = this.quoteTargetContentType,
metadata = null
@@ -40,6 +40,9 @@ class DatabaseAttachment : Attachment {
@JvmField
val archiveTransferState: AttachmentTable.ArchiveTransferState
@JvmField
val archiveThumbnailTransferState: AttachmentTable.ArchiveTransferState
/** Metadata for this attachment, if null, no attempt was made to load the metadata and does not imply there is none */
@JvmField
val metadata: AttachmentMetadata?
@@ -80,6 +83,7 @@ class DatabaseAttachment : Attachment {
archiveCdn: Int?,
thumbnailRestoreState: AttachmentTable.ThumbnailRestoreState,
archiveTransferState: AttachmentTable.ArchiveTransferState,
archiveThumbnailTransferState: AttachmentTable.ArchiveTransferState,
uuid: UUID?,
quoteTargetContentType: String?,
metadata: AttachmentMetadata?
@@ -118,6 +122,7 @@ class DatabaseAttachment : Attachment {
this.archiveCdn = archiveCdn
this.thumbnailRestoreState = thumbnailRestoreState
this.archiveTransferState = archiveTransferState
this.archiveThumbnailTransferState = archiveThumbnailTransferState
this.metadata = metadata
}
@@ -131,6 +136,7 @@ class DatabaseAttachment : Attachment {
archiveCdn = parcel.readInt().takeIf { it != NO_ARCHIVE_CDN }
thumbnailRestoreState = AttachmentTable.ThumbnailRestoreState.deserialize(parcel.readInt())
archiveTransferState = AttachmentTable.ArchiveTransferState.deserialize(parcel.readInt())
archiveThumbnailTransferState = AttachmentTable.ArchiveTransferState.deserialize(parcel.readInt())
metadata = ParcelCompat.readParcelable(parcel, AttachmentMetadata::class.java.classLoader, AttachmentMetadata::class.java)
}
@@ -145,6 +151,7 @@ class DatabaseAttachment : Attachment {
dest.writeInt(archiveCdn ?: NO_ARCHIVE_CDN)
dest.writeInt(thumbnailRestoreState.value)
dest.writeInt(archiveTransferState.value)
dest.writeInt(archiveThumbnailTransferState.value)
dest.writeParcelable(metadata, 0)
}
@@ -231,6 +231,7 @@ class AttachmentTable(
THUMBNAIL_FILE,
THUMBNAIL_RESTORE_STATE,
ARCHIVE_TRANSFER_STATE,
ARCHIVE_THUMBNAIL_TRANSFER_STATE,
ATTACHMENT_UUID
)
@@ -288,6 +289,9 @@ class AttachmentTable(
private const val DATA_FILE_INDEX = "attachment_data_index"
private const val DATA_HASH_REMOTE_KEY_INDEX = "attachment_data_hash_end_remote_key_index"
/** How many media names fit in one statement. Each binds two args, so this has to stay under half of [SqlUtil.MAX_QUERY_ARGS]. */
const val ARCHIVE_MEDIA_KEY_BATCH_SIZE = 400
@JvmField
val CREATE_INDEXS = arrayOf(
"CREATE INDEX IF NOT EXISTS attachment_message_id_index ON $TABLE_NAME ($MESSAGE_ID);",
@@ -1150,65 +1154,167 @@ class AttachmentTable(
}
/**
* Resets the archive upload state by hash/key if we believe the attachment should have been uploaded already.
* Internal-only. Drops the local bytes for a single attachment exactly the way [markEligibleAttachmentsAsOptimized] does, bypassing every eligibility rule so a
* specific attachment can be put into the offloaded state on demand.
*/
fun resetArchiveTransferStateByPlaintextHashAndRemoteKeyIfNecessary(plaintextHash: ByteArray, remoteKey: ByteArray): ArchiveTransferStateResetResult {
fun debugOffloadAttachment(attachmentId: AttachmentId): Int {
return writableDatabase
.update(TABLE_NAME)
.values(
TRANSFER_STATE to TRANSFER_RESTORE_OFFLOADED,
DATA_FILE to null,
DATA_RANDOM to null,
TRANSFORM_PROPERTIES to null,
DATA_HASH_START to null,
OFFLOAD_RESTORED_AT to 0
)
.where("$ID = ?", attachmentId.id)
.run()
.also { AppDependencies.databaseObserver.notifyAttachmentUpdatedObservers() }
}
/**
* Resets the archive upload state by hash/key for any media we believe should have been uploaded already. Accepts at most
* [ARCHIVE_MEDIA_KEY_BATCH_SIZE] names, since they all go into one statement.
*/
fun resetArchiveTransferStateByPlaintextHashAndRemoteKeyIfNecessary(mediaNames: List<MediaNameParts>): List<ArchiveTransferStateResetResult> {
return resetArchiveTransferStateIfNecessary(
plaintextHash = plaintextHash,
remoteKey = remoteKey,
mediaNames = mediaNames,
stateColumn = ARCHIVE_TRANSFER_STATE,
values = contentValuesOf(
ARCHIVE_TRANSFER_STATE to ArchiveTransferState.NONE.value,
ARCHIVE_CDN to null
)
),
unrecoverableState = null
)
}
/**
* Resets the archive thumbnail upload state by hash/key if we believe the thumbnail should have been uploaded already.
* Resets the archive thumbnail upload state by hash/key for any thumbnail we believe should have been uploaded already. Accepts at most
* [ARCHIVE_MEDIA_KEY_BATCH_SIZE] names, since they all go into one statement.
*
* Unlike full-size media, a thumbnail with no local bytes is marked [ArchiveTransferState.PERMANENT_FAILURE] rather than left alone. The next
* backup stops tracking it and later crawls stop rediscovering something they can never repair. Restoring the full-size media lifts the mark.
*/
fun resetArchiveThumbnailTransferStateByPlaintextHashAndRemoteKeyIfNecessary(plaintextHash: ByteArray, remoteKey: ByteArray): ArchiveTransferStateResetResult {
fun resetArchiveThumbnailTransferStateByPlaintextHashAndRemoteKeyIfNecessary(mediaNames: List<MediaNameParts>): List<ArchiveTransferStateResetResult> {
return resetArchiveTransferStateIfNecessary(
plaintextHash = plaintextHash,
remoteKey = remoteKey,
mediaNames = mediaNames,
stateColumn = ARCHIVE_THUMBNAIL_TRANSFER_STATE,
values = contentValuesOf(ARCHIVE_THUMBNAIL_TRANSFER_STATE to ArchiveTransferState.NONE.value)
values = contentValuesOf(ARCHIVE_THUMBNAIL_TRANSFER_STATE to ArchiveTransferState.NONE.value),
unrecoverableState = ArchiveTransferState.PERMANENT_FAILURE
)
}
private fun resetArchiveTransferStateIfNecessary(plaintextHash: ByteArray, remoteKey: ByteArray, stateColumn: String, values: ContentValues): ArchiveTransferStateResetResult {
val encodedHash = Base64.encodeWithPadding(plaintextHash)
val encodedKey = Base64.encodeWithPadding(remoteKey)
val finishedClause = "$DATA_HASH_END = ? AND $REMOTE_KEY = ? AND $stateColumn = ${ArchiveTransferState.FINISHED.value}"
/**
* For each of the given [mediaNames], clears [stateColumn] if we believe the media was already uploaded but something has told us it isn't there.
*/
private fun resetArchiveTransferStateIfNecessary(
mediaNames: List<MediaNameParts>,
stateColumn: String,
values: ContentValues,
unrecoverableState: ArchiveTransferState?
): List<ArchiveTransferStateResetResult> {
require(mediaNames.size <= ARCHIVE_MEDIA_KEY_BATCH_SIZE) { "Batch of ${mediaNames.size} exceeds what fits in one statement." }
return writableDatabase.withinTransaction { db ->
val isFinished = db
.exists(TABLE_NAME)
.where(finishedClause, encodedHash, encodedKey)
.run()
if (!isFinished) {
return@withinTransaction ArchiveTransferStateResetResult.NOT_NEEDED
}
val hasLocalData = db
.exists(TABLE_NAME)
.where("$DATA_HASH_END = ? AND $REMOTE_KEY = ? AND $DATA_FILE NOT NULL", encodedHash, encodedKey)
.run()
// With no local data file there is nothing to re-upload, and clearing the state makes the export emit a tombstone instead of a locator, which in turn marks
// the media for deletion off of the CDN.
if (!hasLocalData) {
return@withinTransaction ArchiveTransferStateResetResult.SKIPPED_NO_LOCAL_DATA
}
db.update(TABLE_NAME)
.values(values)
.where(finishedClause, encodedHash, encodedKey)
.run()
ArchiveTransferStateResetResult.RESET
if (mediaNames.isEmpty()) {
return emptyList()
}
val hasLocalDataByMediaName: Map<MediaNameParts, Boolean> = getArchiveFinishedMediaWithLocalData(mediaNames, stateColumn)
// Without this, already-recorded media looks like gone-or-in-progress and re-arms a backfill that can never upload it.
val alreadyRecorded: Set<MediaNameParts> = if (unrecoverableState != null) {
getMatchingMediaNames(mediaNames, "$stateColumn = ${unrecoverableState.value}")
} else {
emptySet()
}
val recoverable = mutableListOf<MediaNameParts>()
val unrecoverable = mutableListOf<MediaNameParts>()
val results = mediaNames.map { mediaName ->
when (hasLocalDataByMediaName[mediaName]) {
null -> if (mediaName in alreadyRecorded) {
ArchiveTransferStateResetResult.MARKED_UNRECOVERABLE
} else {
ArchiveTransferStateResetResult.NOT_NEEDED
}
true -> {
recoverable += mediaName
ArchiveTransferStateResetResult.RESET
}
false -> if (unrecoverableState != null) {
unrecoverable += mediaName
ArchiveTransferStateResetResult.MARKED_UNRECOVERABLE
} else {
ArchiveTransferStateResetResult.SKIPPED_NO_LOCAL_DATA
}
}
}
if (recoverable.isNotEmpty()) {
updateArchiveFinishedMedia(recoverable, stateColumn, values, "$DATA_FILE NOT NULL")
}
if (unrecoverable.isNotEmpty() && unrecoverableState != null) {
updateArchiveFinishedMedia(unrecoverable, stateColumn, contentValuesOf(stateColumn to unrecoverableState.value), "$DATA_FILE IS NULL")
}
return results
}
/**
* Given [mediaNames], the ones we believe are finished, mapped to whether we still have local bytes.
*/
private fun getArchiveFinishedMediaWithLocalData(mediaNames: List<MediaNameParts>, stateColumn: String): Map<MediaNameParts, Boolean> {
val archived = getMatchingMediaNames(mediaNames, "$stateColumn = ${ArchiveTransferState.FINISHED.value}")
if (archived.isEmpty()) {
return emptyMap()
}
val withLocalData = getMatchingMediaNames(archived.toList(), "$DATA_FILE NOT NULL")
return archived.associateWith { it in withLocalData }
}
/**
* The subset of [mediaNames] that has at least one attachment satisfying [clause].
*/
private fun getMatchingMediaNames(mediaNames: List<MediaNameParts>, clause: String): Set<MediaNameParts> {
val tuples = mediaNameTuples(mediaNames)
return readableDatabase
.select(DATA_HASH_END, REMOTE_KEY)
.from("$TABLE_NAME INDEXED BY $DATA_HASH_REMOTE_KEY_INDEX")
.where("$clause AND ${tuples.where}", tuples.whereArgs)
.run()
.readToSet { cursor ->
MediaNameParts(
plaintextHash = cursor.requireNonNullString(DATA_HASH_END),
remoteKey = cursor.requireNonNullString(REMOTE_KEY)
)
}
}
/** [localDataClause] re-asserts the unlocked lookup's conclusion, so a concurrent restore or offload can't be written over. */
private fun updateArchiveFinishedMedia(mediaNames: List<MediaNameParts>, stateColumn: String, values: ContentValues, localDataClause: String): Int {
val tuples = mediaNameTuples(mediaNames)
return writableDatabase
.update("$TABLE_NAME INDEXED BY $DATA_HASH_REMOTE_KEY_INDEX")
.values(values)
.where("$stateColumn = ${ArchiveTransferState.FINISHED.value} AND $localDataClause AND ${tuples.where}", tuples.whereArgs)
.run()
}
/** Matches the hash/key pair of any of [mediaNames]. */
private fun mediaNameTuples(mediaNames: List<MediaNameParts>): SqlUtil.Query {
return SqlUtil.Query(
where = "($DATA_HASH_END, $REMOTE_KEY) IN (VALUES ${mediaNames.joinToString(separator = ", ") { "(?, ?)" }})",
whereArgs = mediaNames.flatMap { listOf(it.plaintextHash, it.remoteKey) }.toTypedArray()
)
}
/**
@@ -2017,7 +2123,14 @@ class AttachmentTable(
val dataFilePath = hashMatch?.file?.absolutePath ?: fileWriteResult.file.absolutePath
val updateCount = if (archiveRestore && existingPlaceholder.dataHash != null) {
// Only this branch matches purely by hash/key, which is the granularity the mark is applied at.
val restoringByHashAndKey = archiveRestore && existingPlaceholder.dataHash != null
if (restoringByHashAndKey && existingPlaceholder.archiveThumbnailTransferState == ArchiveTransferState.PERMANENT_FAILURE) {
values.put(ARCHIVE_THUMBNAIL_TRANSFER_STATE, ArchiveTransferState.NONE.value)
}
val updateCount = if (restoringByHashAndKey) {
// Can update all rows with the same mediaName as data_file column will likely be null
db.update(TABLE_NAME)
.values(values)
@@ -3744,6 +3857,7 @@ class AttachmentTable(
archiveCdn = cursor.requireIntOrNull(ARCHIVE_CDN),
thumbnailRestoreState = ThumbnailRestoreState.deserialize(cursor.requireInt(THUMBNAIL_RESTORE_STATE)),
archiveTransferState = ArchiveTransferState.deserialize(cursor.requireInt(ARCHIVE_TRANSFER_STATE)),
archiveThumbnailTransferState = ArchiveTransferState.deserialize(cursor.requireInt(ARCHIVE_THUMBNAIL_TRANSFER_STATE)),
uuid = UuidUtil.parseOrNull(cursor.requireString(ATTACHMENT_UUID)),
metadata = AttachmentMetadataTable.getMetadata(cursor)
)
@@ -4502,7 +4616,16 @@ class AttachmentTable(
/**
* The base64 [DATA_HASH_END] and [REMOTE_KEY] that a [MediaName] is derived from.
*/
data class MediaNameParts(val plaintextHash: String, val remoteKey: String)
data class MediaNameParts(val plaintextHash: String, val remoteKey: String) {
companion object {
fun fromBytes(plaintextHash: ByteArray, remoteKey: ByteArray): MediaNameParts {
return MediaNameParts(
plaintextHash = Base64.encodeWithPadding(plaintextHash),
remoteKey = Base64.encodeWithPadding(remoteKey)
)
}
}
}
enum class ArchiveTransferStateResetResult {
/** We cleared the state, so the media will be re-uploaded. */
@@ -4511,6 +4634,9 @@ class AttachmentTable(
/** We left the state alone because there are no local bytes to re-upload, so the media is unrecoverable rather than merely out of date. */
SKIPPED_NO_LOCAL_DATA,
/** Like [SKIPPED_NO_LOCAL_DATA], but the absence is recorded so it stops being tracked and rediscovered. Covers media already recorded by an earlier pass. */
MARKED_UNRECOVERABLE,
/** There was nothing to clear. The attachment is gone, or an upload is already underway. */
NOT_NEEDED
}
@@ -58,6 +58,7 @@ class MediaTable internal constructor(context: Context?, databaseHelper: SignalD
${AttachmentTable.TABLE_NAME}.${AttachmentTable.ARCHIVE_CDN},
${AttachmentTable.TABLE_NAME}.${AttachmentTable.THUMBNAIL_RESTORE_STATE},
${AttachmentTable.TABLE_NAME}.${AttachmentTable.ARCHIVE_TRANSFER_STATE},
${AttachmentTable.TABLE_NAME}.${AttachmentTable.ARCHIVE_THUMBNAIL_TRANSFER_STATE},
${AttachmentTable.TABLE_NAME}.${AttachmentTable.ATTACHMENT_UUID},
${MessageTable.TABLE_NAME}.${MessageTable.TYPE},
${MessageTable.TABLE_NAME}.${MessageTable.DATE_SENT},
@@ -163,6 +164,7 @@ class MediaTable internal constructor(context: Context?, databaseHelper: SignalD
${AttachmentTable.TABLE_NAME}.${AttachmentTable.ARCHIVE_CDN},
${AttachmentTable.TABLE_NAME}.${AttachmentTable.THUMBNAIL_RESTORE_STATE},
${AttachmentTable.TABLE_NAME}.${AttachmentTable.ARCHIVE_TRANSFER_STATE},
${AttachmentTable.TABLE_NAME}.${AttachmentTable.ARCHIVE_THUMBNAIL_TRANSFER_STATE},
${AttachmentTable.TABLE_NAME}.${AttachmentTable.ATTACHMENT_UUID},
${MessageTable.TABLE_NAME}.${MessageTable.TYPE},
${MessageTable.TABLE_NAME}.${MessageTable.DATE_SENT},
@@ -28,6 +28,7 @@ import org.thoughtcrime.securesms.R
import org.thoughtcrime.securesms.backup.v2.ArchivedMediaObject
import org.thoughtcrime.securesms.database.AttachmentTable
import org.thoughtcrime.securesms.database.AttachmentTable.ArchiveTransferStateResetResult
import org.thoughtcrime.securesms.database.AttachmentTable.MediaNameParts
import org.thoughtcrime.securesms.database.BackupMediaSnapshotTable
import org.thoughtcrime.securesms.database.SignalDatabase
import org.thoughtcrime.securesms.dependencies.AppDependencies
@@ -261,70 +262,42 @@ class ArchiveAttachmentReconciliationJob private constructor(
if (mayNeedReUploadCount > 0) {
Log.w(TAG, "Found $mayNeedReUploadCount attachments that are present in the target snapshot, but could not be found on the CDN. This could be a bookkeeping error, or the upload may still be in progress. Checking.", true)
var bookkeepingErrorCount = 0
var unrecoverableCount = 0
var fullSizeReUploadNeeded = false
var thumbnailReUploadNeeded = false
val tally = RepairTally()
val batch = ArrayList<BackupMediaSnapshotTable.MediaEntry>(AttachmentTable.ARCHIVE_MEDIA_KEY_BATCH_SIZE)
mediaObjectsThatMayNeedReUpload.forEach { mediaObjectCursor ->
val entry = BackupMediaSnapshotTable.MediaEntry.fromCursor(mediaObjectCursor)
batch += entry
if (internalUser) {
mediaIdsThatNeedUpload += MediaId(entry.mediaId)
}
val mediaIdLog = if (internalUser) "[${MediaId(entry.mediaId)}]" else ""
val logPrefix = if (entry.isThumbnail) "[Thumbnail]$mediaIdLog" else "[Fullsize]$mediaIdLog"
val resetResult = if (entry.isThumbnail) {
SignalDatabase.attachments.resetArchiveThumbnailTransferStateByPlaintextHashAndRemoteKeyIfNecessary(entry.plaintextHash, entry.remoteKey)
} else {
SignalDatabase.attachments.resetArchiveTransferStateByPlaintextHashAndRemoteKeyIfNecessary(entry.plaintextHash, entry.remoteKey)
}
when (resetResult) {
ArchiveTransferStateResetResult.RESET -> {
Log.w(TAG, "$logPrefix Reset transfer state by hash/key.", true)
bookkeepingErrorCount++
if (entry.isThumbnail) {
thumbnailReUploadNeeded = true
} else {
fullSizeReUploadNeeded = true
}
}
ArchiveTransferStateResetResult.SKIPPED_NO_LOCAL_DATA -> {
if (internalUser) {
Log.w(TAG, "$logPrefix Missing from the CDN with no local data to re-upload. Leaving the archive state alone so we keep the locator.", true)
}
unrecoverableCount++
}
ArchiveTransferStateResetResult.NOT_NEEDED -> {
Log.i(TAG, "$logPrefix Did not need to reset the transfer state by hash/key because it either no longer exists or the upload is already in-progress.", true)
// Deliberately not set for SKIPPED_NO_LOCAL_DATA, since the precautionary backfills these drive could never upload media that has no local bytes.
if (entry.isThumbnail) {
thumbnailReUploadNeeded = true
} else {
fullSizeReUploadNeeded = true
}
}
if (batch.size >= AttachmentTable.ARCHIVE_MEDIA_KEY_BATCH_SIZE) {
repairBatch(batch, tally, internalUser)
batch.clear()
}
}
repairBatch(batch, tally, internalUser)
stopwatch.split("mark-reupload")
if (bookkeepingErrorCount > 0) {
Log.w(TAG, "Found that $bookkeepingErrorCount/$mayNeedReUploadCount of the CDN mismatches were bookkeeping errors.", true)
if (tally.resetCount > 0) {
Log.w(TAG, "Found that ${tally.resetCount}/$mayNeedReUploadCount of the CDN mismatches were bookkeeping errors.", true)
maybePostReconciliationFailureNotification()
} else {
Log.i(TAG, "None of the $mayNeedReUploadCount CDN mismatches were bookkeeping errors.", true)
}
if (unrecoverableCount > 0) {
Log.w(TAG, "Found that $unrecoverableCount/$mayNeedReUploadCount of the CDN mismatches have no local data to re-upload. That media is not recoverable from this device.", true)
if (tally.unrecoverableCount > 0) {
Log.w(TAG, "Found that ${tally.unrecoverableCount}/$mayNeedReUploadCount of the CDN mismatches have no local data to re-upload. That media is not recoverable from this device.", true)
}
if (tally.markedUnrecoverableCount > 0) {
Log.w(TAG, "Marked ${tally.markedUnrecoverableCount}/$mayNeedReUploadCount of the CDN mismatches as permanently failed thumbnails, since there is no local data to rebuild them from. They will stop being tracked until the media is restored.", true)
}
if (tally.notNeededCount > 0) {
Log.i(TAG, "Did not need to reset ${tally.notNeededCount}/$mayNeedReUploadCount of the CDN mismatches, because they either no longer exist or an upload is already in-progress.", true)
}
Log.d(TAG, "AFTER:\n" + SignalDatabase.attachments.debugGetAttachmentStats().shortPrettyString(), true)
@@ -349,11 +322,11 @@ class ArchiveAttachmentReconciliationJob private constructor(
// No backup is started here on purpose. Re-uploading is the whole repair, and [ArchiveUploadProgress] is what decides whether the resulting CDN numbers
// warrant a fresh export once the backfill finishes uploading.
if (fullSizeReUploadNeeded) {
if (tally.fullSizeReUploadNeeded) {
Log.d(TAG, "Full size mismatch found. Enqueuing an attachment backfill job.", true)
AppDependencies.jobManager.add(ArchiveAttachmentBackfillJob())
}
if (thumbnailReUploadNeeded) {
if (tally.thumbnailReUploadNeeded) {
Log.d(TAG, "Thumbnail mismatch found. Enqueuing a thumbnail backfill job.", true)
AppDependencies.jobManager.add(ArchiveThumbnailBackfillJob())
}
@@ -546,6 +519,54 @@ class ArchiveAttachmentReconciliationJob private constructor(
return null
}
/**
* Clears the archive transfer state for a batch of media objects that a crawl couldn't find on the CDN, so they get re-uploaded, folding the outcomes into
* [tally].
*/
private fun repairBatch(batch: List<BackupMediaSnapshotTable.MediaEntry>, tally: RepairTally, internalUser: Boolean) {
if (batch.isEmpty()) {
return
}
for ((isThumbnail, entries) in batch.groupBy { it.isThumbnail }) {
val mediaNames = entries.map { MediaNameParts.fromBytes(plaintextHash = it.plaintextHash, remoteKey = it.remoteKey) }
val results = if (isThumbnail) {
SignalDatabase.attachments.resetArchiveThumbnailTransferStateByPlaintextHashAndRemoteKeyIfNecessary(mediaNames)
} else {
SignalDatabase.attachments.resetArchiveTransferStateByPlaintextHashAndRemoteKeyIfNecessary(mediaNames)
}
for ((entry, result) in entries.zip(results)) {
when (result) {
ArchiveTransferStateResetResult.RESET -> {
val mediaIdLog = if (internalUser) "[${MediaId(entry.mediaId)}]" else ""
val logPrefix = if (isThumbnail) "[Thumbnail]$mediaIdLog" else "[Fullsize]$mediaIdLog"
Log.w(TAG, "$logPrefix Reset transfer state by hash/key.", true)
tally.resetCount++
tally.markReUploadNeeded(isThumbnail)
}
ArchiveTransferStateResetResult.SKIPPED_NO_LOCAL_DATA -> {
tally.unrecoverableCount++
}
ArchiveTransferStateResetResult.MARKED_UNRECOVERABLE -> {
tally.markedUnrecoverableCount++
}
ArchiveTransferStateResetResult.NOT_NEEDED -> {
tally.notNeededCount++
// Deliberately not done for SKIPPED_NO_LOCAL_DATA, since the precautionary backfills these drive could never upload media that has no local bytes.
tally.markReUploadNeeded(isThumbnail)
}
}
}
}
}
private fun maybePostReconciliationFailureNotification() {
if (!RemoteConfig.internalUser) {
return
@@ -565,6 +586,23 @@ class ArchiveAttachmentReconciliationJob private constructor(
NotificationManagerCompat.from(context).notify(NotificationIds.RECONCILIATION_ERROR, notification)
}
private class RepairTally {
var resetCount = 0
var unrecoverableCount = 0
var markedUnrecoverableCount = 0
var notNeededCount = 0
var fullSizeReUploadNeeded = false
var thumbnailReUploadNeeded = false
fun markReUploadNeeded(isThumbnail: Boolean) {
if (isThumbnail) {
thumbnailReUploadNeeded = true
} else {
fullSizeReUploadNeeded = true
}
}
}
class Factory : Job.Factory<ArchiveAttachmentReconciliationJob> {
override fun create(parameters: Parameters, serializedData: ByteArray?): ArchiveAttachmentReconciliationJob {
val data = ArchiveAttachmentReconciliationJobData.ADAPTER.decode(serializedData!!)
@@ -20,7 +20,11 @@ import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.Surface
import androidx.compose.material3.Text
import androidx.compose.runtime.Composable
import androidx.compose.runtime.LaunchedEffect
import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.remember
import androidx.compose.runtime.setValue
import androidx.compose.ui.Modifier
import androidx.compose.ui.platform.LocalContext
import androidx.compose.ui.text.SpanStyle
@@ -32,9 +36,12 @@ import androidx.compose.ui.tooling.preview.Preview
import androidx.compose.ui.unit.dp
import androidx.core.os.bundleOf
import androidx.fragment.app.FragmentActivity
import org.signal.core.ui.compose.Buttons
import org.signal.core.ui.compose.ComposeFullScreenDialogFragment
import org.signal.core.ui.compose.Dialogs
import org.signal.core.ui.compose.Dividers
import org.signal.core.util.Util
import org.thoughtcrime.securesms.database.AttachmentTable
import org.thoughtcrime.securesms.database.model.MessageRecord
import org.thoughtcrime.securesms.messagedetails.InternalMessageDetailsViewModel.AttachmentInfo
import org.thoughtcrime.securesms.messagedetails.InternalMessageDetailsViewModel.ViewState
@@ -61,15 +68,32 @@ class InternalMessageDetailsFragment : ComposeFullScreenDialogFragment() {
@Composable
override fun DialogContent() {
val state by viewModel.state
val actionResult by viewModel.actionResult
val context = LocalContext.current
LaunchedEffect(actionResult) {
actionResult?.let {
Toast.makeText(context, it, Toast.LENGTH_SHORT).show()
viewModel.consumeActionResult()
}
}
state?.let {
Content(it)
Content(
state = it,
onOffloadLocalData = viewModel::offloadLocalData,
onDeleteFromCdn = viewModel::deleteFromCdn
)
}
}
}
@Composable
private fun Content(state: ViewState) {
private fun Content(
state: ViewState,
onOffloadLocalData: (Long) -> Unit = {},
onDeleteFromCdn: (Long, Boolean) -> Unit = { _, _ -> }
) {
val context = LocalContext.current
Surface(
@@ -145,7 +169,11 @@ private fun Content(state: ViewState) {
)
} else {
state.attachments.forEachIndexed { i, attachment ->
AttachmentBlock(attachment)
AttachmentBlock(
attachment = attachment,
onOffloadLocalData = onOffloadLocalData,
onDeleteFromCdn = onDeleteFromCdn
)
if (i != state.attachments.lastIndex) {
Dividers.Default()
@@ -191,7 +219,11 @@ private fun ClickToCopyRow(name: String, value: String, valueToCopy: String = va
}
@Composable
private fun AttachmentBlock(attachment: AttachmentInfo) {
private fun AttachmentBlock(
attachment: AttachmentInfo,
onOffloadLocalData: (Long) -> Unit,
onDeleteFromCdn: (Long, Boolean) -> Unit
) {
ClickToCopyRow(
name = "ID",
value = attachment.id.toString()
@@ -220,6 +252,128 @@ private fun AttachmentBlock(attachment: AttachmentInfo) {
name = "Transform Properties",
value = attachment.transformProperties ?: "null"
)
ClickToCopyRow(
name = "Has Local Data",
value = attachment.hasLocalData.toString()
)
ClickToCopyRow(
name = "Transfer State",
value = attachment.transferState.toString()
)
ClickToCopyRow(
name = "Archive CDN",
value = attachment.archiveCdn?.toString() ?: "null"
)
ClickToCopyRow(
name = "Archive Transfer State",
value = attachment.archiveTransferState.name
)
ClickToCopyRow(
name = "Archive Thumbnail Transfer State",
value = attachment.archiveThumbnailTransferState.name
)
DestructiveActions(
attachment = attachment,
onOffloadLocalData = onOffloadLocalData,
onDeleteFromCdn = onDeleteFromCdn
)
}
/**
* Collapsed by default, and each action confirms, because these are irreversible and sit in a list people scroll through to copy values.
*/
@Composable
private fun DestructiveActions(
attachment: AttachmentInfo,
onOffloadLocalData: (Long) -> Unit,
onDeleteFromCdn: (Long, Boolean) -> Unit
) {
var expanded by remember(attachment.id) { mutableStateOf(false) }
var pendingAction by remember(attachment.id) { mutableStateOf<PendingAction?>(null) }
Text(
text = if (expanded) "▾ Destructive test actions" else "▸ Destructive test actions",
style = MaterialTheme.typography.labelLarge,
modifier = Modifier
.clickable { expanded = !expanded }
.padding(8.dp)
.fillMaxWidth()
)
if (!expanded) {
return
}
val isArchived = attachment.archiveTransferState == AttachmentTable.ArchiveTransferState.FINISHED && attachment.archiveCdn != null
Buttons.MediumTonal(
onClick = { pendingAction = PendingAction.OFFLOAD },
modifier = Modifier
.padding(horizontal = 8.dp)
.fillMaxWidth()
) {
Text(text = "Offload local data")
}
Buttons.MediumTonal(
onClick = { pendingAction = PendingAction.DELETE_FULL_SIZE },
modifier = Modifier
.padding(horizontal = 8.dp)
.fillMaxWidth()
) {
Text(text = "Delete full-size from CDN")
}
Buttons.MediumTonal(
onClick = { pendingAction = PendingAction.DELETE_THUMBNAIL },
modifier = Modifier
.padding(horizontal = 8.dp)
.fillMaxWidth()
) {
Text(text = "Delete thumbnail from CDN")
}
when (pendingAction) {
null -> Unit
PendingAction.OFFLOAD -> Dialogs.SimpleAlertDialog(
title = "Offload local data?",
body = if (isArchived) {
"Drops the local bytes for attachment ${attachment.id}. The archive CDN copy stays, so it can be restored by tapping the attachment."
} else {
"This attachment is NOT on the archive CDN (state ${attachment.archiveTransferState.name}, cdn ${attachment.archiveCdn ?: "null"}). Offloading it deletes the only copy and it cannot be restored."
},
confirm = if (isArchived) "Offload" else "Delete the only copy",
dismiss = "Cancel",
onConfirm = { onOffloadLocalData(attachment.id) },
onDismiss = { pendingAction = null }
)
PendingAction.DELETE_FULL_SIZE -> Dialogs.SimpleAlertDialog(
title = "Delete full-size from CDN?",
body = "Immediately and irreversibly removes the full-size copy of attachment ${attachment.id} from the archive CDN. If the local bytes are ever offloaded, the media is gone for good.",
confirm = "Delete from CDN",
dismiss = "Cancel",
onConfirm = { onDeleteFromCdn(attachment.id, false) },
onDismiss = { pendingAction = null }
)
PendingAction.DELETE_THUMBNAIL -> Dialogs.SimpleAlertDialog(
title = "Delete thumbnail from CDN?",
body = "Immediately and irreversibly removes the thumbnail for attachment ${attachment.id} from the archive CDN. The full-size copy is untouched.",
confirm = "Delete from CDN",
dismiss = "Cancel",
onConfirm = { onDeleteFromCdn(attachment.id, true) },
onDismiss = { pendingAction = null }
)
}
}
private enum class PendingAction {
OFFLOAD,
DELETE_FULL_SIZE,
DELETE_THUMBNAIL
}
@Preview
@@ -12,45 +12,136 @@ import androidx.lifecycle.ViewModel
import androidx.lifecycle.viewModelScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.launch
import org.signal.core.models.backup.MediaName
import org.signal.core.models.database.AttachmentId
import org.signal.core.util.Base64
import org.signal.core.util.logging.Log
import org.signal.network.util.JsonUtil
import org.thoughtcrime.securesms.backup.v2.ArchivedMediaObject
import org.thoughtcrime.securesms.database.AttachmentTable
import org.thoughtcrime.securesms.database.SignalDatabase
import org.thoughtcrime.securesms.jobs.ArchiveCommitAttachmentDeletesJob
import org.thoughtcrime.securesms.keyvalue.SignalStore
import org.thoughtcrime.securesms.recipients.RecipientId
class InternalMessageDetailsViewModel(val messageId: Long) : ViewModel() {
companion object {
private val TAG = Log.tag(InternalMessageDetailsViewModel::class)
}
private val _state: MutableState<ViewState?> = mutableStateOf(null)
val state: State<ViewState?> = _state
private val _actionResult: MutableState<String?> = mutableStateOf(null)
val actionResult: State<String?> = _actionResult
init {
viewModelScope.launch(Dispatchers.Default) {
val messageRecord = SignalDatabase.messages.getMessageRecord(messageId)
val attachments = SignalDatabase.attachments.getAttachmentsForMessage(messageId)
refresh()
}
_state.value = ViewState(
id = messageRecord.id,
sentTimestamp = messageRecord.dateSent,
receivedTimestamp = messageRecord.dateReceived,
serverSentTimestamp = messageRecord.serverTimestamp,
from = messageRecord.fromRecipient.id,
to = messageRecord.toRecipient.id,
attachments = attachments.map { attachment ->
val info = SignalDatabase.attachments.getDataFileInfo(attachment.attachmentId)
fun consumeActionResult() {
_actionResult.value = null
}
AttachmentInfo(
id = attachment.attachmentId.id,
contentType = attachment.contentType,
quoteTargetContentType = attachment.quoteTargetContentType,
size = attachment.size,
fileName = attachment.fileName,
hashStart = info?.hashStart,
hashEnd = info?.hashEnd,
transformProperties = info?.transformProperties?.let { JsonUtil.toJson(it) } ?: "null"
)
}
)
/**
* Puts the attachment into the offloaded state, so the media is only on the archive CDN.
*/
fun offloadLocalData(attachmentId: Long) {
runAction("Offloaded local data") {
SignalDatabase.attachments.debugOffloadAttachment(AttachmentId(attachmentId))
}
}
/**
* Deletes the archive CDN copy while leaving every local record alone, which is the state a reconciliation is meant to detect and repair.
*/
fun deleteFromCdn(attachmentId: Long, thumbnail: Boolean) {
val label = if (thumbnail) "Deleted thumbnail from CDN" else "Deleted full-size from CDN"
runAction(label) {
val attachment = SignalDatabase.attachments.getAttachment(AttachmentId(attachmentId)) ?: error("Attachment is gone")
val plaintextHash = attachment.dataHash?.let { Base64.decode(it) } ?: error("No plaintext hash")
val remoteKey = attachment.remoteKey?.let { Base64.decode(it) } ?: error("No remote key")
val cdn = attachment.archiveCdn ?: error("No archive CDN, so nothing is up there")
val mediaName = if (thumbnail) {
MediaName.fromPlaintextHashAndRemoteKeyForThumbnail(plaintextHash, remoteKey)
} else {
MediaName.fromPlaintextHashAndRemoteKey(plaintextHash, remoteKey)
}
val mediaObject = ArchivedMediaObject(
mediaId = mediaName.toMediaId(SignalStore.backup.mediaRootBackupKey).encode(),
cdn = cdn
)
val failure = ArchiveCommitAttachmentDeletesJob.deleteMediaObjectsFromCdn(
tag = TAG,
attachmentsToDelete = setOf(mediaObject),
backoffGenerator = { 0 },
cancellationSignal = { false }
)
if (failure != null) {
error("CDN delete did not succeed: $failure")
}
}
}
private fun runAction(successLabel: String, action: suspend () -> Unit) {
viewModelScope.launch(Dispatchers.IO) {
_actionResult.value = try {
action()
successLabel
} catch (e: Exception) {
Log.w(TAG, "Internal attachment action failed.", e)
"Failed: ${e.message}"
}
loadState()
}
}
private fun refresh() {
viewModelScope.launch(Dispatchers.IO) {
loadState()
}
}
private fun loadState() {
val messageRecord = SignalDatabase.messages.getMessageRecord(messageId)
val attachments = SignalDatabase.attachments.getAttachmentsForMessage(messageId)
_state.value = ViewState(
id = messageRecord.id,
sentTimestamp = messageRecord.dateSent,
receivedTimestamp = messageRecord.dateReceived,
serverSentTimestamp = messageRecord.serverTimestamp,
from = messageRecord.fromRecipient.id,
to = messageRecord.toRecipient.id,
attachments = attachments.map { attachment ->
val info = SignalDatabase.attachments.getDataFileInfo(attachment.attachmentId)
AttachmentInfo(
id = attachment.attachmentId.id,
contentType = attachment.contentType,
quoteTargetContentType = attachment.quoteTargetContentType,
size = attachment.size,
fileName = attachment.fileName,
hashStart = info?.hashStart,
hashEnd = info?.hashEnd,
transformProperties = info?.transformProperties?.let { JsonUtil.toJson(it) } ?: "null",
hasLocalData = attachment.hasData,
transferState = attachment.transferState,
archiveCdn = attachment.archiveCdn,
archiveTransferState = attachment.archiveTransferState,
archiveThumbnailTransferState = attachment.archiveThumbnailTransferState
)
}
)
}
data class ViewState(
val id: Long,
val sentTimestamp: Long,
@@ -69,6 +160,11 @@ class InternalMessageDetailsViewModel(val messageId: Long) : ViewModel() {
val fileName: String?,
val hashStart: String?,
val hashEnd: String?,
val transformProperties: String?
val transformProperties: String?,
val hasLocalData: Boolean,
val transferState: Int,
val archiveCdn: Int?,
val archiveTransferState: AttachmentTable.ArchiveTransferState,
val archiveThumbnailTransferState: AttachmentTable.ArchiveTransferState
)
}
@@ -123,6 +123,7 @@ class DatabaseAttachmentArchiveUtilTest {
archiveCdn = archiveCdn,
thumbnailRestoreState = AttachmentTable.ThumbnailRestoreState.NONE,
archiveTransferState = AttachmentTable.ArchiveTransferState.FINISHED,
archiveThumbnailTransferState = AttachmentTable.ArchiveTransferState.NONE,
uuid = null,
quoteTargetContentType = null,
metadata = null
@@ -850,6 +850,7 @@ class IndividualSettingsViewModelTest {
archiveCdn = null,
thumbnailRestoreState = AttachmentTable.ThumbnailRestoreState.NONE,
archiveTransferState = AttachmentTable.ArchiveTransferState.NONE,
archiveThumbnailTransferState = AttachmentTable.ArchiveTransferState.NONE,
uuid = null,
quoteTargetContentType = null,
metadata = null
@@ -351,6 +351,7 @@ class TransferControlsTest {
archiveCdn = null,
thumbnailRestoreState = AttachmentTable.ThumbnailRestoreState.NONE,
archiveTransferState = AttachmentTable.ArchiveTransferState.NONE,
archiveThumbnailTransferState = AttachmentTable.ArchiveTransferState.NONE,
uuid = null,
quoteTargetContentType = null,
metadata = null
@@ -7,6 +7,8 @@ package org.thoughtcrime.securesms.database
import android.app.Application
import assertk.assertThat
import assertk.assertions.containsExactly
import assertk.assertions.isEmpty
import assertk.assertions.isEqualTo
import assertk.assertions.isFalse
import assertk.assertions.isNull
@@ -18,7 +20,6 @@ import org.junit.runner.RunWith
import org.robolectric.RobolectricTestRunner
import org.robolectric.annotation.Config
import org.signal.core.models.database.AttachmentId
import org.signal.core.util.Base64
import org.signal.core.util.logging.Log
import org.thoughtcrime.securesms.attachments.ArchivedAttachment
import org.thoughtcrime.securesms.attachments.Attachment
@@ -96,12 +97,11 @@ class AttachmentTableTest_localRestoreArchiveState {
val attachmentId = insertArchivedAttachment(archiveCdn = 3, localBackupKey = Random.nextBytes(32))
val attachment = SignalDatabase.attachments.getAttachment(attachmentId)!!
val result = SignalDatabase.attachments.resetArchiveTransferStateByPlaintextHashAndRemoteKeyIfNecessary(
plaintextHash = Base64.decode(attachment.dataHash!!),
remoteKey = Base64.decode(attachment.remoteKey!!)
val results = SignalDatabase.attachments.resetArchiveTransferStateByPlaintextHashAndRemoteKeyIfNecessary(
listOf(AttachmentTable.MediaNameParts(plaintextHash = attachment.dataHash!!, remoteKey = attachment.remoteKey!!))
)
assertThat(result).isEqualTo(AttachmentTable.ArchiveTransferStateResetResult.SKIPPED_NO_LOCAL_DATA)
assertThat(results).containsExactly(AttachmentTable.ArchiveTransferStateResetResult.SKIPPED_NO_LOCAL_DATA)
val after = SignalDatabase.attachments.getAttachment(attachmentId)!!
assertThat(after.archiveTransferState).isEqualTo(AttachmentTable.ArchiveTransferState.FINISHED)
@@ -113,12 +113,81 @@ class AttachmentTableTest_localRestoreArchiveState {
val attachmentId = insertArchivedAttachment(archiveCdn = null, localBackupKey = Random.nextBytes(32))
val attachment = SignalDatabase.attachments.getAttachment(attachmentId)!!
val result = SignalDatabase.attachments.resetArchiveTransferStateByPlaintextHashAndRemoteKeyIfNecessary(
plaintextHash = Base64.decode(attachment.dataHash!!),
remoteKey = Base64.decode(attachment.remoteKey!!)
val results = SignalDatabase.attachments.resetArchiveTransferStateByPlaintextHashAndRemoteKeyIfNecessary(
listOf(AttachmentTable.MediaNameParts(plaintextHash = attachment.dataHash!!, remoteKey = attachment.remoteKey!!))
)
assertThat(result).isEqualTo(AttachmentTable.ArchiveTransferStateResetResult.NOT_NEEDED)
assertThat(results).containsExactly(AttachmentTable.ArchiveTransferStateResetResult.NOT_NEEDED)
}
@Test
fun resetArchiveTransferState_batchedResultsLineUpWithTheirKeys() {
val finishedNoLocalData = SignalDatabase.attachments.getAttachment(insertArchivedAttachment(archiveCdn = 3, localBackupKey = Random.nextBytes(32)))!!
val notFinished = SignalDatabase.attachments.getAttachment(insertArchivedAttachment(archiveCdn = null, localBackupKey = Random.nextBytes(32)))!!
val results = SignalDatabase.attachments.resetArchiveTransferStateByPlaintextHashAndRemoteKeyIfNecessary(
listOf(
AttachmentTable.MediaNameParts.fromBytes(plaintextHash = Random.nextBytes(8), remoteKey = Random.nextBytes(8)),
AttachmentTable.MediaNameParts(plaintextHash = finishedNoLocalData.dataHash!!, remoteKey = finishedNoLocalData.remoteKey!!),
AttachmentTable.MediaNameParts(plaintextHash = notFinished.dataHash!!, remoteKey = notFinished.remoteKey!!)
)
)
assertThat(results).containsExactly(
AttachmentTable.ArchiveTransferStateResetResult.NOT_NEEDED,
AttachmentTable.ArchiveTransferStateResetResult.SKIPPED_NO_LOCAL_DATA,
AttachmentTable.ArchiveTransferStateResetResult.NOT_NEEDED
)
}
@Test
fun resetArchiveThumbnailTransferState_marksUnrecoverableWhenThereIsNoLocalDataFileToRebuildFrom() {
val attachmentId = insertArchivedAttachment(archiveCdn = 3, localBackupKey = Random.nextBytes(32))
val attachment = SignalDatabase.attachments.getAttachment(attachmentId)!!
SignalDatabase.attachments.setArchiveThumbnailTransferState(attachmentId, AttachmentTable.ArchiveTransferState.FINISHED)
val results = SignalDatabase.attachments.resetArchiveThumbnailTransferStateByPlaintextHashAndRemoteKeyIfNecessary(
listOf(AttachmentTable.MediaNameParts(plaintextHash = attachment.dataHash!!, remoteKey = attachment.remoteKey!!))
)
assertThat(results).containsExactly(AttachmentTable.ArchiveTransferStateResetResult.MARKED_UNRECOVERABLE)
assertThat(SignalDatabase.attachments.getArchiveThumbnailTransferState(attachmentId)).isEqualTo(AttachmentTable.ArchiveTransferState.PERMANENT_FAILURE)
// The full-size locator is deliberately left intact, since that media is still on the CDN.
val after = SignalDatabase.attachments.getAttachment(attachmentId)!!
assertThat(after.archiveTransferState).isEqualTo(AttachmentTable.ArchiveTransferState.FINISHED)
assertThat(after.archiveCdn).isEqualTo(3)
}
@Test
fun resetArchiveThumbnailTransferState_reportsAlreadyRecordedMediaAsUnrecoverableRatherThanNotNeeded() {
val attachmentId = insertArchivedAttachment(archiveCdn = 3, localBackupKey = Random.nextBytes(32))
val attachment = SignalDatabase.attachments.getAttachment(attachmentId)!!
SignalDatabase.attachments.setArchiveThumbnailTransferState(attachmentId, AttachmentTable.ArchiveTransferState.FINISHED)
val mediaNames = listOf(
AttachmentTable.MediaNameParts(plaintextHash = attachment.dataHash!!, remoteKey = attachment.remoteKey!!)
)
assertThat(SignalDatabase.attachments.resetArchiveThumbnailTransferStateByPlaintextHashAndRemoteKeyIfNecessary(mediaNames))
.containsExactly(AttachmentTable.ArchiveTransferStateResetResult.MARKED_UNRECOVERABLE)
assertThat(SignalDatabase.attachments.resetArchiveThumbnailTransferStateByPlaintextHashAndRemoteKeyIfNecessary(mediaNames))
.containsExactly(AttachmentTable.ArchiveTransferStateResetResult.MARKED_UNRECOVERABLE)
}
@Test(expected = IllegalArgumentException::class)
fun resetArchiveTransferState_rejectsABatchTooLargeForOneStatement() {
val tooMany = List(AttachmentTable.ARCHIVE_MEDIA_KEY_BATCH_SIZE + 1) {
AttachmentTable.MediaNameParts.fromBytes(plaintextHash = Random.nextBytes(8), remoteKey = Random.nextBytes(8))
}
SignalDatabase.attachments.resetArchiveTransferStateByPlaintextHashAndRemoteKeyIfNecessary(tooMany)
}
@Test
fun resetArchiveTransferState_noResultsForAnEmptyBatch() {
assertThat(SignalDatabase.attachments.resetArchiveThumbnailTransferStateByPlaintextHashAndRemoteKeyIfNecessary(emptyList())).isEmpty()
}
private fun insertArchivedAttachment(archiveCdn: Int?, localBackupKey: ByteArray?): AttachmentId {
@@ -134,6 +134,7 @@ class AttachmentBackfillTest {
archiveCdn = null,
thumbnailRestoreState = AttachmentTable.ThumbnailRestoreState.NONE,
archiveTransferState = AttachmentTable.ArchiveTransferState.NONE,
archiveThumbnailTransferState = AttachmentTable.ArchiveTransferState.NONE,
uuid = null,
quoteTargetContentType = null,
metadata = null
@@ -256,6 +256,7 @@ class UploadDependencyGraphTest {
archiveCdn = 0,
thumbnailRestoreState = AttachmentTable.ThumbnailRestoreState.NONE,
archiveTransferState = AttachmentTable.ArchiveTransferState.NONE,
archiveThumbnailTransferState = AttachmentTable.ArchiveTransferState.NONE,
uuid = null,
quoteTargetContentType = null,
metadata = null
@@ -491,6 +491,7 @@ class AttachmentUtilTest {
archiveCdn = null,
thumbnailRestoreState = AttachmentTable.ThumbnailRestoreState.NONE,
archiveTransferState = AttachmentTable.ArchiveTransferState.NONE,
archiveThumbnailTransferState = AttachmentTable.ArchiveTransferState.NONE,
uuid = null,
quoteTargetContentType = null,
metadata = null
@@ -68,7 +68,8 @@ object FakeMessageRecords {
archiveMediaId: String? = null,
archiveThumbnailId: String? = null,
thumbnailRestoreState: AttachmentTable.ThumbnailRestoreState = AttachmentTable.ThumbnailRestoreState.NONE,
archiveTransferState: AttachmentTable.ArchiveTransferState = AttachmentTable.ArchiveTransferState.NONE
archiveTransferState: AttachmentTable.ArchiveTransferState = AttachmentTable.ArchiveTransferState.NONE,
archiveThumbnailTransferState: AttachmentTable.ArchiveTransferState = AttachmentTable.ArchiveTransferState.NONE
): DatabaseAttachment {
return DatabaseAttachment(
attachmentId = attachmentId,
@@ -103,6 +104,7 @@ object FakeMessageRecords {
archiveCdn = archiveCdn,
thumbnailRestoreState = thumbnailRestoreState,
archiveTransferState = archiveTransferState,
archiveThumbnailTransferState = archiveThumbnailTransferState,
uuid = null,
quoteTargetContentType = null,
metadata = null