diff --git a/app/src/androidTest/java/org/thoughtcrime/securesms/database/AttachmentTableTest.kt b/app/src/androidTest/java/org/thoughtcrime/securesms/database/AttachmentTableTest.kt index 63115ac0cf..2dc341dece 100644 --- a/app/src/androidTest/java/org/thoughtcrime/securesms/database/AttachmentTableTest.kt +++ b/app/src/androidTest/java/org/thoughtcrime/securesms/database/AttachmentTableTest.kt @@ -6,6 +6,7 @@ import androidx.test.ext.junit.runners.AndroidJUnit4 import androidx.test.filters.FlakyTest import androidx.test.platform.app.InstrumentationRegistry import assertk.assertThat +import assertk.assertions.containsExactly import assertk.assertions.hasSize import assertk.assertions.isEmpty import assertk.assertions.isEqualTo @@ -218,14 +219,41 @@ class AttachmentTableTest { SignalDatabase.attachments.setArchiveTransferState(attachmentId, AttachmentTable.ArchiveTransferState.FINISHED) // Reset the transfer state by plaintextHash+remoteKey - val plaintextHash = SignalDatabase.attachments.getAttachment(attachmentId)!!.dataHash!!.decodeBase64OrThrow() - val remoteKey = SignalDatabase.attachments.getAttachment(attachmentId)!!.remoteKey!!.decodeBase64OrThrow() - SignalDatabase.attachments.resetArchiveTransferStateByPlaintextHashAndRemoteKeyIfNecessary(plaintextHash, remoteKey) + val inserted = SignalDatabase.attachments.getAttachment(attachmentId)!! + SignalDatabase.attachments.resetArchiveTransferStateByPlaintextHashAndRemoteKeyIfNecessary( + listOf(AttachmentTable.MediaNameParts(plaintextHash = inserted.dataHash!!, remoteKey = inserted.remoteKey!!)) + ) // Verify it's been reset assertThat(SignalDatabase.attachments.getAttachment(attachmentId)!!.archiveTransferState).isEqualTo(AttachmentTable.ArchiveTransferState.NONE) } + @Test + fun resetArchiveTransferStateByPlaintextHashAndRemoteKey_batchedMatch() { + // Given two archive-finished attachments that still have their local data + val attachmentIds = listOf(byteArrayOf(1, 2, 3, 4, 5), byteArrayOf(6, 7, 8, 9, 10)).mapIndexed { index, data -> + val blob = AppDependencies.blobs.forData(data).createForSingleSessionInMemory() + val attachment = createAttachment(index + 1L, blob, TransformProperties.empty()) + val attachmentId = SignalDatabase.attachments.insertAttachmentsForMessage(-1L, listOf(attachment), emptyList()).values.first() + SignalDatabase.attachments.finalizeAttachmentAfterUpload(attachmentId, AttachmentTableTestUtil.createUploadResult(attachmentId)) + SignalDatabase.attachments.setArchiveTransferState(attachmentId, AttachmentTable.ArchiveTransferState.FINISHED) + attachmentId + } + + val mediaNames = attachmentIds.map { attachmentId -> + val attachment = SignalDatabase.attachments.getAttachment(attachmentId)!! + AttachmentTable.MediaNameParts(plaintextHash = attachment.dataHash!!, remoteKey = attachment.remoteKey!!) + } + + val results = SignalDatabase.attachments.resetArchiveTransferStateByPlaintextHashAndRemoteKeyIfNecessary(mediaNames) + + // Both are reported as reset, and both actually are + assertThat(results).containsExactly(AttachmentTable.ArchiveTransferStateResetResult.RESET, AttachmentTable.ArchiveTransferStateResetResult.RESET) + attachmentIds.forEach { attachmentId -> + assertThat(SignalDatabase.attachments.getAttachment(attachmentId)!!.archiveTransferState).isEqualTo(AttachmentTable.ArchiveTransferState.NONE) + } + } + @Test fun resetArchiveTransferStateForLocalBackupMedia_onlyResetsLocalBackupMedia() { // Given one archive-finished attachment restored from a local backup, and one that wasn't @@ -462,6 +490,55 @@ class AttachmentTableTest { assertThat(result.archiveCdn).isNull() } + @Test + fun givenPermanentlyFailedThumbnail_whenIFinalizeAttachment_thenIExpectThumbnailStateNoneSoItCanBeArchivedAgain() { + val data = byteArrayOf(1, 2, 3, 4, 5) + val attachment = createAttachmentPointer("remote-key-1".toByteArray(), data.size) + + val messageResult = SignalDatabase.messages.insertMessageInbox(createIncomingMessage(serverTime = 0.days, attachment = attachment)).get() + val attachmentId = messageResult.insertedAttachments!![attachment]!! + SignalDatabase.attachments.setTransferState(messageResult.messageId, attachmentId, AttachmentTable.TRANSFER_PROGRESS_STARTED) + + setDataHashEnd(attachmentId, "kMDoNIQjPGiuKzZHUCcLBiIF5wsvE7dqLKvvKG1JLmc=") + SignalDatabase.attachments.setArchiveThumbnailTransferState(attachmentId, AttachmentTable.ArchiveTransferState.PERMANENT_FAILURE) + + SignalDatabase.attachments.finalizeAttachmentAfterDownload(messageResult.messageId, attachmentId, ByteArrayInputStream(data), archiveRestore = true, restoredFromArchiveCdn = true) + + // Local bytes are back, so the thumbnail is eligible for archiving again + assertThat(SignalDatabase.attachments.getArchiveThumbnailTransferState(attachmentId)).isEqualTo(AttachmentTable.ArchiveTransferState.NONE) + } + + @Test + fun givenPermanentlyFailedThumbnail_whenIFinalizeAnOrdinaryDownload_thenIExpectTheMarkToSurvive() { + val data = byteArrayOf(1, 2, 3, 4, 5) + val attachment = createAttachmentPointer("remote-key-1".toByteArray(), data.size) + + val messageResult = SignalDatabase.messages.insertMessageInbox(createIncomingMessage(serverTime = 0.days, attachment = attachment)).get() + val attachmentId = messageResult.insertedAttachments!![attachment]!! + SignalDatabase.attachments.setTransferState(messageResult.messageId, attachmentId, AttachmentTable.TRANSFER_PROGRESS_STARTED) + SignalDatabase.attachments.setArchiveThumbnailTransferState(attachmentId, AttachmentTable.ArchiveTransferState.PERMANENT_FAILURE) + + SignalDatabase.attachments.finalizeAttachmentAfterDownload(messageResult.messageId, attachmentId, ByteArrayInputStream(data)) + + assertThat(SignalDatabase.attachments.getArchiveThumbnailTransferState(attachmentId)).isEqualTo(AttachmentTable.ArchiveTransferState.PERMANENT_FAILURE) + } + + @Test + fun givenFinishedThumbnail_whenIFinalizeAttachment_thenIExpectThumbnailStateUntouched() { + val data = byteArrayOf(1, 2, 3, 4, 5) + val attachment = createAttachmentPointer("remote-key-1".toByteArray(), data.size) + + val messageResult = SignalDatabase.messages.insertMessageInbox(createIncomingMessage(serverTime = 0.days, attachment = attachment)).get() + val attachmentId = messageResult.insertedAttachments!![attachment]!! + SignalDatabase.attachments.setTransferState(messageResult.messageId, attachmentId, AttachmentTable.TRANSFER_PROGRESS_STARTED) + SignalDatabase.attachments.setArchiveThumbnailTransferState(attachmentId, AttachmentTable.ArchiveTransferState.FINISHED) + + SignalDatabase.attachments.finalizeAttachmentAfterDownload(messageResult.messageId, attachmentId, ByteArrayInputStream(data), archiveRestore = true, restoredFromArchiveCdn = true) + + // Only PERMANENT_FAILURE is lifted, an already-archived thumbnail keeps its state + assertThat(SignalDatabase.attachments.getArchiveThumbnailTransferState(attachmentId)).isEqualTo(AttachmentTable.ArchiveTransferState.FINISHED) + } + @Test fun givenArchiveCdnRestore_whenIFinalizeAttachment_thenIExpectArchiveStateFinished() { val data = byteArrayOf(1, 2, 3, 4, 5) @@ -738,6 +815,14 @@ class AttachmentTableTest { .run() } + private fun setDataHashEnd(attachmentId: AttachmentId, hashEnd: String) { + SignalDatabase.rawDatabase + .update(AttachmentTable.TABLE_NAME) + .values(AttachmentTable.DATA_HASH_END to hashEnd) + .where("${AttachmentTable.ID} = ?", attachmentId.id) + .run() + } + /** Defaults to a non-media content type, which keeps the candidate eligible without also having to generate a thumbnail file. */ private fun seedOffloadCandidate(contentType: String = "application/pdf"): AttachmentId { val data = byteArrayOf(1, 2, 3, 4, 5) diff --git a/app/src/androidTest/java/org/thoughtcrime/securesms/messages/SyncMessageProcessorTest_synchronizeDeleteForMe.kt b/app/src/androidTest/java/org/thoughtcrime/securesms/messages/SyncMessageProcessorTest_synchronizeDeleteForMe.kt index 029c657b7b..04caee40e9 100644 --- a/app/src/androidTest/java/org/thoughtcrime/securesms/messages/SyncMessageProcessorTest_synchronizeDeleteForMe.kt +++ b/app/src/androidTest/java/org/thoughtcrime/securesms/messages/SyncMessageProcessorTest_synchronizeDeleteForMe.kt @@ -704,6 +704,7 @@ class SyncMessageProcessorTest_synchronizeDeleteForMe { archiveCdn = this.archiveCdn, thumbnailRestoreState = this.thumbnailRestoreState, archiveTransferState = this.archiveTransferState, + archiveThumbnailTransferState = this.archiveThumbnailTransferState, uuid = uuid, quoteTargetContentType = this.quoteTargetContentType, metadata = null diff --git a/app/src/main/java/org/thoughtcrime/securesms/attachments/DatabaseAttachment.kt b/app/src/main/java/org/thoughtcrime/securesms/attachments/DatabaseAttachment.kt index 4234631367..c0d5239538 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/attachments/DatabaseAttachment.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/attachments/DatabaseAttachment.kt @@ -40,6 +40,9 @@ class DatabaseAttachment : Attachment { @JvmField val archiveTransferState: AttachmentTable.ArchiveTransferState + @JvmField + val archiveThumbnailTransferState: AttachmentTable.ArchiveTransferState + /** Metadata for this attachment, if null, no attempt was made to load the metadata and does not imply there is none */ @JvmField val metadata: AttachmentMetadata? @@ -80,6 +83,7 @@ class DatabaseAttachment : Attachment { archiveCdn: Int?, thumbnailRestoreState: AttachmentTable.ThumbnailRestoreState, archiveTransferState: AttachmentTable.ArchiveTransferState, + archiveThumbnailTransferState: AttachmentTable.ArchiveTransferState, uuid: UUID?, quoteTargetContentType: String?, metadata: AttachmentMetadata? @@ -118,6 +122,7 @@ class DatabaseAttachment : Attachment { this.archiveCdn = archiveCdn this.thumbnailRestoreState = thumbnailRestoreState this.archiveTransferState = archiveTransferState + this.archiveThumbnailTransferState = archiveThumbnailTransferState this.metadata = metadata } @@ -131,6 +136,7 @@ class DatabaseAttachment : Attachment { archiveCdn = parcel.readInt().takeIf { it != NO_ARCHIVE_CDN } thumbnailRestoreState = AttachmentTable.ThumbnailRestoreState.deserialize(parcel.readInt()) archiveTransferState = AttachmentTable.ArchiveTransferState.deserialize(parcel.readInt()) + archiveThumbnailTransferState = AttachmentTable.ArchiveTransferState.deserialize(parcel.readInt()) metadata = ParcelCompat.readParcelable(parcel, AttachmentMetadata::class.java.classLoader, AttachmentMetadata::class.java) } @@ -145,6 +151,7 @@ class DatabaseAttachment : Attachment { dest.writeInt(archiveCdn ?: NO_ARCHIVE_CDN) dest.writeInt(thumbnailRestoreState.value) dest.writeInt(archiveTransferState.value) + dest.writeInt(archiveThumbnailTransferState.value) dest.writeParcelable(metadata, 0) } diff --git a/app/src/main/java/org/thoughtcrime/securesms/database/AttachmentTable.kt b/app/src/main/java/org/thoughtcrime/securesms/database/AttachmentTable.kt index ca9f723c41..1e92473dd9 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/database/AttachmentTable.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/database/AttachmentTable.kt @@ -231,6 +231,7 @@ class AttachmentTable( THUMBNAIL_FILE, THUMBNAIL_RESTORE_STATE, ARCHIVE_TRANSFER_STATE, + ARCHIVE_THUMBNAIL_TRANSFER_STATE, ATTACHMENT_UUID ) @@ -288,6 +289,9 @@ class AttachmentTable( private const val DATA_FILE_INDEX = "attachment_data_index" private const val DATA_HASH_REMOTE_KEY_INDEX = "attachment_data_hash_end_remote_key_index" + /** How many media names fit in one statement. Each binds two args, so this has to stay under half of [SqlUtil.MAX_QUERY_ARGS]. */ + const val ARCHIVE_MEDIA_KEY_BATCH_SIZE = 400 + @JvmField val CREATE_INDEXS = arrayOf( "CREATE INDEX IF NOT EXISTS attachment_message_id_index ON $TABLE_NAME ($MESSAGE_ID);", @@ -1150,65 +1154,167 @@ class AttachmentTable( } /** - * Resets the archive upload state by hash/key if we believe the attachment should have been uploaded already. + * Internal-only. Drops the local bytes for a single attachment exactly the way [markEligibleAttachmentsAsOptimized] does, bypassing every eligibility rule so a + * specific attachment can be put into the offloaded state on demand. */ - fun resetArchiveTransferStateByPlaintextHashAndRemoteKeyIfNecessary(plaintextHash: ByteArray, remoteKey: ByteArray): ArchiveTransferStateResetResult { + fun debugOffloadAttachment(attachmentId: AttachmentId): Int { + return writableDatabase + .update(TABLE_NAME) + .values( + TRANSFER_STATE to TRANSFER_RESTORE_OFFLOADED, + DATA_FILE to null, + DATA_RANDOM to null, + TRANSFORM_PROPERTIES to null, + DATA_HASH_START to null, + OFFLOAD_RESTORED_AT to 0 + ) + .where("$ID = ?", attachmentId.id) + .run() + .also { AppDependencies.databaseObserver.notifyAttachmentUpdatedObservers() } + } + + /** + * Resets the archive upload state by hash/key for any media we believe should have been uploaded already. Accepts at most + * [ARCHIVE_MEDIA_KEY_BATCH_SIZE] names, since they all go into one statement. + */ + fun resetArchiveTransferStateByPlaintextHashAndRemoteKeyIfNecessary(mediaNames: List): List { return resetArchiveTransferStateIfNecessary( - plaintextHash = plaintextHash, - remoteKey = remoteKey, + mediaNames = mediaNames, stateColumn = ARCHIVE_TRANSFER_STATE, values = contentValuesOf( ARCHIVE_TRANSFER_STATE to ArchiveTransferState.NONE.value, ARCHIVE_CDN to null - ) + ), + unrecoverableState = null ) } /** - * Resets the archive thumbnail upload state by hash/key if we believe the thumbnail should have been uploaded already. + * Resets the archive thumbnail upload state by hash/key for any thumbnail we believe should have been uploaded already. Accepts at most + * [ARCHIVE_MEDIA_KEY_BATCH_SIZE] names, since they all go into one statement. + * + * Unlike full-size media, a thumbnail with no local bytes is marked [ArchiveTransferState.PERMANENT_FAILURE] rather than left alone. The next + * backup stops tracking it and later crawls stop rediscovering something they can never repair. Restoring the full-size media lifts the mark. */ - fun resetArchiveThumbnailTransferStateByPlaintextHashAndRemoteKeyIfNecessary(plaintextHash: ByteArray, remoteKey: ByteArray): ArchiveTransferStateResetResult { + fun resetArchiveThumbnailTransferStateByPlaintextHashAndRemoteKeyIfNecessary(mediaNames: List): List { return resetArchiveTransferStateIfNecessary( - plaintextHash = plaintextHash, - remoteKey = remoteKey, + mediaNames = mediaNames, stateColumn = ARCHIVE_THUMBNAIL_TRANSFER_STATE, - values = contentValuesOf(ARCHIVE_THUMBNAIL_TRANSFER_STATE to ArchiveTransferState.NONE.value) + values = contentValuesOf(ARCHIVE_THUMBNAIL_TRANSFER_STATE to ArchiveTransferState.NONE.value), + unrecoverableState = ArchiveTransferState.PERMANENT_FAILURE ) } - private fun resetArchiveTransferStateIfNecessary(plaintextHash: ByteArray, remoteKey: ByteArray, stateColumn: String, values: ContentValues): ArchiveTransferStateResetResult { - val encodedHash = Base64.encodeWithPadding(plaintextHash) - val encodedKey = Base64.encodeWithPadding(remoteKey) - val finishedClause = "$DATA_HASH_END = ? AND $REMOTE_KEY = ? AND $stateColumn = ${ArchiveTransferState.FINISHED.value}" + /** + * For each of the given [mediaNames], clears [stateColumn] if we believe the media was already uploaded but something has told us it isn't there. + */ + private fun resetArchiveTransferStateIfNecessary( + mediaNames: List, + stateColumn: String, + values: ContentValues, + unrecoverableState: ArchiveTransferState? + ): List { + require(mediaNames.size <= ARCHIVE_MEDIA_KEY_BATCH_SIZE) { "Batch of ${mediaNames.size} exceeds what fits in one statement." } - return writableDatabase.withinTransaction { db -> - val isFinished = db - .exists(TABLE_NAME) - .where(finishedClause, encodedHash, encodedKey) - .run() - - if (!isFinished) { - return@withinTransaction ArchiveTransferStateResetResult.NOT_NEEDED - } - - val hasLocalData = db - .exists(TABLE_NAME) - .where("$DATA_HASH_END = ? AND $REMOTE_KEY = ? AND $DATA_FILE NOT NULL", encodedHash, encodedKey) - .run() - - // With no local data file there is nothing to re-upload, and clearing the state makes the export emit a tombstone instead of a locator, which in turn marks - // the media for deletion off of the CDN. - if (!hasLocalData) { - return@withinTransaction ArchiveTransferStateResetResult.SKIPPED_NO_LOCAL_DATA - } - - db.update(TABLE_NAME) - .values(values) - .where(finishedClause, encodedHash, encodedKey) - .run() - - ArchiveTransferStateResetResult.RESET + if (mediaNames.isEmpty()) { + return emptyList() } + + val hasLocalDataByMediaName: Map = getArchiveFinishedMediaWithLocalData(mediaNames, stateColumn) + + // Without this, already-recorded media looks like gone-or-in-progress and re-arms a backfill that can never upload it. + val alreadyRecorded: Set = if (unrecoverableState != null) { + getMatchingMediaNames(mediaNames, "$stateColumn = ${unrecoverableState.value}") + } else { + emptySet() + } + + val recoverable = mutableListOf() + val unrecoverable = mutableListOf() + + val results = mediaNames.map { mediaName -> + when (hasLocalDataByMediaName[mediaName]) { + null -> if (mediaName in alreadyRecorded) { + ArchiveTransferStateResetResult.MARKED_UNRECOVERABLE + } else { + ArchiveTransferStateResetResult.NOT_NEEDED + } + + true -> { + recoverable += mediaName + ArchiveTransferStateResetResult.RESET + } + + false -> if (unrecoverableState != null) { + unrecoverable += mediaName + ArchiveTransferStateResetResult.MARKED_UNRECOVERABLE + } else { + ArchiveTransferStateResetResult.SKIPPED_NO_LOCAL_DATA + } + } + } + + if (recoverable.isNotEmpty()) { + updateArchiveFinishedMedia(recoverable, stateColumn, values, "$DATA_FILE NOT NULL") + } + + if (unrecoverable.isNotEmpty() && unrecoverableState != null) { + updateArchiveFinishedMedia(unrecoverable, stateColumn, contentValuesOf(stateColumn to unrecoverableState.value), "$DATA_FILE IS NULL") + } + + return results + } + + /** + * Given [mediaNames], the ones we believe are finished, mapped to whether we still have local bytes. + */ + private fun getArchiveFinishedMediaWithLocalData(mediaNames: List, stateColumn: String): Map { + val archived = getMatchingMediaNames(mediaNames, "$stateColumn = ${ArchiveTransferState.FINISHED.value}") + if (archived.isEmpty()) { + return emptyMap() + } + + val withLocalData = getMatchingMediaNames(archived.toList(), "$DATA_FILE NOT NULL") + + return archived.associateWith { it in withLocalData } + } + + /** + * The subset of [mediaNames] that has at least one attachment satisfying [clause]. + */ + private fun getMatchingMediaNames(mediaNames: List, clause: String): Set { + val tuples = mediaNameTuples(mediaNames) + + return readableDatabase + .select(DATA_HASH_END, REMOTE_KEY) + .from("$TABLE_NAME INDEXED BY $DATA_HASH_REMOTE_KEY_INDEX") + .where("$clause AND ${tuples.where}", tuples.whereArgs) + .run() + .readToSet { cursor -> + MediaNameParts( + plaintextHash = cursor.requireNonNullString(DATA_HASH_END), + remoteKey = cursor.requireNonNullString(REMOTE_KEY) + ) + } + } + + /** [localDataClause] re-asserts the unlocked lookup's conclusion, so a concurrent restore or offload can't be written over. */ + private fun updateArchiveFinishedMedia(mediaNames: List, stateColumn: String, values: ContentValues, localDataClause: String): Int { + val tuples = mediaNameTuples(mediaNames) + + return writableDatabase + .update("$TABLE_NAME INDEXED BY $DATA_HASH_REMOTE_KEY_INDEX") + .values(values) + .where("$stateColumn = ${ArchiveTransferState.FINISHED.value} AND $localDataClause AND ${tuples.where}", tuples.whereArgs) + .run() + } + + /** Matches the hash/key pair of any of [mediaNames]. */ + private fun mediaNameTuples(mediaNames: List): SqlUtil.Query { + return SqlUtil.Query( + where = "($DATA_HASH_END, $REMOTE_KEY) IN (VALUES ${mediaNames.joinToString(separator = ", ") { "(?, ?)" }})", + whereArgs = mediaNames.flatMap { listOf(it.plaintextHash, it.remoteKey) }.toTypedArray() + ) } /** @@ -2017,7 +2123,14 @@ class AttachmentTable( val dataFilePath = hashMatch?.file?.absolutePath ?: fileWriteResult.file.absolutePath - val updateCount = if (archiveRestore && existingPlaceholder.dataHash != null) { + // Only this branch matches purely by hash/key, which is the granularity the mark is applied at. + val restoringByHashAndKey = archiveRestore && existingPlaceholder.dataHash != null + + if (restoringByHashAndKey && existingPlaceholder.archiveThumbnailTransferState == ArchiveTransferState.PERMANENT_FAILURE) { + values.put(ARCHIVE_THUMBNAIL_TRANSFER_STATE, ArchiveTransferState.NONE.value) + } + + val updateCount = if (restoringByHashAndKey) { // Can update all rows with the same mediaName as data_file column will likely be null db.update(TABLE_NAME) .values(values) @@ -3744,6 +3857,7 @@ class AttachmentTable( archiveCdn = cursor.requireIntOrNull(ARCHIVE_CDN), thumbnailRestoreState = ThumbnailRestoreState.deserialize(cursor.requireInt(THUMBNAIL_RESTORE_STATE)), archiveTransferState = ArchiveTransferState.deserialize(cursor.requireInt(ARCHIVE_TRANSFER_STATE)), + archiveThumbnailTransferState = ArchiveTransferState.deserialize(cursor.requireInt(ARCHIVE_THUMBNAIL_TRANSFER_STATE)), uuid = UuidUtil.parseOrNull(cursor.requireString(ATTACHMENT_UUID)), metadata = AttachmentMetadataTable.getMetadata(cursor) ) @@ -4502,7 +4616,16 @@ class AttachmentTable( /** * The base64 [DATA_HASH_END] and [REMOTE_KEY] that a [MediaName] is derived from. */ - data class MediaNameParts(val plaintextHash: String, val remoteKey: String) + data class MediaNameParts(val plaintextHash: String, val remoteKey: String) { + companion object { + fun fromBytes(plaintextHash: ByteArray, remoteKey: ByteArray): MediaNameParts { + return MediaNameParts( + plaintextHash = Base64.encodeWithPadding(plaintextHash), + remoteKey = Base64.encodeWithPadding(remoteKey) + ) + } + } + } enum class ArchiveTransferStateResetResult { /** We cleared the state, so the media will be re-uploaded. */ @@ -4511,6 +4634,9 @@ class AttachmentTable( /** We left the state alone because there are no local bytes to re-upload, so the media is unrecoverable rather than merely out of date. */ SKIPPED_NO_LOCAL_DATA, + /** Like [SKIPPED_NO_LOCAL_DATA], but the absence is recorded so it stops being tracked and rediscovered. Covers media already recorded by an earlier pass. */ + MARKED_UNRECOVERABLE, + /** There was nothing to clear. The attachment is gone, or an upload is already underway. */ NOT_NEEDED } diff --git a/app/src/main/java/org/thoughtcrime/securesms/database/MediaTable.kt b/app/src/main/java/org/thoughtcrime/securesms/database/MediaTable.kt index 751824a247..6dfd087030 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/database/MediaTable.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/database/MediaTable.kt @@ -58,6 +58,7 @@ class MediaTable internal constructor(context: Context?, databaseHelper: SignalD ${AttachmentTable.TABLE_NAME}.${AttachmentTable.ARCHIVE_CDN}, ${AttachmentTable.TABLE_NAME}.${AttachmentTable.THUMBNAIL_RESTORE_STATE}, ${AttachmentTable.TABLE_NAME}.${AttachmentTable.ARCHIVE_TRANSFER_STATE}, + ${AttachmentTable.TABLE_NAME}.${AttachmentTable.ARCHIVE_THUMBNAIL_TRANSFER_STATE}, ${AttachmentTable.TABLE_NAME}.${AttachmentTable.ATTACHMENT_UUID}, ${MessageTable.TABLE_NAME}.${MessageTable.TYPE}, ${MessageTable.TABLE_NAME}.${MessageTable.DATE_SENT}, @@ -163,6 +164,7 @@ class MediaTable internal constructor(context: Context?, databaseHelper: SignalD ${AttachmentTable.TABLE_NAME}.${AttachmentTable.ARCHIVE_CDN}, ${AttachmentTable.TABLE_NAME}.${AttachmentTable.THUMBNAIL_RESTORE_STATE}, ${AttachmentTable.TABLE_NAME}.${AttachmentTable.ARCHIVE_TRANSFER_STATE}, + ${AttachmentTable.TABLE_NAME}.${AttachmentTable.ARCHIVE_THUMBNAIL_TRANSFER_STATE}, ${AttachmentTable.TABLE_NAME}.${AttachmentTable.ATTACHMENT_UUID}, ${MessageTable.TABLE_NAME}.${MessageTable.TYPE}, ${MessageTable.TABLE_NAME}.${MessageTable.DATE_SENT}, diff --git a/app/src/main/java/org/thoughtcrime/securesms/jobs/ArchiveAttachmentReconciliationJob.kt b/app/src/main/java/org/thoughtcrime/securesms/jobs/ArchiveAttachmentReconciliationJob.kt index b731fbfef7..72e3c69885 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/jobs/ArchiveAttachmentReconciliationJob.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/jobs/ArchiveAttachmentReconciliationJob.kt @@ -28,6 +28,7 @@ import org.thoughtcrime.securesms.R import org.thoughtcrime.securesms.backup.v2.ArchivedMediaObject import org.thoughtcrime.securesms.database.AttachmentTable import org.thoughtcrime.securesms.database.AttachmentTable.ArchiveTransferStateResetResult +import org.thoughtcrime.securesms.database.AttachmentTable.MediaNameParts import org.thoughtcrime.securesms.database.BackupMediaSnapshotTable import org.thoughtcrime.securesms.database.SignalDatabase import org.thoughtcrime.securesms.dependencies.AppDependencies @@ -261,70 +262,42 @@ class ArchiveAttachmentReconciliationJob private constructor( if (mayNeedReUploadCount > 0) { Log.w(TAG, "Found $mayNeedReUploadCount attachments that are present in the target snapshot, but could not be found on the CDN. This could be a bookkeeping error, or the upload may still be in progress. Checking.", true) - var bookkeepingErrorCount = 0 - var unrecoverableCount = 0 - - var fullSizeReUploadNeeded = false - var thumbnailReUploadNeeded = false + val tally = RepairTally() + val batch = ArrayList(AttachmentTable.ARCHIVE_MEDIA_KEY_BATCH_SIZE) mediaObjectsThatMayNeedReUpload.forEach { mediaObjectCursor -> val entry = BackupMediaSnapshotTable.MediaEntry.fromCursor(mediaObjectCursor) + batch += entry if (internalUser) { mediaIdsThatNeedUpload += MediaId(entry.mediaId) } - val mediaIdLog = if (internalUser) "[${MediaId(entry.mediaId)}]" else "" - val logPrefix = if (entry.isThumbnail) "[Thumbnail]$mediaIdLog" else "[Fullsize]$mediaIdLog" - - val resetResult = if (entry.isThumbnail) { - SignalDatabase.attachments.resetArchiveThumbnailTransferStateByPlaintextHashAndRemoteKeyIfNecessary(entry.plaintextHash, entry.remoteKey) - } else { - SignalDatabase.attachments.resetArchiveTransferStateByPlaintextHashAndRemoteKeyIfNecessary(entry.plaintextHash, entry.remoteKey) - } - - when (resetResult) { - ArchiveTransferStateResetResult.RESET -> { - Log.w(TAG, "$logPrefix Reset transfer state by hash/key.", true) - bookkeepingErrorCount++ - - if (entry.isThumbnail) { - thumbnailReUploadNeeded = true - } else { - fullSizeReUploadNeeded = true - } - } - - ArchiveTransferStateResetResult.SKIPPED_NO_LOCAL_DATA -> { - if (internalUser) { - Log.w(TAG, "$logPrefix Missing from the CDN with no local data to re-upload. Leaving the archive state alone so we keep the locator.", true) - } - unrecoverableCount++ - } - - ArchiveTransferStateResetResult.NOT_NEEDED -> { - Log.i(TAG, "$logPrefix Did not need to reset the transfer state by hash/key because it either no longer exists or the upload is already in-progress.", true) - - // Deliberately not set for SKIPPED_NO_LOCAL_DATA, since the precautionary backfills these drive could never upload media that has no local bytes. - if (entry.isThumbnail) { - thumbnailReUploadNeeded = true - } else { - fullSizeReUploadNeeded = true - } - } + if (batch.size >= AttachmentTable.ARCHIVE_MEDIA_KEY_BATCH_SIZE) { + repairBatch(batch, tally, internalUser) + batch.clear() } } + repairBatch(batch, tally, internalUser) stopwatch.split("mark-reupload") - if (bookkeepingErrorCount > 0) { - Log.w(TAG, "Found that $bookkeepingErrorCount/$mayNeedReUploadCount of the CDN mismatches were bookkeeping errors.", true) + if (tally.resetCount > 0) { + Log.w(TAG, "Found that ${tally.resetCount}/$mayNeedReUploadCount of the CDN mismatches were bookkeeping errors.", true) maybePostReconciliationFailureNotification() } else { Log.i(TAG, "None of the $mayNeedReUploadCount CDN mismatches were bookkeeping errors.", true) } - if (unrecoverableCount > 0) { - Log.w(TAG, "Found that $unrecoverableCount/$mayNeedReUploadCount of the CDN mismatches have no local data to re-upload. That media is not recoverable from this device.", true) + if (tally.unrecoverableCount > 0) { + Log.w(TAG, "Found that ${tally.unrecoverableCount}/$mayNeedReUploadCount of the CDN mismatches have no local data to re-upload. That media is not recoverable from this device.", true) + } + + if (tally.markedUnrecoverableCount > 0) { + Log.w(TAG, "Marked ${tally.markedUnrecoverableCount}/$mayNeedReUploadCount of the CDN mismatches as permanently failed thumbnails, since there is no local data to rebuild them from. They will stop being tracked until the media is restored.", true) + } + + if (tally.notNeededCount > 0) { + Log.i(TAG, "Did not need to reset ${tally.notNeededCount}/$mayNeedReUploadCount of the CDN mismatches, because they either no longer exist or an upload is already in-progress.", true) } Log.d(TAG, "AFTER:\n" + SignalDatabase.attachments.debugGetAttachmentStats().shortPrettyString(), true) @@ -349,11 +322,11 @@ class ArchiveAttachmentReconciliationJob private constructor( // No backup is started here on purpose. Re-uploading is the whole repair, and [ArchiveUploadProgress] is what decides whether the resulting CDN numbers // warrant a fresh export once the backfill finishes uploading. - if (fullSizeReUploadNeeded) { + if (tally.fullSizeReUploadNeeded) { Log.d(TAG, "Full size mismatch found. Enqueuing an attachment backfill job.", true) AppDependencies.jobManager.add(ArchiveAttachmentBackfillJob()) } - if (thumbnailReUploadNeeded) { + if (tally.thumbnailReUploadNeeded) { Log.d(TAG, "Thumbnail mismatch found. Enqueuing a thumbnail backfill job.", true) AppDependencies.jobManager.add(ArchiveThumbnailBackfillJob()) } @@ -546,6 +519,54 @@ class ArchiveAttachmentReconciliationJob private constructor( return null } + /** + * Clears the archive transfer state for a batch of media objects that a crawl couldn't find on the CDN, so they get re-uploaded, folding the outcomes into + * [tally]. + */ + private fun repairBatch(batch: List, tally: RepairTally, internalUser: Boolean) { + if (batch.isEmpty()) { + return + } + + for ((isThumbnail, entries) in batch.groupBy { it.isThumbnail }) { + val mediaNames = entries.map { MediaNameParts.fromBytes(plaintextHash = it.plaintextHash, remoteKey = it.remoteKey) } + + val results = if (isThumbnail) { + SignalDatabase.attachments.resetArchiveThumbnailTransferStateByPlaintextHashAndRemoteKeyIfNecessary(mediaNames) + } else { + SignalDatabase.attachments.resetArchiveTransferStateByPlaintextHashAndRemoteKeyIfNecessary(mediaNames) + } + + for ((entry, result) in entries.zip(results)) { + when (result) { + ArchiveTransferStateResetResult.RESET -> { + val mediaIdLog = if (internalUser) "[${MediaId(entry.mediaId)}]" else "" + val logPrefix = if (isThumbnail) "[Thumbnail]$mediaIdLog" else "[Fullsize]$mediaIdLog" + Log.w(TAG, "$logPrefix Reset transfer state by hash/key.", true) + + tally.resetCount++ + tally.markReUploadNeeded(isThumbnail) + } + + ArchiveTransferStateResetResult.SKIPPED_NO_LOCAL_DATA -> { + tally.unrecoverableCount++ + } + + ArchiveTransferStateResetResult.MARKED_UNRECOVERABLE -> { + tally.markedUnrecoverableCount++ + } + + ArchiveTransferStateResetResult.NOT_NEEDED -> { + tally.notNeededCount++ + + // Deliberately not done for SKIPPED_NO_LOCAL_DATA, since the precautionary backfills these drive could never upload media that has no local bytes. + tally.markReUploadNeeded(isThumbnail) + } + } + } + } + } + private fun maybePostReconciliationFailureNotification() { if (!RemoteConfig.internalUser) { return @@ -565,6 +586,23 @@ class ArchiveAttachmentReconciliationJob private constructor( NotificationManagerCompat.from(context).notify(NotificationIds.RECONCILIATION_ERROR, notification) } + private class RepairTally { + var resetCount = 0 + var unrecoverableCount = 0 + var markedUnrecoverableCount = 0 + var notNeededCount = 0 + var fullSizeReUploadNeeded = false + var thumbnailReUploadNeeded = false + + fun markReUploadNeeded(isThumbnail: Boolean) { + if (isThumbnail) { + thumbnailReUploadNeeded = true + } else { + fullSizeReUploadNeeded = true + } + } + } + class Factory : Job.Factory { override fun create(parameters: Parameters, serializedData: ByteArray?): ArchiveAttachmentReconciliationJob { val data = ArchiveAttachmentReconciliationJobData.ADAPTER.decode(serializedData!!) diff --git a/app/src/main/java/org/thoughtcrime/securesms/messagedetails/InternalMessageDetailsFragment.kt b/app/src/main/java/org/thoughtcrime/securesms/messagedetails/InternalMessageDetailsFragment.kt index feffaebc13..ec387bfed1 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/messagedetails/InternalMessageDetailsFragment.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/messagedetails/InternalMessageDetailsFragment.kt @@ -20,7 +20,11 @@ import androidx.compose.material3.MaterialTheme import androidx.compose.material3.Surface import androidx.compose.material3.Text import androidx.compose.runtime.Composable +import androidx.compose.runtime.LaunchedEffect import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember +import androidx.compose.runtime.setValue import androidx.compose.ui.Modifier import androidx.compose.ui.platform.LocalContext import androidx.compose.ui.text.SpanStyle @@ -32,9 +36,12 @@ import androidx.compose.ui.tooling.preview.Preview import androidx.compose.ui.unit.dp import androidx.core.os.bundleOf import androidx.fragment.app.FragmentActivity +import org.signal.core.ui.compose.Buttons import org.signal.core.ui.compose.ComposeFullScreenDialogFragment +import org.signal.core.ui.compose.Dialogs import org.signal.core.ui.compose.Dividers import org.signal.core.util.Util +import org.thoughtcrime.securesms.database.AttachmentTable import org.thoughtcrime.securesms.database.model.MessageRecord import org.thoughtcrime.securesms.messagedetails.InternalMessageDetailsViewModel.AttachmentInfo import org.thoughtcrime.securesms.messagedetails.InternalMessageDetailsViewModel.ViewState @@ -61,15 +68,32 @@ class InternalMessageDetailsFragment : ComposeFullScreenDialogFragment() { @Composable override fun DialogContent() { val state by viewModel.state + val actionResult by viewModel.actionResult + val context = LocalContext.current + + LaunchedEffect(actionResult) { + actionResult?.let { + Toast.makeText(context, it, Toast.LENGTH_SHORT).show() + viewModel.consumeActionResult() + } + } state?.let { - Content(it) + Content( + state = it, + onOffloadLocalData = viewModel::offloadLocalData, + onDeleteFromCdn = viewModel::deleteFromCdn + ) } } } @Composable -private fun Content(state: ViewState) { +private fun Content( + state: ViewState, + onOffloadLocalData: (Long) -> Unit = {}, + onDeleteFromCdn: (Long, Boolean) -> Unit = { _, _ -> } +) { val context = LocalContext.current Surface( @@ -145,7 +169,11 @@ private fun Content(state: ViewState) { ) } else { state.attachments.forEachIndexed { i, attachment -> - AttachmentBlock(attachment) + AttachmentBlock( + attachment = attachment, + onOffloadLocalData = onOffloadLocalData, + onDeleteFromCdn = onDeleteFromCdn + ) if (i != state.attachments.lastIndex) { Dividers.Default() @@ -191,7 +219,11 @@ private fun ClickToCopyRow(name: String, value: String, valueToCopy: String = va } @Composable -private fun AttachmentBlock(attachment: AttachmentInfo) { +private fun AttachmentBlock( + attachment: AttachmentInfo, + onOffloadLocalData: (Long) -> Unit, + onDeleteFromCdn: (Long, Boolean) -> Unit +) { ClickToCopyRow( name = "ID", value = attachment.id.toString() @@ -220,6 +252,128 @@ private fun AttachmentBlock(attachment: AttachmentInfo) { name = "Transform Properties", value = attachment.transformProperties ?: "null" ) + ClickToCopyRow( + name = "Has Local Data", + value = attachment.hasLocalData.toString() + ) + ClickToCopyRow( + name = "Transfer State", + value = attachment.transferState.toString() + ) + ClickToCopyRow( + name = "Archive CDN", + value = attachment.archiveCdn?.toString() ?: "null" + ) + ClickToCopyRow( + name = "Archive Transfer State", + value = attachment.archiveTransferState.name + ) + ClickToCopyRow( + name = "Archive Thumbnail Transfer State", + value = attachment.archiveThumbnailTransferState.name + ) + + DestructiveActions( + attachment = attachment, + onOffloadLocalData = onOffloadLocalData, + onDeleteFromCdn = onDeleteFromCdn + ) +} + +/** + * Collapsed by default, and each action confirms, because these are irreversible and sit in a list people scroll through to copy values. + */ +@Composable +private fun DestructiveActions( + attachment: AttachmentInfo, + onOffloadLocalData: (Long) -> Unit, + onDeleteFromCdn: (Long, Boolean) -> Unit +) { + var expanded by remember(attachment.id) { mutableStateOf(false) } + var pendingAction by remember(attachment.id) { mutableStateOf(null) } + + Text( + text = if (expanded) "▾ Destructive test actions" else "▸ Destructive test actions", + style = MaterialTheme.typography.labelLarge, + modifier = Modifier + .clickable { expanded = !expanded } + .padding(8.dp) + .fillMaxWidth() + ) + + if (!expanded) { + return + } + + val isArchived = attachment.archiveTransferState == AttachmentTable.ArchiveTransferState.FINISHED && attachment.archiveCdn != null + + Buttons.MediumTonal( + onClick = { pendingAction = PendingAction.OFFLOAD }, + modifier = Modifier + .padding(horizontal = 8.dp) + .fillMaxWidth() + ) { + Text(text = "Offload local data") + } + + Buttons.MediumTonal( + onClick = { pendingAction = PendingAction.DELETE_FULL_SIZE }, + modifier = Modifier + .padding(horizontal = 8.dp) + .fillMaxWidth() + ) { + Text(text = "Delete full-size from CDN") + } + + Buttons.MediumTonal( + onClick = { pendingAction = PendingAction.DELETE_THUMBNAIL }, + modifier = Modifier + .padding(horizontal = 8.dp) + .fillMaxWidth() + ) { + Text(text = "Delete thumbnail from CDN") + } + + when (pendingAction) { + null -> Unit + + PendingAction.OFFLOAD -> Dialogs.SimpleAlertDialog( + title = "Offload local data?", + body = if (isArchived) { + "Drops the local bytes for attachment ${attachment.id}. The archive CDN copy stays, so it can be restored by tapping the attachment." + } else { + "This attachment is NOT on the archive CDN (state ${attachment.archiveTransferState.name}, cdn ${attachment.archiveCdn ?: "null"}). Offloading it deletes the only copy and it cannot be restored." + }, + confirm = if (isArchived) "Offload" else "Delete the only copy", + dismiss = "Cancel", + onConfirm = { onOffloadLocalData(attachment.id) }, + onDismiss = { pendingAction = null } + ) + + PendingAction.DELETE_FULL_SIZE -> Dialogs.SimpleAlertDialog( + title = "Delete full-size from CDN?", + body = "Immediately and irreversibly removes the full-size copy of attachment ${attachment.id} from the archive CDN. If the local bytes are ever offloaded, the media is gone for good.", + confirm = "Delete from CDN", + dismiss = "Cancel", + onConfirm = { onDeleteFromCdn(attachment.id, false) }, + onDismiss = { pendingAction = null } + ) + + PendingAction.DELETE_THUMBNAIL -> Dialogs.SimpleAlertDialog( + title = "Delete thumbnail from CDN?", + body = "Immediately and irreversibly removes the thumbnail for attachment ${attachment.id} from the archive CDN. The full-size copy is untouched.", + confirm = "Delete from CDN", + dismiss = "Cancel", + onConfirm = { onDeleteFromCdn(attachment.id, true) }, + onDismiss = { pendingAction = null } + ) + } +} + +private enum class PendingAction { + OFFLOAD, + DELETE_FULL_SIZE, + DELETE_THUMBNAIL } @Preview diff --git a/app/src/main/java/org/thoughtcrime/securesms/messagedetails/InternalMessageDetailsViewModel.kt b/app/src/main/java/org/thoughtcrime/securesms/messagedetails/InternalMessageDetailsViewModel.kt index d79a2be5db..ec331cbf49 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/messagedetails/InternalMessageDetailsViewModel.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/messagedetails/InternalMessageDetailsViewModel.kt @@ -12,45 +12,136 @@ import androidx.lifecycle.ViewModel import androidx.lifecycle.viewModelScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.launch +import org.signal.core.models.backup.MediaName +import org.signal.core.models.database.AttachmentId +import org.signal.core.util.Base64 +import org.signal.core.util.logging.Log import org.signal.network.util.JsonUtil +import org.thoughtcrime.securesms.backup.v2.ArchivedMediaObject +import org.thoughtcrime.securesms.database.AttachmentTable import org.thoughtcrime.securesms.database.SignalDatabase +import org.thoughtcrime.securesms.jobs.ArchiveCommitAttachmentDeletesJob +import org.thoughtcrime.securesms.keyvalue.SignalStore import org.thoughtcrime.securesms.recipients.RecipientId class InternalMessageDetailsViewModel(val messageId: Long) : ViewModel() { + companion object { + private val TAG = Log.tag(InternalMessageDetailsViewModel::class) + } + private val _state: MutableState = mutableStateOf(null) val state: State = _state + private val _actionResult: MutableState = mutableStateOf(null) + val actionResult: State = _actionResult + init { - viewModelScope.launch(Dispatchers.Default) { - val messageRecord = SignalDatabase.messages.getMessageRecord(messageId) - val attachments = SignalDatabase.attachments.getAttachmentsForMessage(messageId) + refresh() + } - _state.value = ViewState( - id = messageRecord.id, - sentTimestamp = messageRecord.dateSent, - receivedTimestamp = messageRecord.dateReceived, - serverSentTimestamp = messageRecord.serverTimestamp, - from = messageRecord.fromRecipient.id, - to = messageRecord.toRecipient.id, - attachments = attachments.map { attachment -> - val info = SignalDatabase.attachments.getDataFileInfo(attachment.attachmentId) + fun consumeActionResult() { + _actionResult.value = null + } - AttachmentInfo( - id = attachment.attachmentId.id, - contentType = attachment.contentType, - quoteTargetContentType = attachment.quoteTargetContentType, - size = attachment.size, - fileName = attachment.fileName, - hashStart = info?.hashStart, - hashEnd = info?.hashEnd, - transformProperties = info?.transformProperties?.let { JsonUtil.toJson(it) } ?: "null" - ) - } - ) + /** + * Puts the attachment into the offloaded state, so the media is only on the archive CDN. + */ + fun offloadLocalData(attachmentId: Long) { + runAction("Offloaded local data") { + SignalDatabase.attachments.debugOffloadAttachment(AttachmentId(attachmentId)) } } + /** + * Deletes the archive CDN copy while leaving every local record alone, which is the state a reconciliation is meant to detect and repair. + */ + fun deleteFromCdn(attachmentId: Long, thumbnail: Boolean) { + val label = if (thumbnail) "Deleted thumbnail from CDN" else "Deleted full-size from CDN" + + runAction(label) { + val attachment = SignalDatabase.attachments.getAttachment(AttachmentId(attachmentId)) ?: error("Attachment is gone") + val plaintextHash = attachment.dataHash?.let { Base64.decode(it) } ?: error("No plaintext hash") + val remoteKey = attachment.remoteKey?.let { Base64.decode(it) } ?: error("No remote key") + val cdn = attachment.archiveCdn ?: error("No archive CDN, so nothing is up there") + + val mediaName = if (thumbnail) { + MediaName.fromPlaintextHashAndRemoteKeyForThumbnail(plaintextHash, remoteKey) + } else { + MediaName.fromPlaintextHashAndRemoteKey(plaintextHash, remoteKey) + } + + val mediaObject = ArchivedMediaObject( + mediaId = mediaName.toMediaId(SignalStore.backup.mediaRootBackupKey).encode(), + cdn = cdn + ) + + val failure = ArchiveCommitAttachmentDeletesJob.deleteMediaObjectsFromCdn( + tag = TAG, + attachmentsToDelete = setOf(mediaObject), + backoffGenerator = { 0 }, + cancellationSignal = { false } + ) + + if (failure != null) { + error("CDN delete did not succeed: $failure") + } + } + } + + private fun runAction(successLabel: String, action: suspend () -> Unit) { + viewModelScope.launch(Dispatchers.IO) { + _actionResult.value = try { + action() + successLabel + } catch (e: Exception) { + Log.w(TAG, "Internal attachment action failed.", e) + "Failed: ${e.message}" + } + + loadState() + } + } + + private fun refresh() { + viewModelScope.launch(Dispatchers.IO) { + loadState() + } + } + + private fun loadState() { + val messageRecord = SignalDatabase.messages.getMessageRecord(messageId) + val attachments = SignalDatabase.attachments.getAttachmentsForMessage(messageId) + + _state.value = ViewState( + id = messageRecord.id, + sentTimestamp = messageRecord.dateSent, + receivedTimestamp = messageRecord.dateReceived, + serverSentTimestamp = messageRecord.serverTimestamp, + from = messageRecord.fromRecipient.id, + to = messageRecord.toRecipient.id, + attachments = attachments.map { attachment -> + val info = SignalDatabase.attachments.getDataFileInfo(attachment.attachmentId) + + AttachmentInfo( + id = attachment.attachmentId.id, + contentType = attachment.contentType, + quoteTargetContentType = attachment.quoteTargetContentType, + size = attachment.size, + fileName = attachment.fileName, + hashStart = info?.hashStart, + hashEnd = info?.hashEnd, + transformProperties = info?.transformProperties?.let { JsonUtil.toJson(it) } ?: "null", + hasLocalData = attachment.hasData, + transferState = attachment.transferState, + archiveCdn = attachment.archiveCdn, + archiveTransferState = attachment.archiveTransferState, + archiveThumbnailTransferState = attachment.archiveThumbnailTransferState + ) + } + ) + } + data class ViewState( val id: Long, val sentTimestamp: Long, @@ -69,6 +160,11 @@ class InternalMessageDetailsViewModel(val messageId: Long) : ViewModel() { val fileName: String?, val hashStart: String?, val hashEnd: String?, - val transformProperties: String? + val transformProperties: String?, + val hasLocalData: Boolean, + val transferState: Int, + val archiveCdn: Int?, + val archiveTransferState: AttachmentTable.ArchiveTransferState, + val archiveThumbnailTransferState: AttachmentTable.ArchiveTransferState ) } diff --git a/app/src/test/java/org/thoughtcrime/securesms/backup/v2/DatabaseAttachmentArchiveUtilTest.kt b/app/src/test/java/org/thoughtcrime/securesms/backup/v2/DatabaseAttachmentArchiveUtilTest.kt index c6beaa40e3..9b4060b1ef 100644 --- a/app/src/test/java/org/thoughtcrime/securesms/backup/v2/DatabaseAttachmentArchiveUtilTest.kt +++ b/app/src/test/java/org/thoughtcrime/securesms/backup/v2/DatabaseAttachmentArchiveUtilTest.kt @@ -123,6 +123,7 @@ class DatabaseAttachmentArchiveUtilTest { archiveCdn = archiveCdn, thumbnailRestoreState = AttachmentTable.ThumbnailRestoreState.NONE, archiveTransferState = AttachmentTable.ArchiveTransferState.FINISHED, + archiveThumbnailTransferState = AttachmentTable.ArchiveTransferState.NONE, uuid = null, quoteTargetContentType = null, metadata = null diff --git a/app/src/test/java/org/thoughtcrime/securesms/components/settings/conversation/individual/IndividualSettingsViewModelTest.kt b/app/src/test/java/org/thoughtcrime/securesms/components/settings/conversation/individual/IndividualSettingsViewModelTest.kt index 7ae005abc5..c120a9bb2f 100644 --- a/app/src/test/java/org/thoughtcrime/securesms/components/settings/conversation/individual/IndividualSettingsViewModelTest.kt +++ b/app/src/test/java/org/thoughtcrime/securesms/components/settings/conversation/individual/IndividualSettingsViewModelTest.kt @@ -850,6 +850,7 @@ class IndividualSettingsViewModelTest { archiveCdn = null, thumbnailRestoreState = AttachmentTable.ThumbnailRestoreState.NONE, archiveTransferState = AttachmentTable.ArchiveTransferState.NONE, + archiveThumbnailTransferState = AttachmentTable.ArchiveTransferState.NONE, uuid = null, quoteTargetContentType = null, metadata = null diff --git a/app/src/test/java/org/thoughtcrime/securesms/components/transfercontrols/TransferControlsTest.kt b/app/src/test/java/org/thoughtcrime/securesms/components/transfercontrols/TransferControlsTest.kt index 959ef2c7e0..0ef97fd488 100644 --- a/app/src/test/java/org/thoughtcrime/securesms/components/transfercontrols/TransferControlsTest.kt +++ b/app/src/test/java/org/thoughtcrime/securesms/components/transfercontrols/TransferControlsTest.kt @@ -351,6 +351,7 @@ class TransferControlsTest { archiveCdn = null, thumbnailRestoreState = AttachmentTable.ThumbnailRestoreState.NONE, archiveTransferState = AttachmentTable.ArchiveTransferState.NONE, + archiveThumbnailTransferState = AttachmentTable.ArchiveTransferState.NONE, uuid = null, quoteTargetContentType = null, metadata = null diff --git a/app/src/test/java/org/thoughtcrime/securesms/database/AttachmentTableTest_localRestoreArchiveState.kt b/app/src/test/java/org/thoughtcrime/securesms/database/AttachmentTableTest_localRestoreArchiveState.kt index 56d7a1a776..cdf111fd81 100644 --- a/app/src/test/java/org/thoughtcrime/securesms/database/AttachmentTableTest_localRestoreArchiveState.kt +++ b/app/src/test/java/org/thoughtcrime/securesms/database/AttachmentTableTest_localRestoreArchiveState.kt @@ -7,6 +7,8 @@ package org.thoughtcrime.securesms.database import android.app.Application import assertk.assertThat +import assertk.assertions.containsExactly +import assertk.assertions.isEmpty import assertk.assertions.isEqualTo import assertk.assertions.isFalse import assertk.assertions.isNull @@ -18,7 +20,6 @@ import org.junit.runner.RunWith import org.robolectric.RobolectricTestRunner import org.robolectric.annotation.Config import org.signal.core.models.database.AttachmentId -import org.signal.core.util.Base64 import org.signal.core.util.logging.Log import org.thoughtcrime.securesms.attachments.ArchivedAttachment import org.thoughtcrime.securesms.attachments.Attachment @@ -96,12 +97,11 @@ class AttachmentTableTest_localRestoreArchiveState { val attachmentId = insertArchivedAttachment(archiveCdn = 3, localBackupKey = Random.nextBytes(32)) val attachment = SignalDatabase.attachments.getAttachment(attachmentId)!! - val result = SignalDatabase.attachments.resetArchiveTransferStateByPlaintextHashAndRemoteKeyIfNecessary( - plaintextHash = Base64.decode(attachment.dataHash!!), - remoteKey = Base64.decode(attachment.remoteKey!!) + val results = SignalDatabase.attachments.resetArchiveTransferStateByPlaintextHashAndRemoteKeyIfNecessary( + listOf(AttachmentTable.MediaNameParts(plaintextHash = attachment.dataHash!!, remoteKey = attachment.remoteKey!!)) ) - assertThat(result).isEqualTo(AttachmentTable.ArchiveTransferStateResetResult.SKIPPED_NO_LOCAL_DATA) + assertThat(results).containsExactly(AttachmentTable.ArchiveTransferStateResetResult.SKIPPED_NO_LOCAL_DATA) val after = SignalDatabase.attachments.getAttachment(attachmentId)!! assertThat(after.archiveTransferState).isEqualTo(AttachmentTable.ArchiveTransferState.FINISHED) @@ -113,12 +113,81 @@ class AttachmentTableTest_localRestoreArchiveState { val attachmentId = insertArchivedAttachment(archiveCdn = null, localBackupKey = Random.nextBytes(32)) val attachment = SignalDatabase.attachments.getAttachment(attachmentId)!! - val result = SignalDatabase.attachments.resetArchiveTransferStateByPlaintextHashAndRemoteKeyIfNecessary( - plaintextHash = Base64.decode(attachment.dataHash!!), - remoteKey = Base64.decode(attachment.remoteKey!!) + val results = SignalDatabase.attachments.resetArchiveTransferStateByPlaintextHashAndRemoteKeyIfNecessary( + listOf(AttachmentTable.MediaNameParts(plaintextHash = attachment.dataHash!!, remoteKey = attachment.remoteKey!!)) ) - assertThat(result).isEqualTo(AttachmentTable.ArchiveTransferStateResetResult.NOT_NEEDED) + assertThat(results).containsExactly(AttachmentTable.ArchiveTransferStateResetResult.NOT_NEEDED) + } + + @Test + fun resetArchiveTransferState_batchedResultsLineUpWithTheirKeys() { + val finishedNoLocalData = SignalDatabase.attachments.getAttachment(insertArchivedAttachment(archiveCdn = 3, localBackupKey = Random.nextBytes(32)))!! + val notFinished = SignalDatabase.attachments.getAttachment(insertArchivedAttachment(archiveCdn = null, localBackupKey = Random.nextBytes(32)))!! + + val results = SignalDatabase.attachments.resetArchiveTransferStateByPlaintextHashAndRemoteKeyIfNecessary( + listOf( + AttachmentTable.MediaNameParts.fromBytes(plaintextHash = Random.nextBytes(8), remoteKey = Random.nextBytes(8)), + AttachmentTable.MediaNameParts(plaintextHash = finishedNoLocalData.dataHash!!, remoteKey = finishedNoLocalData.remoteKey!!), + AttachmentTable.MediaNameParts(plaintextHash = notFinished.dataHash!!, remoteKey = notFinished.remoteKey!!) + ) + ) + + assertThat(results).containsExactly( + AttachmentTable.ArchiveTransferStateResetResult.NOT_NEEDED, + AttachmentTable.ArchiveTransferStateResetResult.SKIPPED_NO_LOCAL_DATA, + AttachmentTable.ArchiveTransferStateResetResult.NOT_NEEDED + ) + } + + @Test + fun resetArchiveThumbnailTransferState_marksUnrecoverableWhenThereIsNoLocalDataFileToRebuildFrom() { + val attachmentId = insertArchivedAttachment(archiveCdn = 3, localBackupKey = Random.nextBytes(32)) + val attachment = SignalDatabase.attachments.getAttachment(attachmentId)!! + SignalDatabase.attachments.setArchiveThumbnailTransferState(attachmentId, AttachmentTable.ArchiveTransferState.FINISHED) + + val results = SignalDatabase.attachments.resetArchiveThumbnailTransferStateByPlaintextHashAndRemoteKeyIfNecessary( + listOf(AttachmentTable.MediaNameParts(plaintextHash = attachment.dataHash!!, remoteKey = attachment.remoteKey!!)) + ) + + assertThat(results).containsExactly(AttachmentTable.ArchiveTransferStateResetResult.MARKED_UNRECOVERABLE) + assertThat(SignalDatabase.attachments.getArchiveThumbnailTransferState(attachmentId)).isEqualTo(AttachmentTable.ArchiveTransferState.PERMANENT_FAILURE) + + // The full-size locator is deliberately left intact, since that media is still on the CDN. + val after = SignalDatabase.attachments.getAttachment(attachmentId)!! + assertThat(after.archiveTransferState).isEqualTo(AttachmentTable.ArchiveTransferState.FINISHED) + assertThat(after.archiveCdn).isEqualTo(3) + } + + @Test + fun resetArchiveThumbnailTransferState_reportsAlreadyRecordedMediaAsUnrecoverableRatherThanNotNeeded() { + val attachmentId = insertArchivedAttachment(archiveCdn = 3, localBackupKey = Random.nextBytes(32)) + val attachment = SignalDatabase.attachments.getAttachment(attachmentId)!! + SignalDatabase.attachments.setArchiveThumbnailTransferState(attachmentId, AttachmentTable.ArchiveTransferState.FINISHED) + + val mediaNames = listOf( + AttachmentTable.MediaNameParts(plaintextHash = attachment.dataHash!!, remoteKey = attachment.remoteKey!!) + ) + + assertThat(SignalDatabase.attachments.resetArchiveThumbnailTransferStateByPlaintextHashAndRemoteKeyIfNecessary(mediaNames)) + .containsExactly(AttachmentTable.ArchiveTransferStateResetResult.MARKED_UNRECOVERABLE) + + assertThat(SignalDatabase.attachments.resetArchiveThumbnailTransferStateByPlaintextHashAndRemoteKeyIfNecessary(mediaNames)) + .containsExactly(AttachmentTable.ArchiveTransferStateResetResult.MARKED_UNRECOVERABLE) + } + + @Test(expected = IllegalArgumentException::class) + fun resetArchiveTransferState_rejectsABatchTooLargeForOneStatement() { + val tooMany = List(AttachmentTable.ARCHIVE_MEDIA_KEY_BATCH_SIZE + 1) { + AttachmentTable.MediaNameParts.fromBytes(plaintextHash = Random.nextBytes(8), remoteKey = Random.nextBytes(8)) + } + + SignalDatabase.attachments.resetArchiveTransferStateByPlaintextHashAndRemoteKeyIfNecessary(tooMany) + } + + @Test + fun resetArchiveTransferState_noResultsForAnEmptyBatch() { + assertThat(SignalDatabase.attachments.resetArchiveThumbnailTransferStateByPlaintextHashAndRemoteKeyIfNecessary(emptyList())).isEmpty() } private fun insertArchivedAttachment(archiveCdn: Int?, localBackupKey: ByteArray?): AttachmentId { diff --git a/app/src/test/java/org/thoughtcrime/securesms/jobs/AttachmentBackfillTest.kt b/app/src/test/java/org/thoughtcrime/securesms/jobs/AttachmentBackfillTest.kt index cb3e08776f..2c99dbcdb0 100644 --- a/app/src/test/java/org/thoughtcrime/securesms/jobs/AttachmentBackfillTest.kt +++ b/app/src/test/java/org/thoughtcrime/securesms/jobs/AttachmentBackfillTest.kt @@ -134,6 +134,7 @@ class AttachmentBackfillTest { archiveCdn = null, thumbnailRestoreState = AttachmentTable.ThumbnailRestoreState.NONE, archiveTransferState = AttachmentTable.ArchiveTransferState.NONE, + archiveThumbnailTransferState = AttachmentTable.ArchiveTransferState.NONE, uuid = null, quoteTargetContentType = null, metadata = null diff --git a/app/src/test/java/org/thoughtcrime/securesms/sms/UploadDependencyGraphTest.kt b/app/src/test/java/org/thoughtcrime/securesms/sms/UploadDependencyGraphTest.kt index b14f680eea..12980c50b0 100644 --- a/app/src/test/java/org/thoughtcrime/securesms/sms/UploadDependencyGraphTest.kt +++ b/app/src/test/java/org/thoughtcrime/securesms/sms/UploadDependencyGraphTest.kt @@ -256,6 +256,7 @@ class UploadDependencyGraphTest { archiveCdn = 0, thumbnailRestoreState = AttachmentTable.ThumbnailRestoreState.NONE, archiveTransferState = AttachmentTable.ArchiveTransferState.NONE, + archiveThumbnailTransferState = AttachmentTable.ArchiveTransferState.NONE, uuid = null, quoteTargetContentType = null, metadata = null diff --git a/app/src/test/java/org/thoughtcrime/securesms/util/AttachmentUtilTest.kt b/app/src/test/java/org/thoughtcrime/securesms/util/AttachmentUtilTest.kt index 77b9e04751..220957a372 100644 --- a/app/src/test/java/org/thoughtcrime/securesms/util/AttachmentUtilTest.kt +++ b/app/src/test/java/org/thoughtcrime/securesms/util/AttachmentUtilTest.kt @@ -491,6 +491,7 @@ class AttachmentUtilTest { archiveCdn = null, thumbnailRestoreState = AttachmentTable.ThumbnailRestoreState.NONE, archiveTransferState = AttachmentTable.ArchiveTransferState.NONE, + archiveThumbnailTransferState = AttachmentTable.ArchiveTransferState.NONE, uuid = null, quoteTargetContentType = null, metadata = null diff --git a/app/src/testShared/org/thoughtcrime/securesms/database/FakeMessageRecords.kt b/app/src/testShared/org/thoughtcrime/securesms/database/FakeMessageRecords.kt index b0aa51c73f..60407eca55 100644 --- a/app/src/testShared/org/thoughtcrime/securesms/database/FakeMessageRecords.kt +++ b/app/src/testShared/org/thoughtcrime/securesms/database/FakeMessageRecords.kt @@ -68,7 +68,8 @@ object FakeMessageRecords { archiveMediaId: String? = null, archiveThumbnailId: String? = null, thumbnailRestoreState: AttachmentTable.ThumbnailRestoreState = AttachmentTable.ThumbnailRestoreState.NONE, - archiveTransferState: AttachmentTable.ArchiveTransferState = AttachmentTable.ArchiveTransferState.NONE + archiveTransferState: AttachmentTable.ArchiveTransferState = AttachmentTable.ArchiveTransferState.NONE, + archiveThumbnailTransferState: AttachmentTable.ArchiveTransferState = AttachmentTable.ArchiveTransferState.NONE ): DatabaseAttachment { return DatabaseAttachment( attachmentId = attachmentId, @@ -103,6 +104,7 @@ object FakeMessageRecords { archiveCdn = archiveCdn, thumbnailRestoreState = thumbnailRestoreState, archiveTransferState = archiveTransferState, + archiveThumbnailTransferState = archiveThumbnailTransferState, uuid = null, quoteTargetContentType = null, metadata = null