diff --git a/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/backup/InternalBackupStatsTab.kt b/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/backup/InternalBackupStatsTab.kt index cd00a213d5..56541e7578 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/backup/InternalBackupStatsTab.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/backup/InternalBackupStatsTab.kt @@ -17,6 +17,7 @@ import androidx.compose.runtime.Composable import androidx.compose.ui.Modifier import androidx.compose.ui.unit.dp import org.signal.core.ui.compose.Dividers +import org.signal.core.ui.compose.Rows import org.signal.core.ui.compose.Texts import org.signal.core.util.bytes @@ -24,49 +25,65 @@ import org.signal.core.util.bytes fun InternalBackupStatsTab(stats: InternalBackupPlaygroundViewModel.StatsState, callbacks: StatsCallbacks) { val scrollState = rememberScrollState() Column(modifier = Modifier.verticalScroll(scrollState)) { - Texts.SectionHeader(text = "Local Attachment State") - if (stats.attachmentStats != null) { - Text(text = "Attachment Count: ${stats.attachmentStats.attachmentCount}") + Texts.SectionHeader(text = "Local Attachments") - Text(text = "Transit Download State:") - stats.attachmentStats.transferStateCounts.forEach { (state, count) -> - if (count > 0) { - Text(text = "$state: $count") - } - } + Rows.TextRow( + text = "Total attachment rows", + label = "${stats.attachmentStats.totalAttachmentRows}" + ) - Text(text = "Valid for archive Transit Download State:") - stats.attachmentStats.validForArchiveTransferStateCounts.forEach { (state, count) -> - if (count > 0) { - Text(text = "$state: $count") - } - } + Rows.TextRow( + text = "Total unique data files", + label = "${stats.attachmentStats.totalUniqueDataFiles}" + ) - Spacer(modifier = Modifier.size(4.dp)) + Rows.TextRow( + text = "Total unique media names", + label = "${stats.attachmentStats.totalUniqueMediaNames}" + ) - Text(text = "Archive State:") - stats.attachmentStats.archiveStateCounts.forEach { (state, count) -> - if (count > 0) { - Text(text = "$state: $count") - } - } + Rows.TextRow( + text = "Total eligible for upload rows", + label = "${stats.attachmentStats.totalEligibleForUploadRows}" + ) + + Rows.TextRow( + text = "Total unique media names eligible for upload ⭐", + label = "${stats.attachmentStats.totalUniqueMediaNamesEligibleForUpload}" + ) + + Rows.TextRow( + text = "Eligible attachments by status ⭐", + label = stats.attachmentStats.archiveStatusMediaNameCounts.entries.joinToString("\n") { (status, count) -> "$status: $count" } + ) + + Rows.TextRow( + text = "Total media names with thumbnails", + label = "${stats.attachmentStats.mediaNamesWithThumbnailsCount}" + ) + + Rows.TextRow( + text = "Eligible thumbnails by status ⭐", + label = stats.attachmentStats.archiveStatusMediaNameThumbnailCounts.entries.joinToString("\n") { (status, count) -> "$status: $count" } + ) + + Rows.TextRow( + text = "Pending attachment upload bytes ⭐", + label = "${stats.attachmentStats.pendingAttachmentUploadBytes} (~${stats.attachmentStats.pendingAttachmentUploadBytes.bytes.toUnitString()})" + ) + + Rows.TextRow( + text = "Uploaded attachment bytes ⭐", + label = "${stats.attachmentStats.uploadedAttachmentBytes} (~${stats.attachmentStats.uploadedAttachmentBytes.bytes.toUnitString()})" + ) + + Rows.TextRow( + text = "Uploaded thumbnail bytes (estimated)", + label = "${stats.attachmentStats.uploadedThumbnailBytes} (~${stats.attachmentStats.uploadedThumbnailBytes.bytes.toUnitString()})" + ) Spacer(modifier = Modifier.size(16.dp)) - - Text(text = "Unique/archived data files: ${stats.attachmentStats.attachmentFileCount}/${stats.attachmentStats.finishedAttachmentFileCount}") - Text(text = "Unique/archived verified plaintextHash count: ${stats.attachmentStats.attachmentPlaintextHashAndKeyCount}/${stats.attachmentStats.finishedAttachmentPlaintextHashAndKeyCount}") - Text(text = "Unique/expected thumbnail files: ${stats.attachmentStats.thumbnailFileCount}/${stats.attachmentStats.estimatedThumbnailCount}") - Text(text = "Local Total: ${stats.attachmentStats.attachmentFileCount + stats.attachmentStats.thumbnailFileCount}") - Text(text = "Expected remote total: ${stats.attachmentStats.estimatedThumbnailCount + stats.attachmentStats.finishedAttachmentPlaintextHashAndKeyCount}") - - Spacer(modifier = Modifier.size(16.dp)) - - Text(text = "Pending upload: ${stats.attachmentStats.pendingUploadBytes} (~${stats.attachmentStats.pendingUploadBytes.bytes.toUnitString()})") - Text(text = "Est uploaded attachments: ${stats.attachmentStats.uploadedAttachmentBytes} (~${stats.attachmentStats.uploadedAttachmentBytes.bytes.toUnitString()})") - Text(text = "Est uploaded thumbnails: ${stats.attachmentStats.thumbnailBytes} (~${stats.attachmentStats.thumbnailBytes.bytes.toUnitString()})") - val total = stats.attachmentStats.thumbnailBytes + stats.attachmentStats.uploadedAttachmentBytes - Text(text = "Est total: $total (~${total.bytes.toUnitString()})") } else { CircularProgressIndicator() } @@ -79,28 +96,43 @@ fun InternalBackupStatsTab(stats: InternalBackupPlaygroundViewModel.StatsState, Button(onClick = callbacks::loadRemoteState) { Text(text = "Load remote stats (expensive and long)") } + } else if (stats.remoteFailureMsg != null) { + Text(text = stats.remoteFailureMsg) + } else if (stats.loadingRemoteStats) { + CircularProgressIndicator() + } else if (stats.remoteState != null) { + Rows.TextRow( + "Total media items ⭐", + label = "${stats.remoteState.mediaCount}" + ) + + Rows.TextRow( + "Total media size ⭐", + label = "${stats.remoteState.mediaSize} (~${stats.remoteState.mediaSize.bytes.toUnitString()})" + ) + + Rows.TextRow( + text = "Server estimated used size", + label = "${stats.remoteState.usedSpace} (~${stats.remoteState.usedSpace.bytes.toUnitString()})" + ) + } + + Dividers.Default() + + Texts.SectionHeader(text = "Expected vs Actual") + + if (stats.attachmentStats != null && stats.remoteState != null) { + Rows.TextRow( + text = "Counts ⭐", + label = "Local: ${stats.attachmentStats.totalUploadCount}\nRemote: ${stats.remoteState.mediaCount}" + ) + + Rows.TextRow( + text = "Bytes ⭐", + label = "Local: ${stats.attachmentStats.totalUploadBytes} (~${stats.attachmentStats.totalUploadBytes.bytes.toUnitString()}, thumbnails are estimated)\nRemote: ${stats.remoteState.mediaSize} (~${stats.remoteState.mediaSize.bytes.toUnitString()})" + ) } else { - if (stats.loadingRemoteStats) { - CircularProgressIndicator() - } else if (stats.remoteState != null) { - Text(text = "Media item count: ${stats.remoteState.mediaCount}") - Text(text = "Media items sum size: ${stats.remoteState.mediaSize} (~${stats.remoteState.mediaSize.bytes.toUnitString()})") - Text(text = "Server estimated used size: ${stats.remoteState.usedSpace} (~${stats.remoteState.usedSpace.bytes.toUnitString()})") - } else if (stats.remoteFailureMsg != null) { - Text(text = stats.remoteFailureMsg) - } - - Dividers.Default() - - Texts.SectionHeader(text = "Expected vs Actual") - - if (stats.attachmentStats != null && stats.remoteState != null) { - val finished = stats.attachmentStats.finishedAttachmentFileCount - val thumbnails = stats.attachmentStats.thumbnailFileCount - Text(text = "Expected Count/Actual Remote Count: ${finished + thumbnails} / ${stats.remoteState.mediaCount}") - } else { - CircularProgressIndicator() - } + CircularProgressIndicator() } } } diff --git a/app/src/main/java/org/thoughtcrime/securesms/database/AttachmentTable.kt b/app/src/main/java/org/thoughtcrime/securesms/database/AttachmentTable.kt index 1e6342e462..7d70bacd8a 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/database/AttachmentTable.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/database/AttachmentTable.kt @@ -177,6 +177,7 @@ class AttachmentTable( const val UPLOAD_TIMESTAMP = "upload_timestamp" const val ARCHIVE_CDN = "archive_cdn" const val ARCHIVE_TRANSFER_STATE = "archive_transfer_state" + const val ARCHIVE_THUMBNAIL_TRANSFER_STATE = "archive_thumbnail_transfer_state" const val THUMBNAIL_RESTORE_STATE = "thumbnail_restore_state" const val ATTACHMENT_UUID = "attachment_uuid" const val OFFLOAD_RESTORED_AT = "offload_restored_at" @@ -282,7 +283,8 @@ class AttachmentTable( $THUMBNAIL_RESTORE_STATE INTEGER DEFAULT ${ThumbnailRestoreState.NONE.value}, $ATTACHMENT_UUID TEXT DEFAULT NULL, $OFFLOAD_RESTORED_AT INTEGER DEFAULT 0, - $QUOTE_TARGET_CONTENT_TYPE TEXT DEFAULT NULL + $QUOTE_TARGET_CONTENT_TYPE TEXT DEFAULT NULL, + $ARCHIVE_THUMBNAIL_TRANSFER_STATE INTEGER DEFAULT ${ArchiveTransferState.NONE.value} ) """ @@ -420,14 +422,14 @@ class AttachmentTable( } /** - * Returns a cursor (with just the plaintextHash+remoteKey+archive_cdn) for all attachments that are eligible for archive upload. - * In practice, this means that the attachments have a plaintextHash and have not hit a permanent archive upload failure. + * Returns a cursor (with just the plaintextHash+remoteKey+archive_cdn) for all attachments that are slated to be included in the current archive upload. + * Used for snapshotting data in [BackupMediaSnapshotTable]. */ - fun getAttachmentsEligibleForArchiveUpload(): Cursor { + fun getAttachmentsThatWillBeIncludedInArchive(): Cursor { return readableDatabase .select(DATA_HASH_END, REMOTE_KEY, ARCHIVE_CDN, QUOTE, CONTENT_TYPE) - .from(TABLE_NAME) - .where("$DATA_HASH_END NOT NULL AND $REMOTE_KEY NOT NULL AND $ARCHIVE_TRANSFER_STATE != ${ArchiveTransferState.PERMANENT_FAILURE.value}") + .from("$TABLE_NAME LEFT JOIN ${MessageTable.TABLE_NAME} ON $TABLE_NAME.$MESSAGE_ID = ${MessageTable.TABLE_NAME}.${MessageTable.ID}") + .where(buildAttachmentsThatNeedUploadQuery(transferStateFilter = "$ARCHIVE_TRANSFER_STATE != ${ArchiveTransferState.PERMANENT_FAILURE.value}")) .run() } @@ -805,6 +807,42 @@ class AttachmentTable( .run() } + /** + * Returns whether or not there are thumbnails that need to be uploaded to the archive. + */ + fun doAnyThumbnailsNeedArchiveUpload(): Boolean { + return readableDatabase + .exists(TABLE_NAME) + .where( + """ + $ARCHIVE_TRANSFER_STATE = ${ArchiveTransferState.FINISHED.value} AND + $ARCHIVE_THUMBNAIL_TRANSFER_STATE = ${ArchiveTransferState.NONE.value} AND + $QUOTE = 0 AND + ($CONTENT_TYPE LIKE 'image%' OR $CONTENT_TYPE LIKE 'video%') + """ + ) + .run() + } + + /** + * Returns whether or not there are thumbnails that need to be uploaded to the archive. + */ + fun getThumbnailsThatNeedArchiveUpload(): List { + return readableDatabase + .select(ID) + .from(TABLE_NAME) + .where( + """ + $ARCHIVE_TRANSFER_STATE = ${ArchiveTransferState.FINISHED.value} AND + $ARCHIVE_THUMBNAIL_TRANSFER_STATE = ${ArchiveTransferState.NONE.value} AND + $QUOTE = 0 AND + ($CONTENT_TYPE LIKE 'image%' OR $CONTENT_TYPE LIKE 'video%') + """ + ) + .run() + .readToList { AttachmentId(it.requireLong(ID)) } + } + /** * Returns the current archive transfer state, if the attachment can be found. */ @@ -817,6 +855,18 @@ class AttachmentTable( .readToSingleObject { ArchiveTransferState.deserialize(it.requireInt(ARCHIVE_TRANSFER_STATE)) } } + /** + * Returns the current archive thumbnail transfer state, if the attachment can be found. + */ + fun getArchiveThumbnailTransferState(id: AttachmentId): ArchiveTransferState? { + return readableDatabase + .select(ARCHIVE_THUMBNAIL_TRANSFER_STATE) + .from(TABLE_NAME) + .where("$ID = ?", id.id) + .run() + .readToSingleObject { ArchiveTransferState.deserialize(it.requireInt(ARCHIVE_THUMBNAIL_TRANSFER_STATE)) } + } + /** * Sets the archive transfer state for the given attachment and all other attachments that share the same data file. */ @@ -839,6 +889,25 @@ class AttachmentTable( AppDependencies.databaseObserver.notifyAttachmentUpdatedObservers() } + fun setArchiveThumbnailTransferState(id: AttachmentId, state: ArchiveTransferState) { + check(state != ArchiveTransferState.COPY_PENDING) { "COPY_PENDING is not a valid transfer state for a thumbnail!" } + + writableDatabase.withinTransaction { + val thumbnailFile: String = readableDatabase + .select(THUMBNAIL_FILE) + .from(TABLE_NAME) + .where("$ID = ?", id.id) + .run() + .readToSingleObject { it.requireString(THUMBNAIL_FILE) } ?: return@withinTransaction + + writableDatabase + .update(TABLE_NAME) + .values(ARCHIVE_THUMBNAIL_TRANSFER_STATE to state.value) + .where("$THUMBNAIL_FILE = ?", thumbnailFile) + .run() + } + } + /** * Sets the archive transfer state for the given attachment and all other attachments that share the same data file iff * the row isn't already marked as a [ArchiveTransferState.PERMANENT_FAILURE]. @@ -862,6 +931,29 @@ class AttachmentTable( AppDependencies.databaseObserver.notifyAttachmentUpdatedObservers() } + /** + * Sets the archive thumbnail transfer state for the given attachment and all other attachments that share the same thumbnail file iff + * the row isn't already marked as a [ArchiveTransferState.PERMANENT_FAILURE]. + */ + fun setArchiveThumbnailTransferStateFailure(id: AttachmentId, state: ArchiveTransferState) { + writableDatabase.withinTransaction { + val thumbnailFile: String = readableDatabase + .select(THUMBNAIL_FILE) + .from(TABLE_NAME) + .where("$ID = ?", id.id) + .run() + .readToSingleObject { it.requireString(THUMBNAIL_FILE) } ?: return@withinTransaction + + writableDatabase + .update(TABLE_NAME) + .values(ARCHIVE_THUMBNAIL_TRANSFER_STATE to state.value) + .where("$ARCHIVE_THUMBNAIL_TRANSFER_STATE != ? AND $THUMBNAIL_FILE = ?", ArchiveTransferState.PERMANENT_FAILURE.value, thumbnailFile) + .run() + } + + AppDependencies.databaseObserver.notifyAttachmentUpdatedObservers() + } + /** * Resets the archive upload state by hash/key if we believe the attachment should have been uploaded already. */ @@ -876,6 +968,19 @@ class AttachmentTable( .run() > 0 } + /** + * Resets the archive thumbnail upload state by hash/key if we believe the thumbnail should have been uploaded already. + */ + fun resetArchiveThumbnailTransferStateByPlaintextHashAndRemoteKeyIfNecessary(plaintextHash: ByteArray, remoteKey: ByteArray): Boolean { + return writableDatabase + .update(TABLE_NAME) + .values( + ARCHIVE_THUMBNAIL_TRANSFER_STATE to ArchiveTransferState.NONE.value + ) + .where("$DATA_HASH_END = ? AND $REMOTE_KEY = ? AND $ARCHIVE_THUMBNAIL_TRANSFER_STATE = ${ArchiveTransferState.FINISHED.value}", Base64.encodeWithPadding(plaintextHash), Base64.encodeWithPadding(remoteKey)) + .run() > 0 + } + /** * Sets the archive transfer state for the given attachment and all other attachments that share the same data file. */ @@ -2957,10 +3062,12 @@ class AttachmentTable( } } - private fun buildAttachmentsThatNeedUploadQuery(): String { + private fun buildAttachmentsThatNeedUploadQuery(transferStateFilter: String = "$ARCHIVE_TRANSFER_STATE IN (${ArchiveTransferState.NONE.value}, ${ArchiveTransferState.TEMPORARY_FAILURE.value})"): String { return """ - $ARCHIVE_TRANSFER_STATE IN (${ArchiveTransferState.NONE.value}, ${ArchiveTransferState.TEMPORARY_FAILURE.value}) AND + $transferStateFilter AND $DATA_FILE NOT NULL AND + $REMOTE_KEY NOT NULL AND + $DATA_HASH_END NOT NULL AND $TRANSFER_STATE = $TRANSFER_PROGRESS_DONE AND (${MessageTable.STORY_TYPE} = 0 OR ${MessageTable.STORY_TYPE} IS NULL) AND (${MessageTable.TABLE_NAME}.${MessageTable.EXPIRES_IN} <= 0 OR ${MessageTable.TABLE_NAME}.${MessageTable.EXPIRES_IN} > ${ChatItemArchiveExporter.EXPIRATION_CUTOFF.inWholeMilliseconds}) AND @@ -3069,47 +3176,53 @@ class AttachmentTable( } fun debugGetAttachmentStats(): DebugAttachmentStats { - val count = readableDatabase.count().from(TABLE_NAME).run().readToSingleLong(0) + val totalAttachmentRows = readableDatabase.count().from(TABLE_NAME).run().readToSingleLong(0) + val totalEligibleForUploadRows = getAttachmentsThatWillBeIncludedInArchive().count - val transferStates = mapOf( - TRANSFER_PROGRESS_DONE to "TRANSFER_PROGRESS_DONE", - TRANSFER_PROGRESS_STARTED to "TRANSFER_PROGRESS_STARTED", - TRANSFER_PROGRESS_PENDING to "TRANSFER_PROGRESS_PENDING", - TRANSFER_PROGRESS_FAILED to "TRANSFER_PROGRESS_FAILED", - TRANSFER_PROGRESS_PERMANENT_FAILURE to "TRANSFER_PROGRESS_PERMANENT_FAILURE", - TRANSFER_NEEDS_RESTORE to "TRANSFER_NEEDS_RESTORE", - TRANSFER_RESTORE_IN_PROGRESS to "TRANSFER_RESTORE_IN_PROGRESS", - TRANSFER_RESTORE_OFFLOADED to "TRANSFER_RESTORE_OFFLOADED" + val totalUniqueDataFiles = readableDatabase.select("COUNT(DISTINCT $DATA_FILE)").from(TABLE_NAME).run().readToSingleLong(0) + val totalUniqueMediaNames = readableDatabase.query("SELECT COUNT(*) FROM (SELECT DISTINCT $DATA_HASH_END, $REMOTE_KEY FROM $TABLE_NAME WHERE $DATA_HASH_END NOT NULL AND $REMOTE_KEY NOT NULL)").readToSingleLong(0) + + val totalUniqueMediaNamesEligibleForUpload = readableDatabase.query( + """ + SELECT COUNT(*) FROM ( + SELECT DISTINCT $DATA_HASH_END, $REMOTE_KEY + FROM $TABLE_NAME LEFT JOIN ${MessageTable.TABLE_NAME} ON $TABLE_NAME.$MESSAGE_ID = ${MessageTable.TABLE_NAME}.${MessageTable.ID} + WHERE ${buildAttachmentsThatNeedUploadQuery(transferStateFilter = "$ARCHIVE_TRANSFER_STATE != ${ArchiveTransferState.PERMANENT_FAILURE.value}")} + ) + """ ) + .readToSingleLong(0) - val transferStateCounts = transferStates - .map { (state, name) -> name to readableDatabase.count().from(TABLE_NAME).where("$TRANSFER_STATE = $state AND $DATA_HASH_END NOT NULL AND $REMOTE_KEY NOT NULL").run().readToSingleLong(-1L) } - .toMap() + val archiveStatusMediaNameCounts: Map = ArchiveTransferState.entries.associateWith { state -> + readableDatabase.query( + """ + SELECT COUNT(*) FROM ( + SELECT DISTINCT $DATA_HASH_END, $REMOTE_KEY + FROM $TABLE_NAME LEFT JOIN ${MessageTable.TABLE_NAME} ON $TABLE_NAME.$MESSAGE_ID = ${MessageTable.TABLE_NAME}.${MessageTable.ID} + WHERE ${buildAttachmentsThatNeedUploadQuery(transferStateFilter = "$ARCHIVE_TRANSFER_STATE = ${state.value}")} + ) + """ + ) + .readToSingleLong(0) + } - val validForArchiveTransferStateCounts = transferStates - .map { (state, name) -> name to readableDatabase.count().from(TABLE_NAME).where("$TRANSFER_STATE = $state AND $DATA_HASH_END NOT NULL AND $REMOTE_KEY NOT NULL AND $DATA_FILE NOT NULL").run().readToSingleLong(-1L) } - .toMap() + val uniqueEligibleMediaNamesWithThumbnailsCount = readableDatabase.query("SELECT COUNT(*) FROM (SELECT DISTINCT $DATA_HASH_END, $REMOTE_KEY FROM $TABLE_NAME WHERE $DATA_HASH_END NOT NULL AND $REMOTE_KEY NOT NULL AND $THUMBNAIL_FILE NOT NULL)").readToSingleLong(-1L) + val archiveStatusMediaNameThumbnailCounts: Map = ArchiveTransferState.entries.associateWith { state -> + readableDatabase.query( + """ + SELECT COUNT(*) FROM ( + SELECT DISTINCT $DATA_HASH_END, $REMOTE_KEY + FROM $TABLE_NAME LEFT JOIN ${MessageTable.TABLE_NAME} ON $TABLE_NAME.$MESSAGE_ID = ${MessageTable.TABLE_NAME}.${MessageTable.ID} + WHERE + ${buildAttachmentsThatNeedUploadQuery(transferStateFilter = "$ARCHIVE_THUMBNAIL_TRANSFER_STATE = ${state.value}")} + AND ($CONTENT_TYPE LIKE 'image%' OR $CONTENT_TYPE LIKE 'video%') + ) + """ + ) + .readToSingleLong(0) + } - val archiveStateCounts = ArchiveTransferState - .entries - .associate { it to readableDatabase.count().from(TABLE_NAME).where("$ARCHIVE_TRANSFER_STATE = ${it.value} AND $DATA_HASH_END NOT NULL AND $REMOTE_KEY NOT NULL").run().readToSingleLong(-1L) } - - val attachmentFileCount = readableDatabase.query("SELECT COUNT(DISTINCT $DATA_FILE) FROM $TABLE_NAME WHERE $DATA_FILE NOT NULL AND $DATA_HASH_END NOT NULL AND $REMOTE_KEY NOT NULL").readToSingleLong(-1L) - val finishedAttachmentFileCount = - readableDatabase.query("SELECT COUNT(DISTINCT $DATA_FILE) FROM $TABLE_NAME WHERE $DATA_FILE NOT NULL AND $DATA_HASH_END NOT NULL AND $REMOTE_KEY NOT NULL AND $ARCHIVE_TRANSFER_STATE = ${ArchiveTransferState.FINISHED.value}") - .readToSingleLong(-1L) - val attachmentPlaintextHashAndKeyCount = - readableDatabase.query("SELECT COUNT(*) FROM (SELECT DISTINCT $DATA_HASH_END, $REMOTE_KEY FROM $TABLE_NAME WHERE $DATA_HASH_END NOT NULL AND $REMOTE_KEY NOT NULL AND $TRANSFER_STATE in ($TRANSFER_PROGRESS_DONE, $TRANSFER_RESTORE_OFFLOADED, $TRANSFER_RESTORE_IN_PROGRESS, $TRANSFER_NEEDS_RESTORE))") - .readToSingleLong(-1L) - val finishedAttachmentDigestCount = - readableDatabase.query("SELECT COUNT(*) FROM (SELECT DISTINCT $DATA_HASH_END, $REMOTE_KEY FROM $TABLE_NAME WHERE $DATA_HASH_END NOT NULL AND $REMOTE_KEY NOT NULL AND $ARCHIVE_TRANSFER_STATE = ${ArchiveTransferState.FINISHED.value})") - .readToSingleLong(-1L) - val thumbnailFileCount = readableDatabase.query("SELECT COUNT(DISTINCT $THUMBNAIL_FILE) FROM $TABLE_NAME WHERE $THUMBNAIL_FILE IS NOT NULL").readToSingleLong(-1L) - val estimatedThumbnailCount = - readableDatabase.query("SELECT COUNT(*) FROM (SELECT DISTINCT $DATA_HASH_END, $REMOTE_KEY FROM $TABLE_NAME WHERE $ARCHIVE_TRANSFER_STATE = ${ArchiveTransferState.FINISHED.value} AND $DATA_HASH_END NOT NULL AND $REMOTE_KEY NOT NULL AND ($CONTENT_TYPE LIKE 'image/%' OR $CONTENT_TYPE LIKE 'video/%'))") - .readToSingleLong(-1L) - - val pendingUploadBytes = getPendingArchiveUploadBytes() + val pendingAttachmentUploadBytes = getPendingArchiveUploadBytes() val uploadedAttachmentBytes = readableDatabase .rawQuery( """ @@ -3128,22 +3241,21 @@ class AttachmentTable( .readToList { it.requireLong(DATA_SIZE) } .sumOf { AttachmentCipherStreamUtil.getCiphertextLength(AttachmentCipherStreamUtil.getCiphertextLength(PaddingInputStream.getPaddedSize(it))) } - val uploadedThumbnailBytes = estimatedThumbnailCount * RemoteConfig.backupMaxThumbnailFileSize.inWholeBytes + val uploadedThumbnailCount = archiveStatusMediaNameThumbnailCounts.getOrDefault(ArchiveTransferState.FINISHED, 0L) + val uploadedThumbnailBytes = uploadedThumbnailCount * RemoteConfig.backupMaxThumbnailFileSize.inWholeBytes return DebugAttachmentStats( - attachmentCount = count, - transferStateCounts = transferStateCounts, - validForArchiveTransferStateCounts = validForArchiveTransferStateCounts, - archiveStateCounts = archiveStateCounts, - attachmentFileCount = attachmentFileCount, - finishedAttachmentFileCount = finishedAttachmentFileCount, - attachmentPlaintextHashAndKeyCount = attachmentPlaintextHashAndKeyCount, - finishedAttachmentPlaintextHashAndKeyCount = finishedAttachmentDigestCount, - thumbnailFileCount = thumbnailFileCount, - estimatedThumbnailCount = estimatedThumbnailCount, - pendingUploadBytes = pendingUploadBytes, + totalAttachmentRows = totalAttachmentRows, + totalEligibleForUploadRows = totalEligibleForUploadRows.toLong(), + totalUniqueMediaNamesEligibleForUpload = totalUniqueMediaNamesEligibleForUpload, + totalUniqueDataFiles = totalUniqueDataFiles, + totalUniqueMediaNames = totalUniqueMediaNames, + archiveStatusMediaNameCounts = archiveStatusMediaNameCounts, + mediaNamesWithThumbnailsCount = uniqueEligibleMediaNamesWithThumbnailsCount, + archiveStatusMediaNameThumbnailCounts = archiveStatusMediaNameThumbnailCounts, + pendingAttachmentUploadBytes = pendingAttachmentUploadBytes, uploadedAttachmentBytes = uploadedAttachmentBytes, - thumbnailBytes = uploadedThumbnailBytes + uploadedThumbnailBytes = uploadedThumbnailBytes ) } @@ -3528,20 +3640,24 @@ class AttachmentTable( } data class DebugAttachmentStats( - val attachmentCount: Long = 0L, - val transferStateCounts: Map = emptyMap(), - val archiveStateCounts: Map = emptyMap(), - val attachmentFileCount: Long = 0L, - val finishedAttachmentFileCount: Long = 0L, - val attachmentPlaintextHashAndKeyCount: Long = 0L, - val finishedAttachmentPlaintextHashAndKeyCount: Long, - val thumbnailFileCount: Long = 0L, - val pendingUploadBytes: Long = 0L, + val totalAttachmentRows: Long = 0L, + val totalEligibleForUploadRows: Long = 0L, + val totalUniqueMediaNamesEligibleForUpload: Long = 0L, + val totalUniqueDataFiles: Long = 0L, + val totalUniqueMediaNames: Long = 0L, + val archiveStatusMediaNameCounts: Map = emptyMap(), + val mediaNamesWithThumbnailsCount: Long = 0L, + val archiveStatusMediaNameThumbnailCounts: Map = emptyMap(), + val pendingAttachmentUploadBytes: Long = 0L, val uploadedAttachmentBytes: Long = 0L, - val thumbnailBytes: Long = 0L, - val validForArchiveTransferStateCounts: Map, - val estimatedThumbnailCount: Long - ) + val uploadedThumbnailBytes: Long = 0L + ) { + val uploadedAttachmentCount get() = archiveStatusMediaNameCounts.getOrDefault(ArchiveTransferState.FINISHED, 0L) + val uploadedThumbnailCount get() = archiveStatusMediaNameThumbnailCounts.getOrDefault(ArchiveTransferState.FINISHED, 0L) + + val totalUploadCount get() = uploadedAttachmentCount + uploadedThumbnailCount + val totalUploadBytes get() = uploadedAttachmentBytes + uploadedThumbnailBytes + } data class CreateRemoteKeyResult(val totalCount: Int, val notQuoteOrSickerDupeNotFoundCount: Int, val notQuoteOrSickerDupeFoundCount: Int) { val unexpectedKeyCreation = notQuoteOrSickerDupeFoundCount > 0 || notQuoteOrSickerDupeNotFoundCount > 0 diff --git a/app/src/main/java/org/thoughtcrime/securesms/database/helpers/SignalDatabaseMigrations.kt b/app/src/main/java/org/thoughtcrime/securesms/database/helpers/SignalDatabaseMigrations.kt index f5e7766b5a..2de459e87e 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/database/helpers/SignalDatabaseMigrations.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/database/helpers/SignalDatabaseMigrations.kt @@ -144,6 +144,7 @@ import org.thoughtcrime.securesms.database.helpers.migration.V286_FixRemoteKeyEn import org.thoughtcrime.securesms.database.helpers.migration.V287_FixInvalidArchiveState import org.thoughtcrime.securesms.database.helpers.migration.V288_CopyStickerDataHashStartToEnd import org.thoughtcrime.securesms.database.helpers.migration.V289_AddQuoteTargetContentTypeColumn +import org.thoughtcrime.securesms.database.helpers.migration.V290_AddArchiveThumbnailTransferStateColumn import org.thoughtcrime.securesms.database.SQLiteDatabase as SignalSqliteDatabase /** @@ -293,10 +294,11 @@ object SignalDatabaseMigrations { 286 to V286_FixRemoteKeyEncoding, 287 to V287_FixInvalidArchiveState, 288 to V288_CopyStickerDataHashStartToEnd, - 289 to V289_AddQuoteTargetContentTypeColumn + 289 to V289_AddQuoteTargetContentTypeColumn, + 290 to V290_AddArchiveThumbnailTransferStateColumn ) - const val DATABASE_VERSION = 289 + const val DATABASE_VERSION = 290 @JvmStatic fun migrate(context: Application, db: SignalSqliteDatabase, oldVersion: Int, newVersion: Int) { diff --git a/app/src/main/java/org/thoughtcrime/securesms/database/helpers/migration/V290_AddArchiveThumbnailTransferStateColumn.kt b/app/src/main/java/org/thoughtcrime/securesms/database/helpers/migration/V290_AddArchiveThumbnailTransferStateColumn.kt new file mode 100644 index 0000000000..ef1661148b --- /dev/null +++ b/app/src/main/java/org/thoughtcrime/securesms/database/helpers/migration/V290_AddArchiveThumbnailTransferStateColumn.kt @@ -0,0 +1,20 @@ +/* + * Copyright 2024 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ + +package org.thoughtcrime.securesms.database.helpers.migration + +import android.app.Application +import org.thoughtcrime.securesms.database.SQLiteDatabase + +/** + * We need to keep track of a transfer state for thumbnails too. + */ +@Suppress("ClassName") +object V290_AddArchiveThumbnailTransferStateColumn : SignalDatabaseMigration { + + override fun migrate(context: Application, db: SQLiteDatabase, oldVersion: Int, newVersion: Int) { + db.execSQL("ALTER TABLE attachment ADD COLUMN archive_thumbnail_transfer_state INTEGER DEFAULT 0;") + } +} diff --git a/app/src/main/java/org/thoughtcrime/securesms/jobmanager/Job.java b/app/src/main/java/org/thoughtcrime/securesms/jobmanager/Job.java index c81b765989..4d9c63b82c 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/jobmanager/Job.java +++ b/app/src/main/java/org/thoughtcrime/securesms/jobmanager/Job.java @@ -279,11 +279,13 @@ public abstract class Job { public static final int UNLIMITED = -1; @Retention(SOURCE) - @IntDef({ PRIORITY_DEFAULT, PRIORITY_LOW, PRIORITY_HIGH}) + @IntDef({ PRIORITY_DEFAULT, PRIORITY_LOW, PRIORITY_LOWER, PRIORITY_HIGH}) public @interface Priority{} public static final int PRIORITY_DEFAULT = 0; public static final int PRIORITY_HIGH = 1; public static final int PRIORITY_LOW = -1; + /** One step lower than {@link #PRIORITY_LOW} */ + public static final int PRIORITY_LOWER = -2; private final String id; private final long createTime; diff --git a/app/src/main/java/org/thoughtcrime/securesms/jobs/ArchiveAttachmentReconciliationJob.kt b/app/src/main/java/org/thoughtcrime/securesms/jobs/ArchiveAttachmentReconciliationJob.kt index 81edc17a4a..71868ada21 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/jobs/ArchiveAttachmentReconciliationJob.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/jobs/ArchiveAttachmentReconciliationJob.kt @@ -12,10 +12,8 @@ import org.thoughtcrime.securesms.backup.v2.ArchivedMediaObject import org.thoughtcrime.securesms.backup.v2.BackupRepository import org.thoughtcrime.securesms.database.BackupMediaSnapshotTable import org.thoughtcrime.securesms.database.SignalDatabase -import org.thoughtcrime.securesms.dependencies.AppDependencies import org.thoughtcrime.securesms.jobmanager.Job import org.thoughtcrime.securesms.jobmanager.impl.NetworkConstraint -import org.thoughtcrime.securesms.jobs.ArchiveThumbnailUploadJob.Companion.isForArchiveThumbnailUploadJob import org.thoughtcrime.securesms.jobs.protos.ArchiveAttachmentReconciliationJobData import org.thoughtcrime.securesms.keyvalue.SignalStore import org.thoughtcrime.securesms.util.RemoteConfig @@ -142,18 +140,12 @@ class ArchiveAttachmentReconciliationJob private constructor( val entry = BackupMediaSnapshotTable.MediaEntry.fromCursor(mediaObjectCursor) if (entry.isThumbnail) { - val parentAttachmentId = SignalDatabase.attachments.getAttachmentIdByPlaintextHashAndRemoteKey(entry.plaintextHash, entry.remoteKey) - if (parentAttachmentId == null) { - Log.w(TAG, "Failed to find parent attachment for thumbnail that may need reupload. Skipping.", true) - return@forEach - } - - if (AppDependencies.jobManager.find { it.isForArchiveThumbnailUploadJob(parentAttachmentId) }.isEmpty()) { - Log.w(TAG, "A thumbnail was missing from remote for $parentAttachmentId and no in-progress job was found. Re-enqueueing one.", true) - ArchiveThumbnailUploadJob.enqueueIfNecessary(parentAttachmentId) + val wasReset = SignalDatabase.attachments.resetArchiveThumbnailTransferStateByPlaintextHashAndRemoteKeyIfNecessary(entry.plaintextHash, entry.remoteKey) + if (wasReset) { + newBackupJobRequired = true bookkeepingErrorCount++ } else { - Log.i(TAG, "A thumbnail was missing from remote for $parentAttachmentId, but a job is already in progress.", true) + Log.w(TAG, "[Thumbnail] Did not need to reset the transfer state by hash/key because the thumbnail either no longer exists or the upload is already in-progress.", true) } } else { val wasReset = SignalDatabase.attachments.resetArchiveTransferStateByPlaintextHashAndRemoteKeyIfNecessary(entry.plaintextHash, entry.remoteKey) @@ -161,7 +153,7 @@ class ArchiveAttachmentReconciliationJob private constructor( newBackupJobRequired = true bookkeepingErrorCount++ } else { - Log.w(TAG, "Did not need to reset the the transfer state by hash/key because the attachment either no longer exists or the upload is already in-progress.", true) + Log.w(TAG, "[Fullsize] Did not need to reset the the transfer state by hash/key because the attachment either no longer exists or the upload is already in-progress.", true) } } } diff --git a/app/src/main/java/org/thoughtcrime/securesms/jobs/ArchiveThumbnailBackfillJob.kt b/app/src/main/java/org/thoughtcrime/securesms/jobs/ArchiveThumbnailBackfillJob.kt new file mode 100644 index 0000000000..945dad972b --- /dev/null +++ b/app/src/main/java/org/thoughtcrime/securesms/jobs/ArchiveThumbnailBackfillJob.kt @@ -0,0 +1,64 @@ +/* + * Copyright 2024 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ + +package org.thoughtcrime.securesms.jobs + +import org.signal.core.util.logging.Log +import org.thoughtcrime.securesms.database.SignalDatabase +import org.thoughtcrime.securesms.dependencies.AppDependencies +import org.thoughtcrime.securesms.jobmanager.Job +import org.thoughtcrime.securesms.keyvalue.SignalStore +import kotlin.time.Duration.Companion.days + +/** + * When run, this will find all of the thumbnails that need to be uploaded to the archive tier and enqueue [ArchiveThumbnailUploadJob]s for them. + */ +class ArchiveThumbnailBackfillJob private constructor(parameters: Parameters) : Job(parameters) { + companion object { + private val TAG = Log.tag(ArchiveThumbnailBackfillJob::class.java) + + const val KEY = "ArchiveThumbnailBackfillJob" + } + + constructor() : this( + parameters = Parameters.Builder() + .setQueue(ArchiveCommitAttachmentDeletesJob.ARCHIVE_ATTACHMENT_QUEUE) + .setMaxInstancesForQueue(2) + .setLifespan(30.days.inWholeMilliseconds) + .setMaxAttempts(Parameters.UNLIMITED) + .build() + ) + + override fun serialize(): ByteArray? = null + + override fun getFactoryKey(): String = KEY + + override fun run(): Result { + if (!SignalStore.backup.backsUpMedia) { + Log.w(TAG, "This user doesn't back up media! Skipping. Tier: ${SignalStore.backup.backupTier}") + return Result.success() + } + + val jobs = SignalDatabase.attachments.getThumbnailsThatNeedArchiveUpload() + .map { attachmentId -> ArchiveThumbnailUploadJob(attachmentId) } + + if (!isCanceled) { + Log.i(TAG, "Adding ${jobs.size} jobs to backfill thumbnails.") + AppDependencies.jobManager.addAll(jobs) + } else { + Log.w(TAG, "Job was canceled. Not enqueuing backfill.") + } + + return Result.success() + } + + override fun onFailure() = Unit + + class Factory : Job.Factory { + override fun create(parameters: Parameters, serializedData: ByteArray?): ArchiveThumbnailBackfillJob { + return ArchiveThumbnailBackfillJob(parameters) + } + } +} diff --git a/app/src/main/java/org/thoughtcrime/securesms/jobs/ArchiveThumbnailUploadJob.kt b/app/src/main/java/org/thoughtcrime/securesms/jobs/ArchiveThumbnailUploadJob.kt index 7d4e098975..cee9cc1300 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/jobs/ArchiveThumbnailUploadJob.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/jobs/ArchiveThumbnailUploadJob.kt @@ -14,10 +14,12 @@ import org.thoughtcrime.securesms.attachments.PointerAttachment import org.thoughtcrime.securesms.backup.v2.BackupRepository import org.thoughtcrime.securesms.backup.v2.hadIntegrityCheckPerformed import org.thoughtcrime.securesms.backup.v2.requireThumbnailMediaName +import org.thoughtcrime.securesms.database.AttachmentTable import org.thoughtcrime.securesms.database.SignalDatabase import org.thoughtcrime.securesms.dependencies.AppDependencies import org.thoughtcrime.securesms.jobmanager.Job import org.thoughtcrime.securesms.jobmanager.impl.NetworkConstraint +import org.thoughtcrime.securesms.jobmanager.impl.NoRemoteArchiveGarbageCollectionPendingConstraint import org.thoughtcrime.securesms.jobmanager.persistence.JobSpec import org.thoughtcrime.securesms.jobs.protos.ArchiveThumbnailUploadJobData import org.thoughtcrime.securesms.keyvalue.SignalStore @@ -58,7 +60,9 @@ class ArchiveThumbnailUploadJob private constructor( /** A set of possible queues this job may use. The number of queues determines the parallelism. */ val QUEUES = setOf( "ArchiveThumbnailUploadJob_1", - "ArchiveThumbnailUploadJob_2" + "ArchiveThumbnailUploadJob_2", + "ArchiveThumbnailUploadJob_3", + "ArchiveThumbnailUploadJob_4" ) fun enqueueIfNecessary(attachmentId: AttachmentId) { @@ -72,13 +76,14 @@ class ArchiveThumbnailUploadJob private constructor( } } - private constructor(attachmentId: AttachmentId) : this( + constructor(attachmentId: AttachmentId) : this( Parameters.Builder() .setQueue(QUEUES.random()) .addConstraint(NetworkConstraint.KEY) + .addConstraint(NoRemoteArchiveGarbageCollectionPendingConstraint.KEY) .setLifespan(1.days.inWholeMilliseconds) .setMaxAttempts(Parameters.UNLIMITED) - .setGlobalPriority(Parameters.PRIORITY_LOW) + .setGlobalPriority(Parameters.PRIORITY_LOWER) .build(), attachmentId ) @@ -91,6 +96,14 @@ class ArchiveThumbnailUploadJob private constructor( override fun getFactoryKey(): String = KEY + override fun onAdded() { + val transferStatus = SignalDatabase.attachments.getArchiveThumbnailTransferState(attachmentId) ?: return + + if (transferStatus == AttachmentTable.ArchiveTransferState.NONE) { + SignalDatabase.attachments.setArchiveThumbnailTransferState(attachmentId, AttachmentTable.ArchiveTransferState.UPLOAD_IN_PROGRESS) + } + } + override fun run(): Result { val attachment = SignalDatabase.attachments.getAttachment(attachmentId) if (attachment == null) { @@ -100,26 +113,31 @@ class ArchiveThumbnailUploadJob private constructor( if (!MediaUtil.isImageOrVideoType(attachment.contentType)) { Log.w(TAG, "$attachmentId isn't visual media (contentType = ${attachment.contentType}). Skipping.") + SignalDatabase.attachments.setArchiveThumbnailTransferState(attachmentId, AttachmentTable.ArchiveTransferState.NONE) return Result.success() } if (attachment.quote) { Log.w(TAG, "$attachmentId is a quote. Skipping.") + SignalDatabase.attachments.setArchiveThumbnailTransferState(attachmentId, AttachmentTable.ArchiveTransferState.NONE) return Result.success() } if (attachment.dataHash == null || attachment.remoteKey == null) { Log.w(TAG, "$attachmentId is missing necessary ingredients for a mediaName!") + SignalDatabase.attachments.setArchiveThumbnailTransferState(attachmentId, AttachmentTable.ArchiveTransferState.NONE) return Result.success() } if (!attachment.hadIntegrityCheckPerformed()) { Log.w(TAG, "$attachmentId has no integrity check! Cannot proceed.") + SignalDatabase.attachments.setArchiveThumbnailTransferState(attachmentId, AttachmentTable.ArchiveTransferState.NONE) return Result.success() } if (SignalDatabase.messages.isStory(attachment.mmsId)) { Log.w(TAG, "$attachmentId is a story. Skipping.") + SignalDatabase.attachments.setArchiveThumbnailTransferState(attachmentId, AttachmentTable.ArchiveTransferState.NONE) return Result.success() } @@ -128,10 +146,12 @@ class ArchiveThumbnailUploadJob private constructor( val thumbnailResult = generateThumbnailIfPossible(attachment) if (thumbnailResult == null) { Log.w(TAG, "Unable to generate a thumbnail result for $attachmentId") + SignalDatabase.attachments.setArchiveThumbnailTransferState(attachmentId, AttachmentTable.ArchiveTransferState.PERMANENT_FAILURE) return Result.success() } if (isCanceled) { + SignalDatabase.attachments.setArchiveThumbnailTransferState(attachmentId, AttachmentTable.ArchiveTransferState.TEMPORARY_FAILURE) return Result.failure() } @@ -148,6 +168,7 @@ class ArchiveThumbnailUploadJob private constructor( } if (isCanceled) { + SignalDatabase.attachments.setArchiveThumbnailTransferState(attachmentId, AttachmentTable.ArchiveTransferState.TEMPORARY_FAILURE) return Result.failure() } @@ -174,6 +195,7 @@ class ArchiveThumbnailUploadJob private constructor( } if (isCanceled) { + SignalDatabase.attachments.setArchiveThumbnailTransferState(attachmentId, AttachmentTable.ArchiveTransferState.TEMPORARY_FAILURE) return Result.failure() } @@ -202,6 +224,7 @@ class ArchiveThumbnailUploadJob private constructor( ) Log.d(TAG, "Successfully archived thumbnail for $attachmentId") + SignalDatabase.attachments.setArchiveThumbnailTransferState(attachmentId, AttachmentTable.ArchiveTransferState.FINISHED) Result.success() } @@ -220,6 +243,13 @@ class ArchiveThumbnailUploadJob private constructor( } override fun onFailure() { + if (this.isCanceled) { + Log.w(TAG, "[$attachmentId] Job was canceled, updating archive thumbnail transfer state to ${AttachmentTable.ArchiveTransferState.NONE}.") + SignalDatabase.attachments.setArchiveThumbnailTransferState(attachmentId, AttachmentTable.ArchiveTransferState.NONE) + } else { + Log.w(TAG, "[$attachmentId] Job failed, updating archive thumbnail transfer state to ${AttachmentTable.ArchiveTransferState.TEMPORARY_FAILURE} (if not already a permanent failure).") + SignalDatabase.attachments.setArchiveThumbnailTransferStateFailure(attachmentId, AttachmentTable.ArchiveTransferState.TEMPORARY_FAILURE) + } } private fun generateThumbnailIfPossible(attachment: DatabaseAttachment): ImageCompressionUtil.Result? { diff --git a/app/src/main/java/org/thoughtcrime/securesms/jobs/BackupMessagesJob.kt b/app/src/main/java/org/thoughtcrime/securesms/jobs/BackupMessagesJob.kt index f9630ba29b..552e05e206 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/jobs/BackupMessagesJob.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/jobs/BackupMessagesJob.kt @@ -318,6 +318,13 @@ class BackupMessagesJob private constructor( ArchiveUploadProgress.onMessageBackupFinishedEarly() } + if (SignalStore.backup.backsUpMedia && SignalDatabase.attachments.doAnyThumbnailsNeedArchiveUpload()) { + Log.i(TAG, "Enqueuing thumbnail backfill job.") + AppDependencies.jobManager.add(ArchiveThumbnailBackfillJob()) + } else { + Log.i(TAG, "No thumbnails need to be uploaded: ${SignalStore.backup.backupTier}") + } + BackupRepository.clearBackupFailure() SignalDatabase.backupMediaSnapshots.commitPendingRows() @@ -413,7 +420,7 @@ class BackupMessagesJob private constructor( private fun writeMediaCursorToTemporaryTable(db: SignalDatabase, mediaBackupEnabled: Boolean) { if (mediaBackupEnabled) { - db.attachmentTable.getAttachmentsEligibleForArchiveUpload().use { + db.attachmentTable.getAttachmentsThatWillBeIncludedInArchive().use { SignalDatabase.backupMediaSnapshots.writePendingMediaObjects( mediaObjects = ArchiveMediaItemIterator(it).asSequence() ) diff --git a/app/src/main/java/org/thoughtcrime/securesms/jobs/CopyAttachmentToArchiveJob.kt b/app/src/main/java/org/thoughtcrime/securesms/jobs/CopyAttachmentToArchiveJob.kt index 589f2da55f..2127b3dabf 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/jobs/CopyAttachmentToArchiveJob.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/jobs/CopyAttachmentToArchiveJob.kt @@ -50,6 +50,7 @@ class CopyAttachmentToArchiveJob private constructor(private val attachmentId: A .setMaxAttempts(Parameters.UNLIMITED) .setQueue(UploadAttachmentToArchiveJob.QUEUES.random()) .setQueuePriority(Parameters.PRIORITY_HIGH) + .setGlobalPriority(Parameters.PRIORITY_LOW) .build() ) @@ -145,6 +146,12 @@ class CopyAttachmentToArchiveJob private constructor(private val attachmentId: A is NetworkResult.StatusCodeError -> { when (archiveResult.code) { + 400 -> { + Log.w(TAG, "[$attachmentId] Something is invalid about our request. Possibly the length. Scheduling a re-upload. Body: ${archiveResult.exception.stringBody}") + SignalDatabase.attachments.setArchiveTransferState(attachmentId, AttachmentTable.ArchiveTransferState.NONE) + AppDependencies.jobManager.add(UploadAttachmentToArchiveJob(attachmentId, canReuseUpload = false)) + Result.success() + } 403 -> { Log.w(TAG, "[$attachmentId] Insufficient permissions to upload. Handled in parent handler.") Result.success() diff --git a/app/src/main/java/org/thoughtcrime/securesms/jobs/JobManagerFactories.java b/app/src/main/java/org/thoughtcrime/securesms/jobs/JobManagerFactories.java index edd1b3cadb..7ff3713329 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/jobs/JobManagerFactories.java +++ b/app/src/main/java/org/thoughtcrime/securesms/jobs/JobManagerFactories.java @@ -130,6 +130,7 @@ public final class JobManagerFactories { put(ArchiveAttachmentReconciliationJob.KEY, new ArchiveAttachmentReconciliationJob.Factory()); put(ArchiveBackupIdReservationJob.KEY, new ArchiveBackupIdReservationJob.Factory()); put(ArchiveCommitAttachmentDeletesJob.KEY, new ArchiveCommitAttachmentDeletesJob.Factory()); + put(ArchiveThumbnailBackfillJob.KEY, new ArchiveThumbnailBackfillJob.Factory()); put(ArchiveThumbnailUploadJob.KEY, new ArchiveThumbnailUploadJob.Factory()); put(AttachmentCompressionJob.KEY, new AttachmentCompressionJob.Factory()); put(AttachmentCopyJob.KEY, new AttachmentCopyJob.Factory()); diff --git a/app/src/main/java/org/thoughtcrime/securesms/jobs/RestoreAttachmentJob.kt b/app/src/main/java/org/thoughtcrime/securesms/jobs/RestoreAttachmentJob.kt index f9bc5e8055..b116272e7c 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/jobs/RestoreAttachmentJob.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/jobs/RestoreAttachmentJob.kt @@ -111,7 +111,7 @@ class RestoreAttachmentJob private constructor( /** * Create a restore job for the initial large batch of media on a fresh restore. - * Will enqueue with some amount of parallization with low job priority. + * Will enqueue with some amount of parallelization with low job priority. */ fun forInitialRestore(attachmentId: AttachmentId, messageId: Long, stickerPackId: String?): RestoreAttachmentJob { return RestoreAttachmentJob( diff --git a/app/src/main/java/org/thoughtcrime/securesms/jobs/UploadAttachmentToArchiveJob.kt b/app/src/main/java/org/thoughtcrime/securesms/jobs/UploadAttachmentToArchiveJob.kt index 56e82be4a7..e53b49817e 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/jobs/UploadAttachmentToArchiveJob.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/jobs/UploadAttachmentToArchiveJob.kt @@ -80,6 +80,7 @@ class UploadAttachmentToArchiveJob private constructor( .setLifespan(30.days.inWholeMilliseconds) .setMaxAttempts(Parameters.UNLIMITED) .setQueue(QUEUES.random()) + .setGlobalPriority(Parameters.PRIORITY_LOW) .build() ) diff --git a/app/src/main/java/org/thoughtcrime/securesms/logsubmit/LogSectionRemoteBackups.kt b/app/src/main/java/org/thoughtcrime/securesms/logsubmit/LogSectionRemoteBackups.kt index 9b11c1544e..9506982649 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/logsubmit/LogSectionRemoteBackups.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/logsubmit/LogSectionRemoteBackups.kt @@ -87,19 +87,6 @@ class LogSectionRemoteBackups : LogSection { if (SignalStore.backup.archiveUploadState!!.state !in setOf(ArchiveUploadProgressState.State.None, ArchiveUploadProgressState.State.UserCanceled)) { output.append("Pending bytes: ${SignalDatabase.attachments.getPendingArchiveUploadBytes()}\n") - - val pendingAttachments = SignalDatabase.attachments.debugGetPendingArchiveUploadAttachments() - if (pendingAttachments.isNotEmpty()) { - output.append("Pending attachments:\n") - output.append(" Count: ${pendingAttachments.size}\n") - output.append(" Sum of Size: ${pendingAttachments.sumOf { it.size }}\n") - output.append(" Content types:\n") - pendingAttachments.groupBy { it.contentType }.forEach { (contentType, attachments) -> - output.append(" $contentType: ${attachments.size}\n") - } - } else { - output.append("Pending attachments: None!\n") - } } } else { output.append("None\n") diff --git a/libsignal-service/src/main/java/org/whispersystems/signalservice/api/archive/ArchiveApi.kt b/libsignal-service/src/main/java/org/whispersystems/signalservice/api/archive/ArchiveApi.kt index a012e6ffda..87f518fbb4 100644 --- a/libsignal-service/src/main/java/org/whispersystems/signalservice/api/archive/ArchiveApi.kt +++ b/libsignal-service/src/main/java/org/whispersystems/signalservice/api/archive/ArchiveApi.kt @@ -267,7 +267,7 @@ class ArchiveApi( var cursor: String? = null do { - val response: ArchiveGetMediaItemsResponse = getArchiveMediaItemsPage(aci, archiveServiceAccess, 512, cursor).successOrThrow() + val response: ArchiveGetMediaItemsResponse = getArchiveMediaItemsPage(aci, archiveServiceAccess, 10_000, cursor).successOrThrow() mediaObjects += response.storedMediaObjects cursor = response.cursor } while (cursor != null)