Signal/Android
1
0
Fork 0
mirror of https://github.com/signalapp/Signal-Android.git synced 2025-12-24 04:58:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add separate column to track archive thumbnail status.

Browse Source
This commit is contained in:
Greyson Parrelli
2025-09-04 11:51:22 -04:00
parent 2c47cd2422
commit ca2cc722d4
15 changed files with 423 additions and 162 deletions
Download Patch File Download Diff File Expand all files Collapse all files

146
app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/backup/InternalBackupStatsTab.kt
View File

@@ -17,6 +17,7 @@ import androidx.compose.runtime.Composable
import androidx.compose.ui.Modifier
import androidx.compose.ui.unit.dp
import org.signal.core.ui.compose.Dividers
import org.signal.core.ui.compose.Rows
import org.signal.core.ui.compose.Texts
import org.signal.core.util.bytes
@@ -24,49 +25,65 @@ import org.signal.core.util.bytes
fun InternalBackupStatsTab(stats: InternalBackupPlaygroundViewModel.StatsState, callbacks: StatsCallbacks) {
val scrollState = rememberScrollState()
Column(modifier = Modifier.verticalScroll(scrollState)) {
Texts.SectionHeader(text = "Local Attachment State")
if (stats.attachmentStats != null) {
Text(text = "Attachment Count: ${stats.attachmentStats.attachmentCount}")
Texts.SectionHeader(text = "Local Attachments")
Text(text = "Transit Download State:")
stats.attachmentStats.transferStateCounts.forEach { (state, count) ->
if (count > 0) {
Text(text = "$state: $count")
}
}
Rows.TextRow(
text = "Total attachment rows",
label = "${stats.attachmentStats.totalAttachmentRows}"
)
Text(text = "Valid for archive Transit Download State:")
stats.attachmentStats.validForArchiveTransferStateCounts.forEach { (state, count) ->
if (count > 0) {
Text(text = "$state: $count")
}
}
Rows.TextRow(
text = "Total unique data files",
label = "${stats.attachmentStats.totalUniqueDataFiles}"
)
Spacer(modifier = Modifier.size(4.dp))
Rows.TextRow(
text = "Total unique media names",
label = "${stats.attachmentStats.totalUniqueMediaNames}"
)
Text(text = "Archive State:")
stats.attachmentStats.archiveStateCounts.forEach { (state, count) ->
if (count > 0) {
Text(text = "$state: $count")
}
}
Rows.TextRow(
text = "Total eligible for upload rows",
label = "${stats.attachmentStats.totalEligibleForUploadRows}"
)
Rows.TextRow(
text = "Total unique media names eligible for upload ⭐",
label = "${stats.attachmentStats.totalUniqueMediaNamesEligibleForUpload}"
)
Rows.TextRow(
text = "Eligible attachments by status ⭐",
label = stats.attachmentStats.archiveStatusMediaNameCounts.entries.joinToString("\n") { (status, count) -> "$status: $count" }
)
Rows.TextRow(
text = "Total media names with thumbnails",
label = "${stats.attachmentStats.mediaNamesWithThumbnailsCount}"
)
Rows.TextRow(
text = "Eligible thumbnails by status ⭐",
label = stats.attachmentStats.archiveStatusMediaNameThumbnailCounts.entries.joinToString("\n") { (status, count) -> "$status: $count" }
)
Rows.TextRow(
text = "Pending attachment upload bytes ⭐",
label = "${stats.attachmentStats.pendingAttachmentUploadBytes} (~${stats.attachmentStats.pendingAttachmentUploadBytes.bytes.toUnitString()})"
)
Rows.TextRow(
text = "Uploaded attachment bytes ⭐",
label = "${stats.attachmentStats.uploadedAttachmentBytes} (~${stats.attachmentStats.uploadedAttachmentBytes.bytes.toUnitString()})"
)
Rows.TextRow(
text = "Uploaded thumbnail bytes (estimated)",
label = "${stats.attachmentStats.uploadedThumbnailBytes} (~${stats.attachmentStats.uploadedThumbnailBytes.bytes.toUnitString()})"
)
Spacer(modifier = Modifier.size(16.dp))
Text(text = "Unique/archived data files: ${stats.attachmentStats.attachmentFileCount}/${stats.attachmentStats.finishedAttachmentFileCount}")
Text(text = "Unique/archived verified plaintextHash count: ${stats.attachmentStats.attachmentPlaintextHashAndKeyCount}/${stats.attachmentStats.finishedAttachmentPlaintextHashAndKeyCount}")
Text(text = "Unique/expected thumbnail files: ${stats.attachmentStats.thumbnailFileCount}/${stats.attachmentStats.estimatedThumbnailCount}")
Text(text = "Local Total: ${stats.attachmentStats.attachmentFileCount + stats.attachmentStats.thumbnailFileCount}")
Text(text = "Expected remote total: ${stats.attachmentStats.estimatedThumbnailCount + stats.attachmentStats.finishedAttachmentPlaintextHashAndKeyCount}")
Spacer(modifier = Modifier.size(16.dp))
Text(text = "Pending upload: ${stats.attachmentStats.pendingUploadBytes} (~${stats.attachmentStats.pendingUploadBytes.bytes.toUnitString()})")
Text(text = "Est uploaded attachments: ${stats.attachmentStats.uploadedAttachmentBytes} (~${stats.attachmentStats.uploadedAttachmentBytes.bytes.toUnitString()})")
Text(text = "Est uploaded thumbnails: ${stats.attachmentStats.thumbnailBytes} (~${stats.attachmentStats.thumbnailBytes.bytes.toUnitString()})")
val total = stats.attachmentStats.thumbnailBytes + stats.attachmentStats.uploadedAttachmentBytes
Text(text = "Est total: $total (~${total.bytes.toUnitString()})")
} else {
CircularProgressIndicator()
}
@@ -79,28 +96,43 @@ fun InternalBackupStatsTab(stats: InternalBackupPlaygroundViewModel.StatsState,
Button(onClick = callbacks::loadRemoteState) {
Text(text = "Load remote stats (expensive and long)")
}
} else if (stats.remoteFailureMsg != null) {
Text(text = stats.remoteFailureMsg)
} else if (stats.loadingRemoteStats) {
CircularProgressIndicator()
} else if (stats.remoteState != null) {
Rows.TextRow(
"Total media items ⭐",
label = "${stats.remoteState.mediaCount}"
)
Rows.TextRow(
"Total media size ⭐",
label = "${stats.remoteState.mediaSize} (~${stats.remoteState.mediaSize.bytes.toUnitString()})"
)
Rows.TextRow(
text = "Server estimated used size",
label = "${stats.remoteState.usedSpace} (~${stats.remoteState.usedSpace.bytes.toUnitString()})"
)
}
Dividers.Default()
Texts.SectionHeader(text = "Expected vs Actual")
if (stats.attachmentStats != null && stats.remoteState != null) {
Rows.TextRow(
text = "Counts ⭐",
label = "Local: ${stats.attachmentStats.totalUploadCount}\nRemote: ${stats.remoteState.mediaCount}"
)
Rows.TextRow(
text = "Bytes ⭐",
label = "Local: ${stats.attachmentStats.totalUploadBytes} (~${stats.attachmentStats.totalUploadBytes.bytes.toUnitString()}, thumbnails are estimated)\nRemote: ${stats.remoteState.mediaSize} (~${stats.remoteState.mediaSize.bytes.toUnitString()})"
)
} else {
if (stats.loadingRemoteStats) {
CircularProgressIndicator()
} else if (stats.remoteState != null) {
Text(text = "Media item count: ${stats.remoteState.mediaCount}")
Text(text = "Media items sum size: ${stats.remoteState.mediaSize} (~${stats.remoteState.mediaSize.bytes.toUnitString()})")
Text(text = "Server estimated used size: ${stats.remoteState.usedSpace} (~${stats.remoteState.usedSpace.bytes.toUnitString()})")
} else if (stats.remoteFailureMsg != null) {
Text(text = stats.remoteFailureMsg)
}
Dividers.Default()
Texts.SectionHeader(text = "Expected vs Actual")
if (stats.attachmentStats != null && stats.remoteState != null) {
val finished = stats.attachmentStats.finishedAttachmentFileCount
val thumbnails = stats.attachmentStats.thumbnailFileCount
Text(text = "Expected Count/Actual Remote Count: ${finished + thumbnails} / ${stats.remoteState.mediaCount}")
} else {
CircularProgressIndicator()
}
CircularProgressIndicator()
}
}
}

256
app/src/main/java/org/thoughtcrime/securesms/database/AttachmentTable.kt
View File

@@ -177,6 +177,7 @@ class AttachmentTable(
const val UPLOAD_TIMESTAMP = "upload_timestamp"
const val ARCHIVE_CDN = "archive_cdn"
const val ARCHIVE_TRANSFER_STATE = "archive_transfer_state"
const val ARCHIVE_THUMBNAIL_TRANSFER_STATE = "archive_thumbnail_transfer_state"
const val THUMBNAIL_RESTORE_STATE = "thumbnail_restore_state"
const val ATTACHMENT_UUID = "attachment_uuid"
const val OFFLOAD_RESTORED_AT = "offload_restored_at"
@@ -282,7 +283,8 @@ class AttachmentTable(
$THUMBNAIL_RESTORE_STATE INTEGER DEFAULT ${ThumbnailRestoreState.NONE.value},
$ATTACHMENT_UUID TEXT DEFAULT NULL,
$OFFLOAD_RESTORED_AT INTEGER DEFAULT 0,
$QUOTE_TARGET_CONTENT_TYPE TEXT DEFAULT NULL
$QUOTE_TARGET_CONTENT_TYPE TEXT DEFAULT NULL,
$ARCHIVE_THUMBNAIL_TRANSFER_STATE INTEGER DEFAULT ${ArchiveTransferState.NONE.value}
)
"""
@@ -420,14 +422,14 @@ class AttachmentTable(
}
/**
* Returns a cursor (with just the plaintextHash+remoteKey+archive_cdn) for all attachments that are eligible for archive upload.
* In practice, this means that the attachments have a plaintextHash and have not hit a permanent archive upload failure.
* Returns a cursor (with just the plaintextHash+remoteKey+archive_cdn) for all attachments that are slated to be included in the current archive upload.
* Used for snapshotting data in [BackupMediaSnapshotTable].
*/
fun getAttachmentsEligibleForArchiveUpload(): Cursor {
fun getAttachmentsThatWillBeIncludedInArchive(): Cursor {
return readableDatabase
.select(DATA_HASH_END, REMOTE_KEY, ARCHIVE_CDN, QUOTE, CONTENT_TYPE)
.from(TABLE_NAME)
.where("$DATA_HASH_END NOT NULL AND $REMOTE_KEY NOT NULL AND $ARCHIVE_TRANSFER_STATE != ${ArchiveTransferState.PERMANENT_FAILURE.value}")
.from("$TABLE_NAME LEFT JOIN ${MessageTable.TABLE_NAME} ON $TABLE_NAME.$MESSAGE_ID = ${MessageTable.TABLE_NAME}.${MessageTable.ID}")
.where(buildAttachmentsThatNeedUploadQuery(transferStateFilter = "$ARCHIVE_TRANSFER_STATE != ${ArchiveTransferState.PERMANENT_FAILURE.value}"))
.run()
}
@@ -805,6 +807,42 @@ class AttachmentTable(
.run()
}
/**
* Returns whether or not there are thumbnails that need to be uploaded to the archive.
*/
fun doAnyThumbnailsNeedArchiveUpload(): Boolean {
return readableDatabase
.exists(TABLE_NAME)
.where(
"""
$ARCHIVE_TRANSFER_STATE = ${ArchiveTransferState.FINISHED.value} AND
$ARCHIVE_THUMBNAIL_TRANSFER_STATE = ${ArchiveTransferState.NONE.value} AND
$QUOTE = 0 AND
($CONTENT_TYPE LIKE 'image%' OR $CONTENT_TYPE LIKE 'video%')
"""
)
.run()
}
/**
* Returns whether or not there are thumbnails that need to be uploaded to the archive.
*/
fun getThumbnailsThatNeedArchiveUpload(): List<AttachmentId> {
return readableDatabase
.select(ID)
.from(TABLE_NAME)
.where(
"""
$ARCHIVE_TRANSFER_STATE = ${ArchiveTransferState.FINISHED.value} AND
$ARCHIVE_THUMBNAIL_TRANSFER_STATE = ${ArchiveTransferState.NONE.value} AND
$QUOTE = 0 AND
($CONTENT_TYPE LIKE 'image%' OR $CONTENT_TYPE LIKE 'video%')
"""
)
.run()
.readToList { AttachmentId(it.requireLong(ID)) }
}
/**
* Returns the current archive transfer state, if the attachment can be found.
*/
@@ -817,6 +855,18 @@ class AttachmentTable(
.readToSingleObject { ArchiveTransferState.deserialize(it.requireInt(ARCHIVE_TRANSFER_STATE)) }
}
/**
* Returns the current archive thumbnail transfer state, if the attachment can be found.
*/
fun getArchiveThumbnailTransferState(id: AttachmentId): ArchiveTransferState? {
return readableDatabase
.select(ARCHIVE_THUMBNAIL_TRANSFER_STATE)
.from(TABLE_NAME)
.where("$ID = ?", id.id)
.run()
.readToSingleObject { ArchiveTransferState.deserialize(it.requireInt(ARCHIVE_THUMBNAIL_TRANSFER_STATE)) }
}
/**
* Sets the archive transfer state for the given attachment and all other attachments that share the same data file.
*/
@@ -839,6 +889,25 @@ class AttachmentTable(
AppDependencies.databaseObserver.notifyAttachmentUpdatedObservers()
}
fun setArchiveThumbnailTransferState(id: AttachmentId, state: ArchiveTransferState) {
check(state != ArchiveTransferState.COPY_PENDING) { "COPY_PENDING is not a valid transfer state for a thumbnail!" }
writableDatabase.withinTransaction {
val thumbnailFile: String = readableDatabase
.select(THUMBNAIL_FILE)
.from(TABLE_NAME)
.where("$ID = ?", id.id)
.run()
.readToSingleObject { it.requireString(THUMBNAIL_FILE) } ?: return@withinTransaction
writableDatabase
.update(TABLE_NAME)
.values(ARCHIVE_THUMBNAIL_TRANSFER_STATE to state.value)
.where("$THUMBNAIL_FILE = ?", thumbnailFile)
.run()
}
}
/**
* Sets the archive transfer state for the given attachment and all other attachments that share the same data file iff
* the row isn't already marked as a [ArchiveTransferState.PERMANENT_FAILURE].
@@ -862,6 +931,29 @@ class AttachmentTable(
AppDependencies.databaseObserver.notifyAttachmentUpdatedObservers()
}
/**
* Sets the archive thumbnail transfer state for the given attachment and all other attachments that share the same thumbnail file iff
* the row isn't already marked as a [ArchiveTransferState.PERMANENT_FAILURE].
*/
fun setArchiveThumbnailTransferStateFailure(id: AttachmentId, state: ArchiveTransferState) {
writableDatabase.withinTransaction {
val thumbnailFile: String = readableDatabase
.select(THUMBNAIL_FILE)
.from(TABLE_NAME)
.where("$ID = ?", id.id)
.run()
.readToSingleObject { it.requireString(THUMBNAIL_FILE) } ?: return@withinTransaction
writableDatabase
.update(TABLE_NAME)
.values(ARCHIVE_THUMBNAIL_TRANSFER_STATE to state.value)
.where("$ARCHIVE_THUMBNAIL_TRANSFER_STATE != ? AND $THUMBNAIL_FILE = ?", ArchiveTransferState.PERMANENT_FAILURE.value, thumbnailFile)
.run()
}
AppDependencies.databaseObserver.notifyAttachmentUpdatedObservers()
}
/**
* Resets the archive upload state by hash/key if we believe the attachment should have been uploaded already.
*/
@@ -876,6 +968,19 @@ class AttachmentTable(
.run() > 0
}
/**
* Resets the archive thumbnail upload state by hash/key if we believe the thumbnail should have been uploaded already.
*/
fun resetArchiveThumbnailTransferStateByPlaintextHashAndRemoteKeyIfNecessary(plaintextHash: ByteArray, remoteKey: ByteArray): Boolean {
return writableDatabase
.update(TABLE_NAME)
.values(
ARCHIVE_THUMBNAIL_TRANSFER_STATE to ArchiveTransferState.NONE.value
)
.where("$DATA_HASH_END = ? AND $REMOTE_KEY = ? AND $ARCHIVE_THUMBNAIL_TRANSFER_STATE = ${ArchiveTransferState.FINISHED.value}", Base64.encodeWithPadding(plaintextHash), Base64.encodeWithPadding(remoteKey))
.run() > 0
}
/**
* Sets the archive transfer state for the given attachment and all other attachments that share the same data file.
*/
@@ -2957,10 +3062,12 @@ class AttachmentTable(
}
}
private fun buildAttachmentsThatNeedUploadQuery(): String {
private fun buildAttachmentsThatNeedUploadQuery(transferStateFilter: String = "$ARCHIVE_TRANSFER_STATE IN (${ArchiveTransferState.NONE.value}, ${ArchiveTransferState.TEMPORARY_FAILURE.value})"): String {
return """
$ARCHIVE_TRANSFER_STATE IN (${ArchiveTransferState.NONE.value}, ${ArchiveTransferState.TEMPORARY_FAILURE.value}) AND
$transferStateFilter AND
$DATA_FILE NOT NULL AND
$REMOTE_KEY NOT NULL AND
$DATA_HASH_END NOT NULL AND
$TRANSFER_STATE = $TRANSFER_PROGRESS_DONE AND
(${MessageTable.STORY_TYPE} = 0 OR ${MessageTable.STORY_TYPE} IS NULL) AND
(${MessageTable.TABLE_NAME}.${MessageTable.EXPIRES_IN} <= 0 OR ${MessageTable.TABLE_NAME}.${MessageTable.EXPIRES_IN} > ${ChatItemArchiveExporter.EXPIRATION_CUTOFF.inWholeMilliseconds}) AND
@@ -3069,47 +3176,53 @@ class AttachmentTable(
}
fun debugGetAttachmentStats(): DebugAttachmentStats {
val count = readableDatabase.count().from(TABLE_NAME).run().readToSingleLong(0)
val totalAttachmentRows = readableDatabase.count().from(TABLE_NAME).run().readToSingleLong(0)
val totalEligibleForUploadRows = getAttachmentsThatWillBeIncludedInArchive().count
val transferStates = mapOf(
TRANSFER_PROGRESS_DONE to "TRANSFER_PROGRESS_DONE",
TRANSFER_PROGRESS_STARTED to "TRANSFER_PROGRESS_STARTED",
TRANSFER_PROGRESS_PENDING to "TRANSFER_PROGRESS_PENDING",
TRANSFER_PROGRESS_FAILED to "TRANSFER_PROGRESS_FAILED",
TRANSFER_PROGRESS_PERMANENT_FAILURE to "TRANSFER_PROGRESS_PERMANENT_FAILURE",
TRANSFER_NEEDS_RESTORE to "TRANSFER_NEEDS_RESTORE",
TRANSFER_RESTORE_IN_PROGRESS to "TRANSFER_RESTORE_IN_PROGRESS",
TRANSFER_RESTORE_OFFLOADED to "TRANSFER_RESTORE_OFFLOADED"
val totalUniqueDataFiles = readableDatabase.select("COUNT(DISTINCT $DATA_FILE)").from(TABLE_NAME).run().readToSingleLong(0)
val totalUniqueMediaNames = readableDatabase.query("SELECT COUNT(*) FROM (SELECT DISTINCT $DATA_HASH_END, $REMOTE_KEY FROM $TABLE_NAME WHERE $DATA_HASH_END NOT NULL AND $REMOTE_KEY NOT NULL)").readToSingleLong(0)
val totalUniqueMediaNamesEligibleForUpload = readableDatabase.query(
"""
SELECT COUNT(*) FROM (
SELECT DISTINCT $DATA_HASH_END, $REMOTE_KEY
FROM $TABLE_NAME LEFT JOIN ${MessageTable.TABLE_NAME} ON $TABLE_NAME.$MESSAGE_ID = ${MessageTable.TABLE_NAME}.${MessageTable.ID}
WHERE ${buildAttachmentsThatNeedUploadQuery(transferStateFilter = "$ARCHIVE_TRANSFER_STATE != ${ArchiveTransferState.PERMANENT_FAILURE.value}")}
)
"""
)
.readToSingleLong(0)
val transferStateCounts = transferStates
.map { (state, name) -> name to readableDatabase.count().from(TABLE_NAME).where("$TRANSFER_STATE = $state AND $DATA_HASH_END NOT NULL AND $REMOTE_KEY NOT NULL").run().readToSingleLong(-1L) }
.toMap()
val archiveStatusMediaNameCounts: Map<ArchiveTransferState, Long> = ArchiveTransferState.entries.associateWith { state ->
readableDatabase.query(
"""
SELECT COUNT(*) FROM (
SELECT DISTINCT $DATA_HASH_END, $REMOTE_KEY
FROM $TABLE_NAME LEFT JOIN ${MessageTable.TABLE_NAME} ON $TABLE_NAME.$MESSAGE_ID = ${MessageTable.TABLE_NAME}.${MessageTable.ID}
WHERE ${buildAttachmentsThatNeedUploadQuery(transferStateFilter = "$ARCHIVE_TRANSFER_STATE = ${state.value}")}
)
"""
)
.readToSingleLong(0)
}
val validForArchiveTransferStateCounts = transferStates
.map { (state, name) -> name to readableDatabase.count().from(TABLE_NAME).where("$TRANSFER_STATE = $state AND $DATA_HASH_END NOT NULL AND $REMOTE_KEY NOT NULL AND $DATA_FILE NOT NULL").run().readToSingleLong(-1L) }
.toMap()
val uniqueEligibleMediaNamesWithThumbnailsCount = readableDatabase.query("SELECT COUNT(*) FROM (SELECT DISTINCT $DATA_HASH_END, $REMOTE_KEY FROM $TABLE_NAME WHERE $DATA_HASH_END NOT NULL AND $REMOTE_KEY NOT NULL AND $THUMBNAIL_FILE NOT NULL)").readToSingleLong(-1L)
val archiveStatusMediaNameThumbnailCounts: Map<ArchiveTransferState, Long> = ArchiveTransferState.entries.associateWith { state ->
readableDatabase.query(
"""
SELECT COUNT(*) FROM (
SELECT DISTINCT $DATA_HASH_END, $REMOTE_KEY
FROM $TABLE_NAME LEFT JOIN ${MessageTable.TABLE_NAME} ON $TABLE_NAME.$MESSAGE_ID = ${MessageTable.TABLE_NAME}.${MessageTable.ID}
WHERE
${buildAttachmentsThatNeedUploadQuery(transferStateFilter = "$ARCHIVE_THUMBNAIL_TRANSFER_STATE = ${state.value}")}
AND ($CONTENT_TYPE LIKE 'image%' OR $CONTENT_TYPE LIKE 'video%')
)
"""
)
.readToSingleLong(0)
}
val archiveStateCounts = ArchiveTransferState
.entries
.associate { it to readableDatabase.count().from(TABLE_NAME).where("$ARCHIVE_TRANSFER_STATE = ${it.value} AND $DATA_HASH_END NOT NULL AND $REMOTE_KEY NOT NULL").run().readToSingleLong(-1L) }
val attachmentFileCount = readableDatabase.query("SELECT COUNT(DISTINCT $DATA_FILE) FROM $TABLE_NAME WHERE $DATA_FILE NOT NULL AND $DATA_HASH_END NOT NULL AND $REMOTE_KEY NOT NULL").readToSingleLong(-1L)
val finishedAttachmentFileCount =
readableDatabase.query("SELECT COUNT(DISTINCT $DATA_FILE) FROM $TABLE_NAME WHERE $DATA_FILE NOT NULL AND $DATA_HASH_END NOT NULL AND $REMOTE_KEY NOT NULL AND $ARCHIVE_TRANSFER_STATE = ${ArchiveTransferState.FINISHED.value}")
.readToSingleLong(-1L)
val attachmentPlaintextHashAndKeyCount =
readableDatabase.query("SELECT COUNT(*) FROM (SELECT DISTINCT $DATA_HASH_END, $REMOTE_KEY FROM $TABLE_NAME WHERE $DATA_HASH_END NOT NULL AND $REMOTE_KEY NOT NULL AND $TRANSFER_STATE in ($TRANSFER_PROGRESS_DONE, $TRANSFER_RESTORE_OFFLOADED, $TRANSFER_RESTORE_IN_PROGRESS, $TRANSFER_NEEDS_RESTORE))")
.readToSingleLong(-1L)
val finishedAttachmentDigestCount =
readableDatabase.query("SELECT COUNT(*) FROM (SELECT DISTINCT $DATA_HASH_END, $REMOTE_KEY FROM $TABLE_NAME WHERE $DATA_HASH_END NOT NULL AND $REMOTE_KEY NOT NULL AND $ARCHIVE_TRANSFER_STATE = ${ArchiveTransferState.FINISHED.value})")
.readToSingleLong(-1L)
val thumbnailFileCount = readableDatabase.query("SELECT COUNT(DISTINCT $THUMBNAIL_FILE) FROM $TABLE_NAME WHERE $THUMBNAIL_FILE IS NOT NULL").readToSingleLong(-1L)
val estimatedThumbnailCount =
readableDatabase.query("SELECT COUNT(*) FROM (SELECT DISTINCT $DATA_HASH_END, $REMOTE_KEY FROM $TABLE_NAME WHERE $ARCHIVE_TRANSFER_STATE = ${ArchiveTransferState.FINISHED.value} AND $DATA_HASH_END NOT NULL AND $REMOTE_KEY NOT NULL AND ($CONTENT_TYPE LIKE 'image/%' OR $CONTENT_TYPE LIKE 'video/%'))")
.readToSingleLong(-1L)
val pendingUploadBytes = getPendingArchiveUploadBytes()
val pendingAttachmentUploadBytes = getPendingArchiveUploadBytes()
val uploadedAttachmentBytes = readableDatabase
.rawQuery(
"""
@@ -3128,22 +3241,21 @@ class AttachmentTable(
.readToList { it.requireLong(DATA_SIZE) }
.sumOf { AttachmentCipherStreamUtil.getCiphertextLength(AttachmentCipherStreamUtil.getCiphertextLength(PaddingInputStream.getPaddedSize(it))) }
val uploadedThumbnailBytes = estimatedThumbnailCount * RemoteConfig.backupMaxThumbnailFileSize.inWholeBytes
val uploadedThumbnailCount = archiveStatusMediaNameThumbnailCounts.getOrDefault(ArchiveTransferState.FINISHED, 0L)
val uploadedThumbnailBytes = uploadedThumbnailCount * RemoteConfig.backupMaxThumbnailFileSize.inWholeBytes
return DebugAttachmentStats(
attachmentCount = count,
transferStateCounts = transferStateCounts,
validForArchiveTransferStateCounts = validForArchiveTransferStateCounts,
archiveStateCounts = archiveStateCounts,
attachmentFileCount = attachmentFileCount,
finishedAttachmentFileCount = finishedAttachmentFileCount,
attachmentPlaintextHashAndKeyCount = attachmentPlaintextHashAndKeyCount,
finishedAttachmentPlaintextHashAndKeyCount = finishedAttachmentDigestCount,
thumbnailFileCount = thumbnailFileCount,
estimatedThumbnailCount = estimatedThumbnailCount,
pendingUploadBytes = pendingUploadBytes,
totalAttachmentRows = totalAttachmentRows,
totalEligibleForUploadRows = totalEligibleForUploadRows.toLong(),
totalUniqueMediaNamesEligibleForUpload = totalUniqueMediaNamesEligibleForUpload,
totalUniqueDataFiles = totalUniqueDataFiles,
totalUniqueMediaNames = totalUniqueMediaNames,
archiveStatusMediaNameCounts = archiveStatusMediaNameCounts,
mediaNamesWithThumbnailsCount = uniqueEligibleMediaNamesWithThumbnailsCount,
archiveStatusMediaNameThumbnailCounts = archiveStatusMediaNameThumbnailCounts,
pendingAttachmentUploadBytes = pendingAttachmentUploadBytes,
uploadedAttachmentBytes = uploadedAttachmentBytes,
thumbnailBytes = uploadedThumbnailBytes
uploadedThumbnailBytes = uploadedThumbnailBytes
)
}
@@ -3528,20 +3640,24 @@ class AttachmentTable(
}
data class DebugAttachmentStats(
val attachmentCount: Long = 0L,
val transferStateCounts: Map<String, Long> = emptyMap(),
val archiveStateCounts: Map<ArchiveTransferState, Long> = emptyMap(),
val attachmentFileCount: Long = 0L,
val finishedAttachmentFileCount: Long = 0L,
val attachmentPlaintextHashAndKeyCount: Long = 0L,
val finishedAttachmentPlaintextHashAndKeyCount: Long,
val thumbnailFileCount: Long = 0L,
val pendingUploadBytes: Long = 0L,
val totalAttachmentRows: Long = 0L,
val totalEligibleForUploadRows: Long = 0L,
val totalUniqueMediaNamesEligibleForUpload: Long = 0L,
val totalUniqueDataFiles: Long = 0L,
val totalUniqueMediaNames: Long = 0L,
val archiveStatusMediaNameCounts: Map<ArchiveTransferState, Long> = emptyMap(),
val mediaNamesWithThumbnailsCount: Long = 0L,
val archiveStatusMediaNameThumbnailCounts: Map<ArchiveTransferState, Long> = emptyMap(),
val pendingAttachmentUploadBytes: Long = 0L,
val uploadedAttachmentBytes: Long = 0L,
val thumbnailBytes: Long = 0L,
val validForArchiveTransferStateCounts: Map<String, Long>,
val estimatedThumbnailCount: Long
)
val uploadedThumbnailBytes: Long = 0L
) {
val uploadedAttachmentCount get() = archiveStatusMediaNameCounts.getOrDefault(ArchiveTransferState.FINISHED, 0L)
val uploadedThumbnailCount get() = archiveStatusMediaNameThumbnailCounts.getOrDefault(ArchiveTransferState.FINISHED, 0L)
val totalUploadCount get() = uploadedAttachmentCount + uploadedThumbnailCount
val totalUploadBytes get() = uploadedAttachmentBytes + uploadedThumbnailBytes
}
data class CreateRemoteKeyResult(val totalCount: Int, val notQuoteOrSickerDupeNotFoundCount: Int, val notQuoteOrSickerDupeFoundCount: Int) {
val unexpectedKeyCreation = notQuoteOrSickerDupeFoundCount > 0 || notQuoteOrSickerDupeNotFoundCount > 0

6
app/src/main/java/org/thoughtcrime/securesms/database/helpers/SignalDatabaseMigrations.kt
View File

@@ -144,6 +144,7 @@ import org.thoughtcrime.securesms.database.helpers.migration.V286_FixRemoteKeyEn
import org.thoughtcrime.securesms.database.helpers.migration.V287_FixInvalidArchiveState
import org.thoughtcrime.securesms.database.helpers.migration.V288_CopyStickerDataHashStartToEnd
import org.thoughtcrime.securesms.database.helpers.migration.V289_AddQuoteTargetContentTypeColumn
import org.thoughtcrime.securesms.database.helpers.migration.V290_AddArchiveThumbnailTransferStateColumn
import org.thoughtcrime.securesms.database.SQLiteDatabase as SignalSqliteDatabase
/**
@@ -293,10 +294,11 @@ object SignalDatabaseMigrations {
286 to V286_FixRemoteKeyEncoding,
287 to V287_FixInvalidArchiveState,
288 to V288_CopyStickerDataHashStartToEnd,
289 to V289_AddQuoteTargetContentTypeColumn
289 to V289_AddQuoteTargetContentTypeColumn,
290 to V290_AddArchiveThumbnailTransferStateColumn
)
const val DATABASE_VERSION = 289
const val DATABASE_VERSION = 290
@JvmStatic
fun migrate(context: Application, db: SignalSqliteDatabase, oldVersion: Int, newVersion: Int) {

20
app/src/main/java/org/thoughtcrime/securesms/database/helpers/migration/V290_AddArchiveThumbnailTransferStateColumn.kt Normal file
View File

@@ -0,0 +1,20 @@
/*
* Copyright 2024 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.thoughtcrime.securesms.database.helpers.migration
import android.app.Application
import org.thoughtcrime.securesms.database.SQLiteDatabase
/**
* We need to keep track of a transfer state for thumbnails too.
*/
@Suppress("ClassName")
object V290_AddArchiveThumbnailTransferStateColumn : SignalDatabaseMigration {
override fun migrate(context: Application, db: SQLiteDatabase, oldVersion: Int, newVersion: Int) {
db.execSQL("ALTER TABLE attachment ADD COLUMN archive_thumbnail_transfer_state INTEGER DEFAULT 0;")
}
}

4
app/src/main/java/org/thoughtcrime/securesms/jobmanager/Job.java
View File

@@ -279,11 +279,13 @@ public abstract class Job {
public static final int UNLIMITED = -1;
@Retention(SOURCE)
@IntDef({ PRIORITY_DEFAULT, PRIORITY_LOW, PRIORITY_HIGH})
@IntDef({ PRIORITY_DEFAULT, PRIORITY_LOW, PRIORITY_LOWER, PRIORITY_HIGH})
public @interface Priority{}
public static final int PRIORITY_DEFAULT = 0;
public static final int PRIORITY_HIGH = 1;
public static final int PRIORITY_LOW = -1;
/** One step lower than {@link #PRIORITY_LOW} */
public static final int PRIORITY_LOWER = -2;
private final String id;
private final long createTime;

18
app/src/main/java/org/thoughtcrime/securesms/jobs/ArchiveAttachmentReconciliationJob.kt
View File

@@ -12,10 +12,8 @@ import org.thoughtcrime.securesms.backup.v2.ArchivedMediaObject
import org.thoughtcrime.securesms.backup.v2.BackupRepository
import org.thoughtcrime.securesms.database.BackupMediaSnapshotTable
import org.thoughtcrime.securesms.database.SignalDatabase
import org.thoughtcrime.securesms.dependencies.AppDependencies
import org.thoughtcrime.securesms.jobmanager.Job
import org.thoughtcrime.securesms.jobmanager.impl.NetworkConstraint
import org.thoughtcrime.securesms.jobs.ArchiveThumbnailUploadJob.Companion.isForArchiveThumbnailUploadJob
import org.thoughtcrime.securesms.jobs.protos.ArchiveAttachmentReconciliationJobData
import org.thoughtcrime.securesms.keyvalue.SignalStore
import org.thoughtcrime.securesms.util.RemoteConfig
@@ -142,18 +140,12 @@ class ArchiveAttachmentReconciliationJob private constructor(
val entry = BackupMediaSnapshotTable.MediaEntry.fromCursor(mediaObjectCursor)
if (entry.isThumbnail) {
val parentAttachmentId = SignalDatabase.attachments.getAttachmentIdByPlaintextHashAndRemoteKey(entry.plaintextHash, entry.remoteKey)
if (parentAttachmentId == null) {
Log.w(TAG, "Failed to find parent attachment for thumbnail that may need reupload. Skipping.", true)
return@forEach
}
if (AppDependencies.jobManager.find { it.isForArchiveThumbnailUploadJob(parentAttachmentId) }.isEmpty()) {
Log.w(TAG, "A thumbnail was missing from remote for $parentAttachmentId and no in-progress job was found. Re-enqueueing one.", true)
ArchiveThumbnailUploadJob.enqueueIfNecessary(parentAttachmentId)
val wasReset = SignalDatabase.attachments.resetArchiveThumbnailTransferStateByPlaintextHashAndRemoteKeyIfNecessary(entry.plaintextHash, entry.remoteKey)
if (wasReset) {
newBackupJobRequired = true
bookkeepingErrorCount++
} else {
Log.i(TAG, "A thumbnail was missing from remote for $parentAttachmentId, but a job is already in progress.", true)
Log.w(TAG, "[Thumbnail] Did not need to reset the transfer state by hash/key because the thumbnail either no longer exists or the upload is already in-progress.", true)
}
} else {
val wasReset = SignalDatabase.attachments.resetArchiveTransferStateByPlaintextHashAndRemoteKeyIfNecessary(entry.plaintextHash, entry.remoteKey)
@@ -161,7 +153,7 @@ class ArchiveAttachmentReconciliationJob private constructor(
newBackupJobRequired = true
bookkeepingErrorCount++
} else {
Log.w(TAG, "Did not need to reset the the transfer state by hash/key because the attachment either no longer exists or the upload is already in-progress.", true)
Log.w(TAG, "[Fullsize] Did not need to reset the the transfer state by hash/key because the attachment either no longer exists or the upload is already in-progress.", true)
}
}
}

64
app/src/main/java/org/thoughtcrime/securesms/jobs/ArchiveThumbnailBackfillJob.kt Normal file
View File

@@ -0,0 +1,64 @@
/*
* Copyright 2024 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.thoughtcrime.securesms.jobs
import org.signal.core.util.logging.Log
import org.thoughtcrime.securesms.database.SignalDatabase
import org.thoughtcrime.securesms.dependencies.AppDependencies
import org.thoughtcrime.securesms.jobmanager.Job
import org.thoughtcrime.securesms.keyvalue.SignalStore
import kotlin.time.Duration.Companion.days
/**
* When run, this will find all of the thumbnails that need to be uploaded to the archive tier and enqueue [ArchiveThumbnailUploadJob]s for them.
*/
class ArchiveThumbnailBackfillJob private constructor(parameters: Parameters) : Job(parameters) {
companion object {
private val TAG = Log.tag(ArchiveThumbnailBackfillJob::class.java)
const val KEY = "ArchiveThumbnailBackfillJob"
}
constructor() : this(
parameters = Parameters.Builder()
.setQueue(ArchiveCommitAttachmentDeletesJob.ARCHIVE_ATTACHMENT_QUEUE)
.setMaxInstancesForQueue(2)
.setLifespan(30.days.inWholeMilliseconds)
.setMaxAttempts(Parameters.UNLIMITED)
.build()
)
override fun serialize(): ByteArray? = null
override fun getFactoryKey(): String = KEY
override fun run(): Result {
if (!SignalStore.backup.backsUpMedia) {
Log.w(TAG, "This user doesn't back up media! Skipping. Tier: ${SignalStore.backup.backupTier}")
return Result.success()
}
val jobs = SignalDatabase.attachments.getThumbnailsThatNeedArchiveUpload()
.map { attachmentId -> ArchiveThumbnailUploadJob(attachmentId) }
if (!isCanceled) {
Log.i(TAG, "Adding ${jobs.size} jobs to backfill thumbnails.")
AppDependencies.jobManager.addAll(jobs)
} else {
Log.w(TAG, "Job was canceled. Not enqueuing backfill.")
}
return Result.success()
}
override fun onFailure() = Unit
class Factory : Job.Factory<ArchiveThumbnailBackfillJob> {
override fun create(parameters: Parameters, serializedData: ByteArray?): ArchiveThumbnailBackfillJob {
return ArchiveThumbnailBackfillJob(parameters)
}
}
}

36
app/src/main/java/org/thoughtcrime/securesms/jobs/ArchiveThumbnailUploadJob.kt
View File

@@ -14,10 +14,12 @@ import org.thoughtcrime.securesms.attachments.PointerAttachment
import org.thoughtcrime.securesms.backup.v2.BackupRepository
import org.thoughtcrime.securesms.backup.v2.hadIntegrityCheckPerformed
import org.thoughtcrime.securesms.backup.v2.requireThumbnailMediaName
import org.thoughtcrime.securesms.database.AttachmentTable
import org.thoughtcrime.securesms.database.SignalDatabase
import org.thoughtcrime.securesms.dependencies.AppDependencies
import org.thoughtcrime.securesms.jobmanager.Job
import org.thoughtcrime.securesms.jobmanager.impl.NetworkConstraint
import org.thoughtcrime.securesms.jobmanager.impl.NoRemoteArchiveGarbageCollectionPendingConstraint
import org.thoughtcrime.securesms.jobmanager.persistence.JobSpec
import org.thoughtcrime.securesms.jobs.protos.ArchiveThumbnailUploadJobData
import org.thoughtcrime.securesms.keyvalue.SignalStore
@@ -58,7 +60,9 @@ class ArchiveThumbnailUploadJob private constructor(
/** A set of possible queues this job may use. The number of queues determines the parallelism. */
val QUEUES = setOf(
"ArchiveThumbnailUploadJob_1",
"ArchiveThumbnailUploadJob_2"
"ArchiveThumbnailUploadJob_2",
"ArchiveThumbnailUploadJob_3",
"ArchiveThumbnailUploadJob_4"
)
fun enqueueIfNecessary(attachmentId: AttachmentId) {
@@ -72,13 +76,14 @@ class ArchiveThumbnailUploadJob private constructor(
}
}
private constructor(attachmentId: AttachmentId) : this(
constructor(attachmentId: AttachmentId) : this(
Parameters.Builder()
.setQueue(QUEUES.random())
.addConstraint(NetworkConstraint.KEY)
.addConstraint(NoRemoteArchiveGarbageCollectionPendingConstraint.KEY)
.setLifespan(1.days.inWholeMilliseconds)
.setMaxAttempts(Parameters.UNLIMITED)
.setGlobalPriority(Parameters.PRIORITY_LOW)
.setGlobalPriority(Parameters.PRIORITY_LOWER)
.build(),
attachmentId
)
@@ -91,6 +96,14 @@ class ArchiveThumbnailUploadJob private constructor(
override fun getFactoryKey(): String = KEY
override fun onAdded() {
val transferStatus = SignalDatabase.attachments.getArchiveThumbnailTransferState(attachmentId) ?: return
if (transferStatus == AttachmentTable.ArchiveTransferState.NONE) {
SignalDatabase.attachments.setArchiveThumbnailTransferState(attachmentId, AttachmentTable.ArchiveTransferState.UPLOAD_IN_PROGRESS)
}
}
override fun run(): Result {
val attachment = SignalDatabase.attachments.getAttachment(attachmentId)
if (attachment == null) {
@@ -100,26 +113,31 @@ class ArchiveThumbnailUploadJob private constructor(
if (!MediaUtil.isImageOrVideoType(attachment.contentType)) {
Log.w(TAG, "$attachmentId isn't visual media (contentType = ${attachment.contentType}). Skipping.")
SignalDatabase.attachments.setArchiveThumbnailTransferState(attachmentId, AttachmentTable.ArchiveTransferState.NONE)
return Result.success()
}
if (attachment.quote) {
Log.w(TAG, "$attachmentId is a quote. Skipping.")
SignalDatabase.attachments.setArchiveThumbnailTransferState(attachmentId, AttachmentTable.ArchiveTransferState.NONE)
return Result.success()
}
if (attachment.dataHash == null || attachment.remoteKey == null) {
Log.w(TAG, "$attachmentId is missing necessary ingredients for a mediaName!")
SignalDatabase.attachments.setArchiveThumbnailTransferState(attachmentId, AttachmentTable.ArchiveTransferState.NONE)
return Result.success()
}
if (!attachment.hadIntegrityCheckPerformed()) {
Log.w(TAG, "$attachmentId has no integrity check! Cannot proceed.")
SignalDatabase.attachments.setArchiveThumbnailTransferState(attachmentId, AttachmentTable.ArchiveTransferState.NONE)
return Result.success()
}
if (SignalDatabase.messages.isStory(attachment.mmsId)) {
Log.w(TAG, "$attachmentId is a story. Skipping.")
SignalDatabase.attachments.setArchiveThumbnailTransferState(attachmentId, AttachmentTable.ArchiveTransferState.NONE)
return Result.success()
}
@@ -128,10 +146,12 @@ class ArchiveThumbnailUploadJob private constructor(
val thumbnailResult = generateThumbnailIfPossible(attachment)
if (thumbnailResult == null) {
Log.w(TAG, "Unable to generate a thumbnail result for $attachmentId")
SignalDatabase.attachments.setArchiveThumbnailTransferState(attachmentId, AttachmentTable.ArchiveTransferState.PERMANENT_FAILURE)
return Result.success()
}
if (isCanceled) {
SignalDatabase.attachments.setArchiveThumbnailTransferState(attachmentId, AttachmentTable.ArchiveTransferState.TEMPORARY_FAILURE)
return Result.failure()
}
@@ -148,6 +168,7 @@ class ArchiveThumbnailUploadJob private constructor(
}
if (isCanceled) {
SignalDatabase.attachments.setArchiveThumbnailTransferState(attachmentId, AttachmentTable.ArchiveTransferState.TEMPORARY_FAILURE)
return Result.failure()
}
@@ -174,6 +195,7 @@ class ArchiveThumbnailUploadJob private constructor(
}
if (isCanceled) {
SignalDatabase.attachments.setArchiveThumbnailTransferState(attachmentId, AttachmentTable.ArchiveTransferState.TEMPORARY_FAILURE)
return Result.failure()
}
@@ -202,6 +224,7 @@ class ArchiveThumbnailUploadJob private constructor(
)
Log.d(TAG, "Successfully archived thumbnail for $attachmentId")
SignalDatabase.attachments.setArchiveThumbnailTransferState(attachmentId, AttachmentTable.ArchiveTransferState.FINISHED)
Result.success()
}
@@ -220,6 +243,13 @@ class ArchiveThumbnailUploadJob private constructor(
}
override fun onFailure() {
if (this.isCanceled) {
Log.w(TAG, "[$attachmentId] Job was canceled, updating archive thumbnail transfer state to ${AttachmentTable.ArchiveTransferState.NONE}.")
SignalDatabase.attachments.setArchiveThumbnailTransferState(attachmentId, AttachmentTable.ArchiveTransferState.NONE)
} else {
Log.w(TAG, "[$attachmentId] Job failed, updating archive thumbnail transfer state to ${AttachmentTable.ArchiveTransferState.TEMPORARY_FAILURE} (if not already a permanent failure).")
SignalDatabase.attachments.setArchiveThumbnailTransferStateFailure(attachmentId, AttachmentTable.ArchiveTransferState.TEMPORARY_FAILURE)
}
}
private fun generateThumbnailIfPossible(attachment: DatabaseAttachment): ImageCompressionUtil.Result? {

9
app/src/main/java/org/thoughtcrime/securesms/jobs/BackupMessagesJob.kt
View File

@@ -318,6 +318,13 @@ class BackupMessagesJob private constructor(
ArchiveUploadProgress.onMessageBackupFinishedEarly()
}
if (SignalStore.backup.backsUpMedia && SignalDatabase.attachments.doAnyThumbnailsNeedArchiveUpload()) {
Log.i(TAG, "Enqueuing thumbnail backfill job.")
AppDependencies.jobManager.add(ArchiveThumbnailBackfillJob())
} else {
Log.i(TAG, "No thumbnails need to be uploaded: ${SignalStore.backup.backupTier}")
}
BackupRepository.clearBackupFailure()
SignalDatabase.backupMediaSnapshots.commitPendingRows()
@@ -413,7 +420,7 @@ class BackupMessagesJob private constructor(
private fun writeMediaCursorToTemporaryTable(db: SignalDatabase, mediaBackupEnabled: Boolean) {
if (mediaBackupEnabled) {
db.attachmentTable.getAttachmentsEligibleForArchiveUpload().use {
db.attachmentTable.getAttachmentsThatWillBeIncludedInArchive().use {
SignalDatabase.backupMediaSnapshots.writePendingMediaObjects(
mediaObjects = ArchiveMediaItemIterator(it).asSequence()
)

7
app/src/main/java/org/thoughtcrime/securesms/jobs/CopyAttachmentToArchiveJob.kt
View File

@@ -50,6 +50,7 @@ class CopyAttachmentToArchiveJob private constructor(private val attachmentId: A
.setMaxAttempts(Parameters.UNLIMITED)
.setQueue(UploadAttachmentToArchiveJob.QUEUES.random())
.setQueuePriority(Parameters.PRIORITY_HIGH)
.setGlobalPriority(Parameters.PRIORITY_LOW)
.build()
)
@@ -145,6 +146,12 @@ class CopyAttachmentToArchiveJob private constructor(private val attachmentId: A
is NetworkResult.StatusCodeError -> {
when (archiveResult.code) {
400 -> {
Log.w(TAG, "[$attachmentId] Something is invalid about our request. Possibly the length. Scheduling a re-upload. Body: ${archiveResult.exception.stringBody}")
SignalDatabase.attachments.setArchiveTransferState(attachmentId, AttachmentTable.ArchiveTransferState.NONE)
AppDependencies.jobManager.add(UploadAttachmentToArchiveJob(attachmentId, canReuseUpload = false))
Result.success()
}
403 -> {
Log.w(TAG, "[$attachmentId] Insufficient permissions to upload. Handled in parent handler.")
Result.success()

1
app/src/main/java/org/thoughtcrime/securesms/jobs/JobManagerFactories.java
View File

@@ -130,6 +130,7 @@ public final class JobManagerFactories {
put(ArchiveAttachmentReconciliationJob.KEY, new ArchiveAttachmentReconciliationJob.Factory());
put(ArchiveBackupIdReservationJob.KEY, new ArchiveBackupIdReservationJob.Factory());
put(ArchiveCommitAttachmentDeletesJob.KEY, new ArchiveCommitAttachmentDeletesJob.Factory());
put(ArchiveThumbnailBackfillJob.KEY, new ArchiveThumbnailBackfillJob.Factory());
put(ArchiveThumbnailUploadJob.KEY, new ArchiveThumbnailUploadJob.Factory());
put(AttachmentCompressionJob.KEY, new AttachmentCompressionJob.Factory());
put(AttachmentCopyJob.KEY, new AttachmentCopyJob.Factory());

2
app/src/main/java/org/thoughtcrime/securesms/jobs/RestoreAttachmentJob.kt
View File

@@ -111,7 +111,7 @@ class RestoreAttachmentJob private constructor(
/**
* Create a restore job for the initial large batch of media on a fresh restore.
* Will enqueue with some amount of parallization with low job priority.
* Will enqueue with some amount of parallelization with low job priority.
*/
fun forInitialRestore(attachmentId: AttachmentId, messageId: Long, stickerPackId: String?): RestoreAttachmentJob {
return RestoreAttachmentJob(

1
app/src/main/java/org/thoughtcrime/securesms/jobs/UploadAttachmentToArchiveJob.kt
View File

@@ -80,6 +80,7 @@ class UploadAttachmentToArchiveJob private constructor(
.setLifespan(30.days.inWholeMilliseconds)
.setMaxAttempts(Parameters.UNLIMITED)
.setQueue(QUEUES.random())
.setGlobalPriority(Parameters.PRIORITY_LOW)
.build()
)

13
app/src/main/java/org/thoughtcrime/securesms/logsubmit/LogSectionRemoteBackups.kt
View File

@@ -87,19 +87,6 @@ class LogSectionRemoteBackups : LogSection {
if (SignalStore.backup.archiveUploadState!!.state !in setOf(ArchiveUploadProgressState.State.None, ArchiveUploadProgressState.State.UserCanceled)) {
output.append("Pending bytes: ${SignalDatabase.attachments.getPendingArchiveUploadBytes()}\n")
val pendingAttachments = SignalDatabase.attachments.debugGetPendingArchiveUploadAttachments()
if (pendingAttachments.isNotEmpty()) {
output.append("Pending attachments:\n")
output.append(" Count: ${pendingAttachments.size}\n")
output.append(" Sum of Size: ${pendingAttachments.sumOf { it.size }}\n")
output.append(" Content types:\n")
pendingAttachments.groupBy { it.contentType }.forEach { (contentType, attachments) ->
output.append(" $contentType: ${attachments.size}\n")
}
} else {
output.append("Pending attachments: None!\n")
}
}
} else {
output.append("None\n")

2
libsignal-service/src/main/java/org/whispersystems/signalservice/api/archive/ArchiveApi.kt
View File

@@ -267,7 +267,7 @@ class ArchiveApi(
var cursor: String? = null
do {
val response: ArchiveGetMediaItemsResponse = getArchiveMediaItemsPage(aci, archiveServiceAccess, 512, cursor).successOrThrow()
val response: ArchiveGetMediaItemsResponse = getArchiveMediaItemsPage(aci, archiveServiceAccess, 10_000, cursor).successOrThrow()
mediaObjects += response.storedMediaObjects
cursor = response.cursor
} while (cursor != null)