mirror of
https://github.com/signalapp/Signal-Android.git
synced 2026-02-24 03:35:58 +00:00
Update BackupMediaSnapshot to be based on attachments in backup frames.
This commit is contained in:
Greyson Parrelli
committed by
Jeffrey Starke
Jeffrey Starke
parent
f39ad24cc1
commit
c5753b96ff
@@ -72,6 +72,7 @@ import org.thoughtcrime.securesms.backup.v2.processor.RecipientArchiveProcessor
|
||||
import org.thoughtcrime.securesms.backup.v2.processor.StickerArchiveProcessor
|
||||
import org.thoughtcrime.securesms.backup.v2.proto.BackupDebugInfo
|
||||
import org.thoughtcrime.securesms.backup.v2.proto.BackupInfo
|
||||
import org.thoughtcrime.securesms.backup.v2.proto.Frame
|
||||
import org.thoughtcrime.securesms.backup.v2.stream.BackupExportWriter
|
||||
import org.thoughtcrime.securesms.backup.v2.stream.BackupImportReader
|
||||
import org.thoughtcrime.securesms.backup.v2.stream.EncryptedBackupReader
|
||||
@@ -80,6 +81,7 @@ import org.thoughtcrime.securesms.backup.v2.stream.PlainTextBackupReader
|
||||
import org.thoughtcrime.securesms.backup.v2.stream.PlainTextBackupWriter
|
||||
import org.thoughtcrime.securesms.backup.v2.ui.BackupAlert
|
||||
import org.thoughtcrime.securesms.backup.v2.ui.subscription.MessageBackupsType
|
||||
import org.thoughtcrime.securesms.backup.v2.util.ArchiveAttachmentInfo
|
||||
import org.thoughtcrime.securesms.components.settings.app.AppSettingsActivity
|
||||
import org.thoughtcrime.securesms.components.settings.app.subscription.RecurringInAppPaymentRepository
|
||||
import org.thoughtcrime.securesms.crypto.AttachmentSecretProvider
|
||||
@@ -722,13 +724,18 @@ object BackupRepository {
|
||||
append = { main.write(it) }
|
||||
)
|
||||
|
||||
val maxBufferSize = 10_000
|
||||
var totalAttachmentCount = 0
|
||||
val attachmentInfos: MutableSet<ArchiveAttachmentInfo> = mutableSetOf()
|
||||
|
||||
export(
|
||||
currentTime = System.currentTimeMillis(),
|
||||
isLocal = true,
|
||||
writer = writer,
|
||||
progressEmitter = localBackupProgressEmitter,
|
||||
cancellationSignal = cancellationSignal,
|
||||
forTransfer = false
|
||||
forTransfer = false,
|
||||
extraFrameOperation = null
|
||||
) { dbSnapshot ->
|
||||
val localArchivableAttachments = dbSnapshot
|
||||
.attachmentTable
|
||||
@@ -764,7 +771,7 @@ object BackupRepository {
|
||||
currentTime: Long,
|
||||
progressEmitter: ExportProgressListener? = null,
|
||||
cancellationSignal: () -> Boolean = { false },
|
||||
extraExportOperations: ((SignalDatabase) -> Unit)?
|
||||
extraFrameOperation: ((Frame) -> Unit)?
|
||||
) {
|
||||
val writer = EncryptedBackupWriter.createForSignalBackup(
|
||||
key = messageBackupKey,
|
||||
@@ -782,7 +789,8 @@ object BackupRepository {
|
||||
forTransfer = false,
|
||||
progressEmitter = progressEmitter,
|
||||
cancellationSignal = cancellationSignal,
|
||||
extraExportOperations = extraExportOperations
|
||||
extraFrameOperation = extraFrameOperation,
|
||||
endingExportOperation = null
|
||||
)
|
||||
}
|
||||
|
||||
@@ -811,7 +819,8 @@ object BackupRepository {
|
||||
forTransfer = true,
|
||||
progressEmitter = progressEmitter,
|
||||
cancellationSignal = cancellationSignal,
|
||||
extraExportOperations = null
|
||||
extraFrameOperation = null,
|
||||
endingExportOperation = null
|
||||
)
|
||||
}
|
||||
|
||||
@@ -825,8 +834,7 @@ object BackupRepository {
|
||||
currentTime: Long = System.currentTimeMillis(),
|
||||
forTransfer: Boolean = false,
|
||||
progressEmitter: ExportProgressListener? = null,
|
||||
cancellationSignal: () -> Boolean = { false },
|
||||
extraExportOperations: ((SignalDatabase) -> Unit)? = null
|
||||
cancellationSignal: () -> Boolean = { false }
|
||||
) {
|
||||
val writer: BackupExportWriter = if (plaintext) {
|
||||
PlainTextBackupWriter(outputStream)
|
||||
@@ -846,7 +854,8 @@ object BackupRepository {
|
||||
forTransfer = forTransfer,
|
||||
progressEmitter = progressEmitter,
|
||||
cancellationSignal = cancellationSignal,
|
||||
extraExportOperations = extraExportOperations
|
||||
extraFrameOperation = null,
|
||||
endingExportOperation = null
|
||||
)
|
||||
}
|
||||
|
||||
@@ -868,7 +877,8 @@ object BackupRepository {
|
||||
forTransfer: Boolean,
|
||||
progressEmitter: ExportProgressListener?,
|
||||
cancellationSignal: () -> Boolean,
|
||||
extraExportOperations: ((SignalDatabase) -> Unit)?
|
||||
extraFrameOperation: ((Frame) -> Unit)?,
|
||||
endingExportOperation: ((SignalDatabase) -> Unit)?
|
||||
) {
|
||||
val eventTimer = EventTimer()
|
||||
val mainDbName = if (isLocal) LOCAL_MAIN_DB_SNAPSHOT_NAME else REMOTE_MAIN_DB_SNAPSHOT_NAME
|
||||
@@ -906,8 +916,9 @@ object BackupRepository {
|
||||
// We're using a snapshot, so the transaction is more for perf than correctness
|
||||
dbSnapshot.rawWritableDatabase.withinTransaction {
|
||||
progressEmitter?.onAccount()
|
||||
AccountDataArchiveProcessor.export(dbSnapshot, signalStoreSnapshot) {
|
||||
writer.write(it)
|
||||
AccountDataArchiveProcessor.export(dbSnapshot, signalStoreSnapshot) { frame ->
|
||||
writer.write(frame)
|
||||
extraFrameOperation?.invoke(frame)
|
||||
eventTimer.emit("account")
|
||||
frameCount++
|
||||
}
|
||||
@@ -919,6 +930,7 @@ object BackupRepository {
|
||||
progressEmitter?.onRecipient()
|
||||
RecipientArchiveProcessor.export(dbSnapshot, signalStoreSnapshot, exportState, selfRecipientId, selfAci) {
|
||||
writer.write(it)
|
||||
extraFrameOperation?.invoke(it)
|
||||
eventTimer.emit("recipient")
|
||||
frameCount++
|
||||
}
|
||||
@@ -930,6 +942,7 @@ object BackupRepository {
|
||||
progressEmitter?.onThread()
|
||||
ChatArchiveProcessor.export(dbSnapshot, exportState) { frame ->
|
||||
writer.write(frame)
|
||||
extraFrameOperation?.invoke(frame)
|
||||
eventTimer.emit("thread")
|
||||
frameCount++
|
||||
}
|
||||
@@ -940,6 +953,7 @@ object BackupRepository {
|
||||
progressEmitter?.onCall()
|
||||
AdHocCallArchiveProcessor.export(dbSnapshot, exportState) { frame ->
|
||||
writer.write(frame)
|
||||
extraFrameOperation?.invoke(frame)
|
||||
eventTimer.emit("call")
|
||||
frameCount++
|
||||
}
|
||||
@@ -951,6 +965,7 @@ object BackupRepository {
|
||||
progressEmitter?.onSticker()
|
||||
StickerArchiveProcessor.export(dbSnapshot) { frame ->
|
||||
writer.write(frame)
|
||||
extraFrameOperation?.invoke(frame)
|
||||
eventTimer.emit("sticker-pack")
|
||||
frameCount++
|
||||
}
|
||||
@@ -962,6 +977,7 @@ object BackupRepository {
|
||||
progressEmitter?.onNotificationProfile()
|
||||
NotificationProfileProcessor.export(dbSnapshot, exportState) { frame ->
|
||||
writer.write(frame)
|
||||
extraFrameOperation?.invoke(frame)
|
||||
eventTimer.emit("notification-profile")
|
||||
frameCount++
|
||||
}
|
||||
@@ -973,6 +989,7 @@ object BackupRepository {
|
||||
progressEmitter?.onChatFolder()
|
||||
ChatFolderProcessor.export(dbSnapshot, exportState) { frame ->
|
||||
writer.write(frame)
|
||||
extraFrameOperation?.invoke(frame)
|
||||
eventTimer.emit("chat-folder")
|
||||
frameCount++
|
||||
}
|
||||
@@ -986,6 +1003,7 @@ object BackupRepository {
|
||||
progressEmitter?.onMessage(0, approximateMessageCount)
|
||||
ChatItemArchiveProcessor.export(dbSnapshot, exportState, selfRecipientId, cancellationSignal) { frame ->
|
||||
writer.write(frame)
|
||||
extraFrameOperation?.invoke(frame)
|
||||
eventTimer.emit("message")
|
||||
frameCount++
|
||||
|
||||
@@ -1001,7 +1019,7 @@ object BackupRepository {
|
||||
}
|
||||
}
|
||||
|
||||
extraExportOperations?.invoke(dbSnapshot)
|
||||
endingExportOperation?.invoke(dbSnapshot)
|
||||
|
||||
Log.d(TAG, "[export] totalFrames: $frameCount | ${eventTimer.stop().summary}")
|
||||
} finally {
|
||||
@@ -2071,7 +2089,7 @@ object BackupRepository {
|
||||
val messageBackupKey = SignalStore.backup.messageBackupKey
|
||||
|
||||
Log.i(TAG, "[remoteRestore] Fetching SVRB data")
|
||||
val svrBAuth = when (val result = BackupRepository.getSvrBAuth()) {
|
||||
val svrBAuth = when (val result = getSvrBAuth()) {
|
||||
is NetworkResult.Success -> result.result
|
||||
is NetworkResult.NetworkError -> return RemoteRestoreResult.NetworkError.logW(TAG, "[remoteRestore] Network error when getting SVRB auth.", result.getCause())
|
||||
is NetworkResult.StatusCodeError -> return RemoteRestoreResult.NetworkError.logW(TAG, "[remoteRestore] Status code error when getting SVRB auth.", result.getCause())
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
/*
|
||||
* Copyright 2025 Signal Messenger, LLC
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
package org.thoughtcrime.securesms.backup.v2.util
|
||||
|
||||
import okio.ByteString
|
||||
import org.thoughtcrime.securesms.attachments.Cdn
|
||||
import org.thoughtcrime.securesms.backup.v2.proto.AccountData
|
||||
import org.thoughtcrime.securesms.backup.v2.proto.Chat
|
||||
import org.thoughtcrime.securesms.backup.v2.proto.ChatItem
|
||||
import org.thoughtcrime.securesms.backup.v2.proto.FilePointer
|
||||
import org.thoughtcrime.securesms.backup.v2.proto.Frame
|
||||
import org.whispersystems.signalservice.api.backup.MediaName
|
||||
|
||||
fun Frame.getAllReferencedArchiveAttachmentInfos(): Set<ArchiveAttachmentInfo> {
|
||||
val infos: MutableSet<ArchiveAttachmentInfo> = mutableSetOf()
|
||||
when {
|
||||
this.account != null -> infos += this.account.getAllReferencedArchiveAttachmentInfos()
|
||||
this.chat != null -> infos += this.chat.getAllReferencedArchiveAttachmentInfos()
|
||||
this.chatItem != null -> infos += this.chatItem.getAllReferencedArchiveAttachmentInfos()
|
||||
}
|
||||
return infos.toSet()
|
||||
}
|
||||
|
||||
private fun AccountData.getAllReferencedArchiveAttachmentInfos(): Set<ArchiveAttachmentInfo> {
|
||||
val info = this.accountSettings?.defaultChatStyle?.wallpaperPhoto?.toArchiveAttachmentInfo()
|
||||
|
||||
return if (info != null) {
|
||||
setOf(info)
|
||||
} else {
|
||||
emptySet()
|
||||
}
|
||||
}
|
||||
|
||||
private fun Chat.getAllReferencedArchiveAttachmentInfos(): Set<ArchiveAttachmentInfo> {
|
||||
val info = this.style?.wallpaperPhoto?.toArchiveAttachmentInfo()
|
||||
|
||||
return if (info != null) {
|
||||
setOf(info)
|
||||
} else {
|
||||
emptySet()
|
||||
}
|
||||
}
|
||||
|
||||
private fun ChatItem.getAllReferencedArchiveAttachmentInfos(): Set<ArchiveAttachmentInfo> {
|
||||
var out: MutableSet<ArchiveAttachmentInfo>? = null
|
||||
|
||||
// The user could have many chat items, and most will not have attachments. To avoid allocating unnecessary sets, we do this little trick.
|
||||
// (Note: emptySet() returns a constant under the hood, so that's fine)
|
||||
fun appendToOutput(item: ArchiveAttachmentInfo) {
|
||||
if (out == null) {
|
||||
out = mutableSetOf()
|
||||
}
|
||||
|
||||
out.add(item)
|
||||
}
|
||||
|
||||
this.contactMessage?.contact?.avatar?.toArchiveAttachmentInfo()?.let { appendToOutput(it) }
|
||||
this.directStoryReplyMessage?.textReply?.longText?.toArchiveAttachmentInfo()?.let { appendToOutput(it) }
|
||||
this.standardMessage?.attachments?.mapNotNull { it.pointer?.toArchiveAttachmentInfo() }?.forEach { appendToOutput(it) }
|
||||
this.standardMessage?.quote?.attachments?.mapNotNull { it.thumbnail?.pointer?.toArchiveAttachmentInfo(forQuote = true) }?.forEach { appendToOutput(it) }
|
||||
this.standardMessage?.linkPreview?.mapNotNull { it.image?.toArchiveAttachmentInfo() }?.forEach { appendToOutput(it) }
|
||||
this.standardMessage?.longText?.toArchiveAttachmentInfo()?.let { appendToOutput(it) }
|
||||
this.stickerMessage?.sticker?.data_?.toArchiveAttachmentInfo()?.let { appendToOutput(it) }
|
||||
this.viewOnceMessage?.attachment?.pointer?.toArchiveAttachmentInfo()?.let { appendToOutput(it) }
|
||||
|
||||
this.revisions.forEach { revision ->
|
||||
revision.getAllReferencedArchiveAttachmentInfos().forEach { appendToOutput(it) }
|
||||
}
|
||||
|
||||
return out ?: emptySet()
|
||||
}
|
||||
|
||||
private fun FilePointer.toArchiveAttachmentInfo(forQuote: Boolean = false): ArchiveAttachmentInfo? {
|
||||
if (this.locatorInfo?.key == null) {
|
||||
return null
|
||||
}
|
||||
|
||||
if (this.locatorInfo.plaintextHash == null) {
|
||||
return null
|
||||
}
|
||||
|
||||
return ArchiveAttachmentInfo(
|
||||
plaintextHash = this.locatorInfo.plaintextHash,
|
||||
remoteKey = this.locatorInfo.key,
|
||||
cdn = this.locatorInfo.mediaTierCdnNumber ?: Cdn.CDN_0.cdnNumber,
|
||||
contentType = this.contentType,
|
||||
forQuote = forQuote
|
||||
)
|
||||
}
|
||||
|
||||
data class ArchiveAttachmentInfo(
|
||||
val plaintextHash: ByteString,
|
||||
val remoteKey: ByteString,
|
||||
val cdn: Int,
|
||||
val contentType: String?,
|
||||
val forQuote: Boolean
|
||||
) {
|
||||
val fullSizeMediaName: MediaName get() = MediaName.fromPlaintextHashAndRemoteKey(plaintextHash.toByteArray(), remoteKey.toByteArray())
|
||||
val thumbnailMediaName: MediaName get() = MediaName.fromPlaintextHashAndRemoteKeyForThumbnail(plaintextHash.toByteArray(), remoteKey.toByteArray())
|
||||
}
|
||||
@@ -43,11 +43,6 @@ fun InternalBackupStatsTab(stats: InternalBackupPlaygroundViewModel.StatsState,
|
||||
label = "${stats.attachmentStats.totalUniqueMediaNames}"
|
||||
)
|
||||
|
||||
Rows.TextRow(
|
||||
text = "Total eligible for upload rows",
|
||||
label = "${stats.attachmentStats.totalEligibleForUploadRows}"
|
||||
)
|
||||
|
||||
Rows.TextRow(
|
||||
text = "Total unique media names eligible for upload ⭐",
|
||||
label = "${stats.attachmentStats.totalUniqueMediaNamesEligibleForUpload}"
|
||||
@@ -73,6 +68,16 @@ fun InternalBackupStatsTab(stats: InternalBackupPlaygroundViewModel.StatsState,
|
||||
label = "${stats.attachmentStats.pendingAttachmentUploadBytes} (~${stats.attachmentStats.pendingAttachmentUploadBytes.bytes.toUnitString()})"
|
||||
)
|
||||
|
||||
Rows.TextRow(
|
||||
text = "Last snapshot full-size count ⭐",
|
||||
label = "${stats.attachmentStats.lastSnapshotFullSizeCount}"
|
||||
)
|
||||
|
||||
Rows.TextRow(
|
||||
text = "Last snapshot thumbnail count ⭐",
|
||||
label = "${stats.attachmentStats.lastSnapshotThumbnailCount}"
|
||||
)
|
||||
|
||||
Rows.TextRow(
|
||||
text = "Uploaded attachment bytes ⭐",
|
||||
label = "${stats.attachmentStats.uploadedAttachmentBytes} (~${stats.attachmentStats.uploadedAttachmentBytes.bytes.toUnitString()})"
|
||||
|
||||
@@ -431,34 +431,27 @@ class AttachmentTable(
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a cursor (with just the plaintextHash+remoteKey+archive_cdn) for all full-size attachments that are slated to be included in the current archive upload.
|
||||
* Used for snapshotting data in [BackupMediaSnapshotTable].
|
||||
* Returns a list that has any permanently-failed thumbnails removed.
|
||||
*/
|
||||
fun getFullSizeAttachmentsThatWillBeIncludedInArchive(): Cursor {
|
||||
return readableDatabase
|
||||
.select(DATA_HASH_END, REMOTE_KEY, ARCHIVE_CDN, QUOTE, CONTENT_TYPE)
|
||||
.from("$TABLE_NAME LEFT JOIN ${MessageTable.TABLE_NAME} ON $TABLE_NAME.$MESSAGE_ID = ${MessageTable.TABLE_NAME}.${MessageTable.ID}")
|
||||
.where(buildAttachmentsThatNeedUploadQuery(transferStateFilter = "$ARCHIVE_TRANSFER_STATE != ${ArchiveTransferState.PERMANENT_FAILURE.value}"))
|
||||
.run()
|
||||
}
|
||||
fun filterPermanentlyFailedThumbnails(entries: Set<BackupMediaSnapshotTable.MediaEntry>): Set<BackupMediaSnapshotTable.MediaEntry> {
|
||||
val entriesByMediaName: MutableMap<String, BackupMediaSnapshotTable.MediaEntry> = entries
|
||||
.associateBy { MediaName.fromPlaintextHashAndRemoteKeyForThumbnail(it.plaintextHash, it.remoteKey).name }
|
||||
.toMutableMap()
|
||||
|
||||
/**
|
||||
* Returns a cursor (with just the plaintextHash+remoteKey+archive_cdn) for all thumbnail attachments that are slated to be included in the current archive upload.
|
||||
* Used for snapshotting data in [BackupMediaSnapshotTable].
|
||||
*/
|
||||
fun getThumbnailAttachmentsThatWillBeIncludedInArchive(): Cursor {
|
||||
return readableDatabase
|
||||
.select(DATA_HASH_END, REMOTE_KEY, ARCHIVE_CDN, QUOTE, CONTENT_TYPE)
|
||||
.from("$TABLE_NAME LEFT JOIN ${MessageTable.TABLE_NAME} ON $TABLE_NAME.$MESSAGE_ID = ${MessageTable.TABLE_NAME}.${MessageTable.ID}")
|
||||
.where(
|
||||
"""
|
||||
${buildAttachmentsThatNeedUploadQuery(transferStateFilter = "$ARCHIVE_THUMBNAIL_TRANSFER_STATE != ${ArchiveTransferState.PERMANENT_FAILURE.value}")} AND
|
||||
$QUOTE = 0 AND
|
||||
($CONTENT_TYPE LIKE 'image/%' OR $CONTENT_TYPE LIKE 'video/%') AND
|
||||
$CONTENT_TYPE != 'image/svg+xml'
|
||||
"""
|
||||
)
|
||||
readableDatabase
|
||||
.select(DATA_HASH_END, REMOTE_KEY)
|
||||
.from(TABLE_NAME)
|
||||
.where("$DATA_HASH_END NOT NULL AND $REMOTE_KEY NOT NULL AND $ARCHIVE_THUMBNAIL_TRANSFER_STATE = ${ArchiveTransferState.PERMANENT_FAILURE.value}")
|
||||
.run()
|
||||
.forEach { cursor ->
|
||||
val hashEnd = cursor.requireNonNullString(DATA_HASH_END)
|
||||
val remoteKey = cursor.requireNonNullString(REMOTE_KEY)
|
||||
val thumbnailMediaName = MediaName.fromPlaintextHashAndRemoteKeyForThumbnail(Base64.decode(hashEnd), Base64.decode(remoteKey)).name
|
||||
|
||||
entriesByMediaName.remove(thumbnailMediaName)
|
||||
}
|
||||
|
||||
return entriesByMediaName.values.toSet()
|
||||
}
|
||||
|
||||
fun hasData(attachmentId: AttachmentId): Boolean {
|
||||
@@ -566,6 +559,25 @@ class AttachmentTable(
|
||||
.flatten()
|
||||
}
|
||||
|
||||
fun getLocalArchivableAttachment(plaintextHash: String, remoteKey: String): LocalArchivableAttachment? {
|
||||
return readableDatabase
|
||||
.select(*PROJECTION)
|
||||
.from(TABLE_NAME)
|
||||
.where("$DATA_HASH_END = ? AND $REMOTE_KEY = ?")
|
||||
.orderBy("$ID DESC")
|
||||
.limit(1)
|
||||
.run()
|
||||
.readToSingleObject {
|
||||
LocalArchivableAttachment(
|
||||
file = File(it.requireNonNullString(DATA_FILE)),
|
||||
random = it.requireNonNullBlob(DATA_RANDOM),
|
||||
size = it.requireLong(DATA_SIZE),
|
||||
remoteKey = Base64.decode(it.requireNonNullString(REMOTE_KEY)),
|
||||
plaintextHash = Base64.decode(it.requireNonNullString(DATA_HASH_END))
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
fun getLocalArchivableAttachments(): List<LocalArchivableAttachment> {
|
||||
return readableDatabase
|
||||
.select(*PROJECTION)
|
||||
@@ -3214,7 +3226,7 @@ class AttachmentTable(
|
||||
.select(*PROJECTION)
|
||||
.from(TABLE_NAME)
|
||||
.where("$REMOTE_KEY NOT NULL AND $DATA_HASH_END NOT NULL")
|
||||
.groupBy(DATA_HASH_END)
|
||||
.groupBy("$DATA_HASH_END, $REMOTE_KEY")
|
||||
.run()
|
||||
.forEach { cursor ->
|
||||
val remoteKey = Base64.decode(cursor.requireNonNullString(REMOTE_KEY))
|
||||
@@ -3239,7 +3251,6 @@ class AttachmentTable(
|
||||
|
||||
fun debugGetAttachmentStats(): DebugAttachmentStats {
|
||||
val totalAttachmentRows = readableDatabase.count().from(TABLE_NAME).run().readToSingleLong(0)
|
||||
val totalEligibleForUploadRows = getFullSizeAttachmentsThatWillBeIncludedInArchive().count
|
||||
|
||||
val totalUniqueDataFiles = readableDatabase.select("COUNT(DISTINCT $DATA_FILE)").from(TABLE_NAME).run().readToSingleLong(0)
|
||||
val totalUniqueMediaNames = readableDatabase.query("SELECT COUNT(*) FROM (SELECT DISTINCT $DATA_HASH_END, $REMOTE_KEY FROM $TABLE_NAME WHERE $DATA_HASH_END NOT NULL AND $REMOTE_KEY NOT NULL)").readToSingleLong(0)
|
||||
@@ -3309,15 +3320,19 @@ class AttachmentTable(
|
||||
val uploadedThumbnailCount = archiveStatusMediaNameThumbnailCounts.getOrDefault(ArchiveTransferState.FINISHED, 0L)
|
||||
val uploadedThumbnailBytes = uploadedThumbnailCount * RemoteConfig.backupMaxThumbnailFileSize.inWholeBytes
|
||||
|
||||
val lastSnapshotFullSizeCount = SignalDatabase.backupMediaSnapshots.debugGetFullSizeAttachmentCountForMostRecentSnapshot()
|
||||
val lastSnapshotThumbnailCount = SignalDatabase.backupMediaSnapshots.debugGetThumbnailAttachmentCountForMostRecentSnapshot()
|
||||
|
||||
return DebugAttachmentStats(
|
||||
totalAttachmentRows = totalAttachmentRows,
|
||||
totalEligibleForUploadRows = totalEligibleForUploadRows.toLong(),
|
||||
totalUniqueMediaNamesEligibleForUpload = totalUniqueMediaNamesEligibleForUpload,
|
||||
totalUniqueDataFiles = totalUniqueDataFiles,
|
||||
totalUniqueMediaNames = totalUniqueMediaNames,
|
||||
archiveStatusMediaNameCounts = archiveStatusMediaNameCounts,
|
||||
mediaNamesWithThumbnailsCount = uniqueEligibleMediaNamesWithThumbnailsCount,
|
||||
archiveStatusMediaNameThumbnailCounts = archiveStatusMediaNameThumbnailCounts,
|
||||
lastSnapshotFullSizeCount = lastSnapshotFullSizeCount.toLong(),
|
||||
lastSnapshotThumbnailCount = lastSnapshotThumbnailCount.toLong(),
|
||||
pendingAttachmentUploadBytes = pendingAttachmentUploadBytes,
|
||||
uploadedAttachmentBytes = uploadedAttachmentBytes,
|
||||
uploadedThumbnailBytes = uploadedThumbnailBytes
|
||||
@@ -3727,13 +3742,14 @@ class AttachmentTable(
|
||||
|
||||
data class DebugAttachmentStats(
|
||||
val totalAttachmentRows: Long = 0L,
|
||||
val totalEligibleForUploadRows: Long = 0L,
|
||||
val totalUniqueMediaNamesEligibleForUpload: Long = 0L,
|
||||
val totalUniqueDataFiles: Long = 0L,
|
||||
val totalUniqueMediaNames: Long = 0L,
|
||||
val archiveStatusMediaNameCounts: Map<ArchiveTransferState, Long> = emptyMap(),
|
||||
val mediaNamesWithThumbnailsCount: Long = 0L,
|
||||
val archiveStatusMediaNameThumbnailCounts: Map<ArchiveTransferState, Long> = emptyMap(),
|
||||
val lastSnapshotFullSizeCount: Long = 0L,
|
||||
val lastSnapshotThumbnailCount: Long = 0L,
|
||||
val pendingAttachmentUploadBytes: Long = 0L,
|
||||
val uploadedAttachmentBytes: Long = 0L,
|
||||
val uploadedThumbnailBytes: Long = 0L
|
||||
@@ -3747,12 +3763,13 @@ class AttachmentTable(
|
||||
fun prettyString(): String {
|
||||
return buildString {
|
||||
appendLine("Total attachment rows: $totalAttachmentRows")
|
||||
appendLine("Total eligible for upload rows: $totalEligibleForUploadRows")
|
||||
appendLine("Total unique media names eligible for upload: $totalUniqueMediaNamesEligibleForUpload")
|
||||
appendLine("Total unique data files: $totalUniqueDataFiles")
|
||||
appendLine("Total unique media names: $totalUniqueMediaNames")
|
||||
appendLine("Media names with thumbnails count: $mediaNamesWithThumbnailsCount")
|
||||
appendLine("Pending attachment upload bytes: $pendingAttachmentUploadBytes")
|
||||
appendLine("Last snapshot full-size count: $lastSnapshotFullSizeCount")
|
||||
appendLine("Last snapshot thumbnail count : $lastSnapshotFullSizeCount")
|
||||
appendLine("Uploaded attachment bytes: $uploadedAttachmentBytes")
|
||||
appendLine("Uploaded thumbnail bytes: $uploadedThumbnailBytes")
|
||||
appendLine("Total upload count: $totalUploadCount")
|
||||
@@ -3776,10 +3793,11 @@ class AttachmentTable(
|
||||
|
||||
fun shortPrettyString(): String {
|
||||
return buildString {
|
||||
appendLine("Total eligible for upload rows: $totalEligibleForUploadRows")
|
||||
appendLine("Total unique media names eligible for upload: $totalUniqueMediaNamesEligibleForUpload")
|
||||
appendLine("Total unique data files: $totalUniqueDataFiles")
|
||||
appendLine("Total unique media names: $totalUniqueMediaNames")
|
||||
appendLine("Last snapshot full-size count: $lastSnapshotFullSizeCount")
|
||||
appendLine("Last snapshot thumbnail count : $lastSnapshotFullSizeCount")
|
||||
appendLine("Pending attachment upload bytes: $pendingAttachmentUploadBytes")
|
||||
|
||||
if (archiveStatusMediaNameCounts.isNotEmpty()) {
|
||||
|
||||
@@ -10,10 +10,12 @@ import android.database.Cursor
|
||||
import androidx.annotation.VisibleForTesting
|
||||
import androidx.core.content.contentValuesOf
|
||||
import org.signal.core.util.SqlUtil
|
||||
import org.signal.core.util.count
|
||||
import org.signal.core.util.delete
|
||||
import org.signal.core.util.forEach
|
||||
import org.signal.core.util.readToList
|
||||
import org.signal.core.util.readToSet
|
||||
import org.signal.core.util.readToSingleInt
|
||||
import org.signal.core.util.readToSingleLong
|
||||
import org.signal.core.util.requireBoolean
|
||||
import org.signal.core.util.requireInt
|
||||
@@ -129,33 +131,43 @@ class BackupMediaSnapshotTable(context: Context, database: SignalDatabase) : Dat
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes the set of full-size media items that are slated to be referenced in the next backup, updating their pending sync time.
|
||||
* Writes a set of [MediaEntry] that are slated to be referenced in the next backup, updating their pending sync time.
|
||||
*/
|
||||
fun writeFullSizePendingMediaObjects(mediaObjects: Sequence<ArchiveMediaItem>) {
|
||||
mediaObjects
|
||||
.chunked(SqlUtil.MAX_QUERY_ARGS)
|
||||
.forEach { chunk ->
|
||||
writePendingMediaObjectsChunk(
|
||||
chunk.map { MediaEntry(it.mediaId, it.cdn, it.plaintextHash, it.remoteKey, isThumbnail = false) }
|
||||
)
|
||||
}
|
||||
fun writePendingMediaEntries(entries: Collection<MediaEntry>) {
|
||||
if (entries.isEmpty()) {
|
||||
return
|
||||
}
|
||||
|
||||
val values = entries.map {
|
||||
contentValuesOf(
|
||||
MEDIA_ID to it.mediaId,
|
||||
CDN to it.cdn,
|
||||
PLAINTEXT_HASH to it.plaintextHash,
|
||||
REMOTE_KEY to it.remoteKey,
|
||||
IS_THUMBNAIL to it.isThumbnail.toInt(),
|
||||
SNAPSHOT_VERSION to UNKNOWN_VERSION,
|
||||
IS_PENDING to 1
|
||||
)
|
||||
}
|
||||
|
||||
SqlUtil.buildBulkInsert(TABLE_NAME, arrayOf(MEDIA_ID, CDN, PLAINTEXT_HASH, REMOTE_KEY, IS_THUMBNAIL, SNAPSHOT_VERSION, IS_PENDING), values).forEach { query ->
|
||||
writableDatabase.execSQL(
|
||||
query.where +
|
||||
"""
|
||||
ON CONFLICT($MEDIA_ID) DO UPDATE SET
|
||||
$CDN = excluded.$CDN,
|
||||
$PLAINTEXT_HASH = excluded.$PLAINTEXT_HASH,
|
||||
$REMOTE_KEY = excluded.$REMOTE_KEY,
|
||||
$IS_THUMBNAIL = excluded.$IS_THUMBNAIL,
|
||||
$IS_PENDING = excluded.$IS_PENDING
|
||||
""",
|
||||
query.whereArgs
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes the set of thumbnail media items that are slated to be referenced in the next backup, updating their pending sync time.
|
||||
*/
|
||||
fun writeThumbnailPendingMediaObjects(mediaObjects: Sequence<ArchiveMediaItem>) {
|
||||
mediaObjects
|
||||
.chunked(SqlUtil.MAX_QUERY_ARGS)
|
||||
.forEach { chunk ->
|
||||
writePendingMediaObjectsChunk(
|
||||
chunk.map { MediaEntry(it.thumbnailMediaId, it.cdn, it.plaintextHash, it.remoteKey, isThumbnail = true) }
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Commits all pending entries (written via [writePendingMediaObjects]) to have a concrete [SNAPSHOT_VERSION]. The version will be 1 higher than the previous
|
||||
* Commits all pending entries (written via [writePendingMediaEntries]) to have a concrete [SNAPSHOT_VERSION]. The version will be 1 higher than the previous
|
||||
* snapshot version.
|
||||
*/
|
||||
fun commitPendingRows() {
|
||||
@@ -326,37 +338,22 @@ class BackupMediaSnapshotTable(context: Context, database: SignalDatabase) : Dat
|
||||
.run()
|
||||
}
|
||||
|
||||
private fun writePendingMediaObjectsChunk(chunk: List<MediaEntry>) {
|
||||
if (chunk.isEmpty()) {
|
||||
return
|
||||
}
|
||||
fun debugGetFullSizeAttachmentCountForMostRecentSnapshot(): Int {
|
||||
return readableDatabase
|
||||
.count()
|
||||
.from(TABLE_NAME)
|
||||
.where("$IS_THUMBNAIL = 0 AND $SNAPSHOT_VERSION = $MAX_VERSION")
|
||||
.run()
|
||||
.readToSingleInt()
|
||||
}
|
||||
|
||||
val values = chunk.map {
|
||||
contentValuesOf(
|
||||
MEDIA_ID to it.mediaId,
|
||||
CDN to it.cdn,
|
||||
PLAINTEXT_HASH to it.plaintextHash,
|
||||
REMOTE_KEY to it.remoteKey,
|
||||
IS_THUMBNAIL to it.isThumbnail.toInt(),
|
||||
SNAPSHOT_VERSION to UNKNOWN_VERSION,
|
||||
IS_PENDING to 1
|
||||
)
|
||||
}
|
||||
|
||||
val query = SqlUtil.buildSingleBulkInsert(TABLE_NAME, arrayOf(MEDIA_ID, CDN, PLAINTEXT_HASH, REMOTE_KEY, IS_THUMBNAIL, SNAPSHOT_VERSION, IS_PENDING), values)
|
||||
|
||||
writableDatabase.execSQL(
|
||||
query.where +
|
||||
"""
|
||||
ON CONFLICT($MEDIA_ID) DO UPDATE SET
|
||||
$CDN = excluded.$CDN,
|
||||
$PLAINTEXT_HASH = excluded.$PLAINTEXT_HASH,
|
||||
$REMOTE_KEY = excluded.$REMOTE_KEY,
|
||||
$IS_THUMBNAIL = excluded.$IS_THUMBNAIL,
|
||||
$IS_PENDING = excluded.$IS_PENDING
|
||||
""",
|
||||
query.whereArgs
|
||||
)
|
||||
fun debugGetThumbnailAttachmentCountForMostRecentSnapshot(): Int {
|
||||
return readableDatabase
|
||||
.count()
|
||||
.from(TABLE_NAME)
|
||||
.where("$IS_THUMBNAIL != 0 AND $SNAPSHOT_VERSION = $MAX_VERSION")
|
||||
.run()
|
||||
.readToSingleInt()
|
||||
}
|
||||
|
||||
class ArchiveMediaItem(
|
||||
|
||||
@@ -23,11 +23,13 @@ import org.signal.libsignal.net.SvrBStoreResponse
|
||||
import org.signal.protos.resumableuploads.ResumableUpload
|
||||
import org.thoughtcrime.securesms.R
|
||||
import org.thoughtcrime.securesms.backup.ArchiveUploadProgress
|
||||
import org.thoughtcrime.securesms.backup.v2.ArchiveMediaItemIterator
|
||||
import org.thoughtcrime.securesms.backup.v2.ArchiveRestoreProgress
|
||||
import org.thoughtcrime.securesms.backup.v2.ArchiveValidator
|
||||
import org.thoughtcrime.securesms.backup.v2.BackupRepository
|
||||
import org.thoughtcrime.securesms.backup.v2.ResumableMessagesBackupUploadSpec
|
||||
import org.thoughtcrime.securesms.backup.v2.util.ArchiveAttachmentInfo
|
||||
import org.thoughtcrime.securesms.backup.v2.util.getAllReferencedArchiveAttachmentInfos
|
||||
import org.thoughtcrime.securesms.database.BackupMediaSnapshotTable
|
||||
import org.thoughtcrime.securesms.database.SignalDatabase
|
||||
import org.thoughtcrime.securesms.dependencies.AppDependencies
|
||||
import org.thoughtcrime.securesms.jobmanager.Job
|
||||
@@ -42,8 +44,10 @@ import org.thoughtcrime.securesms.notifications.NotificationIds
|
||||
import org.thoughtcrime.securesms.providers.BlobProvider
|
||||
import org.thoughtcrime.securesms.recipients.Recipient
|
||||
import org.thoughtcrime.securesms.storage.StorageSyncHelper
|
||||
import org.thoughtcrime.securesms.util.MediaUtil
|
||||
import org.thoughtcrime.securesms.util.RemoteConfig
|
||||
import org.whispersystems.signalservice.api.NetworkResult
|
||||
import org.whispersystems.signalservice.api.backup.MediaRootBackupKey
|
||||
import org.whispersystems.signalservice.api.messages.AttachmentTransferProgress
|
||||
import org.whispersystems.signalservice.api.messages.SignalServiceAttachment
|
||||
import org.whispersystems.signalservice.api.svr.SvrBApi
|
||||
@@ -68,6 +72,7 @@ class BackupMessagesJob private constructor(
|
||||
companion object {
|
||||
private val TAG = Log.tag(BackupMessagesJob::class.java)
|
||||
private val FILE_REUSE_TIMEOUT = 1.hours
|
||||
private const val ATTACHMENT_SNAPSHOT_BUFFER_SIZE = 10_000
|
||||
|
||||
const val KEY = "BackupMessagesJob"
|
||||
|
||||
@@ -360,8 +365,11 @@ class BackupMessagesJob private constructor(
|
||||
|
||||
val outputStream = FileOutputStream(tempBackupFile)
|
||||
val backupKey = SignalStore.backup.messageBackupKey
|
||||
val mediaRootBackupKey = SignalStore.backup.mediaRootBackupKey
|
||||
val currentTime = System.currentTimeMillis()
|
||||
|
||||
val attachmentInfoBuffer: MutableSet<ArchiveAttachmentInfo> = mutableSetOf()
|
||||
|
||||
BackupRepository.exportForSignalBackup(
|
||||
outputStream = outputStream,
|
||||
messageBackupKey = backupKey,
|
||||
@@ -371,8 +379,19 @@ class BackupMessagesJob private constructor(
|
||||
append = { tempBackupFile.appendBytes(it) },
|
||||
cancellationSignal = { this.isCanceled },
|
||||
currentTime = currentTime
|
||||
) {
|
||||
writeMediaCursorToTemporaryTable(it, mediaBackupEnabled = SignalStore.backup.backsUpMedia)
|
||||
) { frame ->
|
||||
attachmentInfoBuffer += frame.getAllReferencedArchiveAttachmentInfos()
|
||||
if (attachmentInfoBuffer.size > ATTACHMENT_SNAPSHOT_BUFFER_SIZE) {
|
||||
SignalDatabase.backupMediaSnapshots.writePendingMediaEntries(attachmentInfoBuffer.toFullSizeMediaEntries(mediaRootBackupKey))
|
||||
SignalDatabase.backupMediaSnapshots.writePendingMediaEntries(attachmentInfoBuffer.toThumbnailMediaEntries(mediaRootBackupKey))
|
||||
attachmentInfoBuffer.clear()
|
||||
}
|
||||
}
|
||||
|
||||
if (attachmentInfoBuffer.isNotEmpty()) {
|
||||
SignalDatabase.backupMediaSnapshots.writePendingMediaEntries(attachmentInfoBuffer.toFullSizeMediaEntries(mediaRootBackupKey))
|
||||
SignalDatabase.backupMediaSnapshots.writePendingMediaEntries(attachmentInfoBuffer.toThumbnailMediaEntries(mediaRootBackupKey))
|
||||
attachmentInfoBuffer.clear()
|
||||
}
|
||||
|
||||
if (isCanceled) {
|
||||
@@ -422,22 +441,6 @@ class BackupMessagesJob private constructor(
|
||||
)
|
||||
}
|
||||
|
||||
private fun writeMediaCursorToTemporaryTable(db: SignalDatabase, mediaBackupEnabled: Boolean) {
|
||||
if (mediaBackupEnabled) {
|
||||
db.attachmentTable.getFullSizeAttachmentsThatWillBeIncludedInArchive().use {
|
||||
SignalDatabase.backupMediaSnapshots.writeFullSizePendingMediaObjects(
|
||||
mediaObjects = ArchiveMediaItemIterator(it).asSequence()
|
||||
)
|
||||
}
|
||||
|
||||
db.attachmentTable.getThumbnailAttachmentsThatWillBeIncludedInArchive().use {
|
||||
SignalDatabase.backupMediaSnapshots.writeThumbnailPendingMediaObjects(
|
||||
mediaObjects = ArchiveMediaItemIterator(it).asSequence()
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun maybePostRemoteKeyMissingNotification() {
|
||||
if (!RemoteConfig.internalUser || !SignalStore.backup.backsUpMedia) {
|
||||
return
|
||||
@@ -457,6 +460,42 @@ class BackupMessagesJob private constructor(
|
||||
NotificationManagerCompat.from(context).notify(NotificationIds.INTERNAL_ERROR, notification)
|
||||
}
|
||||
|
||||
private fun Set<ArchiveAttachmentInfo>.toFullSizeMediaEntries(mediaRootBackupKey: MediaRootBackupKey): Set<BackupMediaSnapshotTable.MediaEntry> {
|
||||
return this
|
||||
.map {
|
||||
BackupMediaSnapshotTable.MediaEntry(
|
||||
mediaId = it.fullSizeMediaName.toMediaId(mediaRootBackupKey).encode(),
|
||||
cdn = it.cdn,
|
||||
plaintextHash = it.plaintextHash.toByteArray(),
|
||||
remoteKey = it.remoteKey.toByteArray(),
|
||||
isThumbnail = false
|
||||
)
|
||||
}
|
||||
.toSet()
|
||||
}
|
||||
|
||||
/**
|
||||
* Note: we have to remove permanently failed thumbnails here because there's no way we can know from the backup frame whether or not the thumbnail
|
||||
* failed permanently independently of the attachment itself. If the attachment itself fails permanently, it's not put in the backup, so we're covered
|
||||
* for full-size stuff.
|
||||
*/
|
||||
private fun Set<ArchiveAttachmentInfo>.toThumbnailMediaEntries(mediaRootBackupKey: MediaRootBackupKey): Set<BackupMediaSnapshotTable.MediaEntry> {
|
||||
return this
|
||||
.filter { MediaUtil.isImageOrVideoType(it.contentType) }
|
||||
.filterNot { it.forQuote }
|
||||
.map {
|
||||
BackupMediaSnapshotTable.MediaEntry(
|
||||
mediaId = it.thumbnailMediaName.toMediaId(mediaRootBackupKey).encode(),
|
||||
cdn = it.cdn,
|
||||
plaintextHash = it.plaintextHash.toByteArray(),
|
||||
remoteKey = it.remoteKey.toByteArray(),
|
||||
isThumbnail = true
|
||||
)
|
||||
}
|
||||
.toSet()
|
||||
.let { SignalDatabase.attachments.filterPermanentlyFailedThumbnails(it) }
|
||||
}
|
||||
|
||||
class Factory : Job.Factory<BackupMessagesJob> {
|
||||
override fun create(parameters: Parameters, serializedData: ByteArray?): BackupMessagesJob {
|
||||
val jobData = if (serializedData != null) {
|
||||
|
||||
Reference in New Issue
Block a user