Signal/Android
1
0
Fork 0
mirror of https://github.com/signalapp/Signal-Android.git synced 2026-02-15 07:28:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Dynamically compute MediaName and MediaId.

Browse Source
This commit is contained in:
Greyson Parrelli
2025-03-28 14:19:16 -04:00
parent f1985cf506
commit 17216316f6
34 changed files with 641 additions and 396 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
app/src/androidTest/java/org/thoughtcrime/securesms/database/AttachmentTableTest_deduping.kt
View File

@@ -18,7 +18,6 @@ import org.signal.core.util.update
import org.thoughtcrime.securesms.attachments.AttachmentId
import org.thoughtcrime.securesms.attachments.Cdn
import org.thoughtcrime.securesms.attachments.PointerAttachment
import org.thoughtcrime.securesms.backup.v2.BackupRepository.getMediaName
import org.thoughtcrime.securesms.database.AttachmentTable.TransformProperties
import org.thoughtcrime.securesms.keyvalue.SignalStore
import org.thoughtcrime.securesms.mms.MediaStream
@@ -30,7 +29,6 @@ import org.thoughtcrime.securesms.recipients.Recipient
import org.thoughtcrime.securesms.util.MediaUtil
import org.thoughtcrime.securesms.util.Util
import org.whispersystems.signalservice.api.attachment.AttachmentUploadResult
import org.whispersystems.signalservice.api.backup.MediaId
import org.whispersystems.signalservice.api.messages.SignalServiceAttachmentRemoteId
import org.whispersystems.signalservice.api.push.ServiceId
import org.whispersystems.signalservice.internal.crypto.PaddingInputStream
@@ -734,12 +732,9 @@ class AttachmentTableTest_deduping {
SignalDatabase.attachments.finalizeAttachmentAfterUpload(attachmentId, createUploadResult(attachmentId, uploadTimestamp))
val attachment = SignalDatabase.attachments.getAttachment(attachmentId)!!
SignalDatabase.attachments.setArchiveData(
SignalDatabase.attachments.setArchiveCdn(
attachmentId = attachmentId,
archiveCdn = Cdn.CDN_3.cdnNumber,
archiveMediaName = attachment.getMediaName().name,
archiveThumbnailMediaId = MediaId(Util.getSecretBytes(15)).encode(),
archiveMediaId = MediaId(Util.getSecretBytes(15)).encode()
archiveCdn = Cdn.CDN_3.cdnNumber
)
}
@@ -861,8 +856,6 @@ class AttachmentTableTest_deduping {
val rhsAttachment = SignalDatabase.attachments.getAttachment(rhs)!!
assertEquals(lhsAttachment.archiveCdn, rhsAttachment.archiveCdn)
assertEquals(lhsAttachment.archiveMediaName, rhsAttachment.archiveMediaName)
assertEquals(lhsAttachment.archiveMediaId, rhsAttachment.archiveMediaId)
}
fun assertDoesNotHaveRemoteFields(attachmentId: AttachmentId) {

131
app/src/androidTest/java/org/thoughtcrime/securesms/database/BackupMediaSnapshotTableTest.kt
View File

@@ -1,5 +1,6 @@
package org.thoughtcrime.securesms.database
import androidx.media3.common.util.Util
import androidx.test.ext.junit.runners.AndroidJUnit4
import assertk.assertThat
import assertk.assertions.isEqualTo
@@ -9,6 +10,7 @@ import org.junit.runner.RunWith
import org.signal.core.util.count
import org.signal.core.util.readToSingleInt
import org.thoughtcrime.securesms.backup.v2.ArchivedMediaObject
import org.thoughtcrime.securesms.database.BackupMediaSnapshotTable.ArchiveMediaItem
import org.thoughtcrime.securesms.testing.SignalActivityRule
@RunWith(AndroidJUnit4::class)
@@ -16,6 +18,7 @@ class BackupMediaSnapshotTableTest {
companion object {
private const val SEQUENCE_COUNT = 100
private const val SEQUENCE_COUNT_WITH_THUMBNAILS = 200
}
@get:Rule
@@ -24,7 +27,7 @@ class BackupMediaSnapshotTableTest {
@Test
fun givenAnEmptyTable_whenIWriteToTable_thenIExpectEmptyTable() {
val pendingSyncTime = 1L
SignalDatabase.backupMediaSnapshots.writePendingMediaObjects(generateArchiveObjectSequence(), pendingSyncTime)
SignalDatabase.backupMediaSnapshots.writePendingMediaObjects(generateArchiveMediaItemSequence(), pendingSyncTime)
val count = getSyncedItemCount(pendingSyncTime)
@@ -34,22 +37,23 @@ class BackupMediaSnapshotTableTest {
@Test
fun givenAnEmptyTable_whenIWriteToTableAndCommit_thenIExpectFilledTable() {
val pendingSyncTime = 1L
SignalDatabase.backupMediaSnapshots.writePendingMediaObjects(generateArchiveObjectSequence(), pendingSyncTime)
SignalDatabase.backupMediaSnapshots.writePendingMediaObjects(generateArchiveMediaItemSequence(), pendingSyncTime)
SignalDatabase.backupMediaSnapshots.commitPendingRows()
val count = getSyncedItemCount(pendingSyncTime)
assertThat(count).isEqualTo(SEQUENCE_COUNT)
assertThat(count).isEqualTo(SEQUENCE_COUNT_WITH_THUMBNAILS)
}
@Test
fun givenAFilledTable_whenIInsertSimilarIds_thenIExpectUncommittedOverrides() {
SignalDatabase.backupMediaSnapshots.writePendingMediaObjects(generateArchiveObjectSequence(), 1L)
SignalDatabase.backupMediaSnapshots.writePendingMediaObjects(generateArchiveMediaItemSequence(), 1L)
SignalDatabase.backupMediaSnapshots.commitPendingRows()
val newPendingTime = 2L
val newObjectCount = 50
SignalDatabase.backupMediaSnapshots.writePendingMediaObjects(generateArchiveObjectSequence(newObjectCount), newPendingTime)
val newObjectCountWithThumbnails = newObjectCount * 2
SignalDatabase.backupMediaSnapshots.writePendingMediaObjects(generateArchiveMediaItemSequence(newObjectCount), newPendingTime)
val count = SignalDatabase.backupMediaSnapshots.readableDatabase.count()
.from(BackupMediaSnapshotTable.TABLE_NAME)
@@ -57,17 +61,18 @@ class BackupMediaSnapshotTableTest {
.run()
.readToSingleInt(-1)
assertThat(count).isEqualTo(50)
assertThat(count).isEqualTo(newObjectCountWithThumbnails)
}
@Test
fun givenAFilledTable_whenIInsertSimilarIdsAndCommit_thenIExpectCommittedOverrides() {
SignalDatabase.backupMediaSnapshots.writePendingMediaObjects(generateArchiveObjectSequence(), 1L)
SignalDatabase.backupMediaSnapshots.writePendingMediaObjects(generateArchiveMediaItemSequence(), 1L)
SignalDatabase.backupMediaSnapshots.commitPendingRows()
val newPendingTime = 2L
val newObjectCount = 50
SignalDatabase.backupMediaSnapshots.writePendingMediaObjects(generateArchiveObjectSequence(newObjectCount), newPendingTime)
val newObjectCountWithThumbnails = newObjectCount * 2
SignalDatabase.backupMediaSnapshots.writePendingMediaObjects(generateArchiveMediaItemSequence(newObjectCount), newPendingTime)
SignalDatabase.backupMediaSnapshots.commitPendingRows()
val count = SignalDatabase.backupMediaSnapshots.readableDatabase.count()
@@ -78,18 +83,19 @@ class BackupMediaSnapshotTableTest {
val total = getTotalItemCount()
assertThat(count).isEqualTo(50)
assertThat(total).isEqualTo(SEQUENCE_COUNT)
assertThat(count).isEqualTo(newObjectCountWithThumbnails)
assertThat(total).isEqualTo(SEQUENCE_COUNT_WITH_THUMBNAILS)
}
@Test
fun givenAFilledTable_whenIInsertSimilarIdsAndCommitThenDelete_thenIExpectOnlyCommittedOverrides() {
SignalDatabase.backupMediaSnapshots.writePendingMediaObjects(generateArchiveObjectSequence(), 1L)
SignalDatabase.backupMediaSnapshots.writePendingMediaObjects(generateArchiveMediaItemSequence(), 1L)
SignalDatabase.backupMediaSnapshots.commitPendingRows()
val newPendingTime = 2L
val newObjectCount = 50
SignalDatabase.backupMediaSnapshots.writePendingMediaObjects(generateArchiveObjectSequence(newObjectCount), newPendingTime)
val newObjectCountWithThumbnails = newObjectCount * 2
SignalDatabase.backupMediaSnapshots.writePendingMediaObjects(generateArchiveMediaItemSequence(newObjectCount), newPendingTime)
SignalDatabase.backupMediaSnapshots.commitPendingRows()
val page = SignalDatabase.backupMediaSnapshots.getPageOfOldMediaObjects(currentSyncTime = newPendingTime, pageSize = 100)
@@ -97,7 +103,86 @@ class BackupMediaSnapshotTableTest {
val total = getTotalItemCount()
assertThat(total).isEqualTo(50)
assertThat(total).isEqualTo(newObjectCountWithThumbnails)
}
@Test
fun getMediaObjectsWithNonMatchingCdn_noMismatches() {
val localData = listOf(
createArchiveMediaItem(seed = 1, cdn = 1),
createArchiveMediaItem(seed = 2, cdn = 2)
)
val remoteData = listOf(
createArchiveMediaObject(seed = 1, cdn = 1),
createArchiveMediaObject(seed = 2, cdn = 2)
)
SignalDatabase.backupMediaSnapshots.writePendingMediaObjects(localData.asSequence(), 1L)
SignalDatabase.backupMediaSnapshots.commitPendingRows()
val mismatches = SignalDatabase.backupMediaSnapshots.getMediaObjectsWithNonMatchingCdn(remoteData)
assertThat(mismatches.size).isEqualTo(0)
}
@Test
fun getMediaObjectsWithNonMatchingCdn_oneMismatch() {
val localData = listOf(
createArchiveMediaItem(seed = 1, cdn = 1),
createArchiveMediaItem(seed = 2, cdn = 2)
)
val remoteData = listOf(
createArchiveMediaObject(seed = 1, cdn = 1),
createArchiveMediaObject(seed = 2, cdn = 99)
)
SignalDatabase.backupMediaSnapshots.writePendingMediaObjects(localData.asSequence(), 1L)
SignalDatabase.backupMediaSnapshots.commitPendingRows()
val mismatches = SignalDatabase.backupMediaSnapshots.getMediaObjectsWithNonMatchingCdn(remoteData)
assertThat(mismatches.size).isEqualTo(1)
assertThat(mismatches.get(0).cdn).isEqualTo(99)
assertThat(mismatches.get(0).digest).isEqualTo(localData.get(1).digest)
}
@Test
fun getMediaObjectsThatCantBeFound_allFound() {
val localData = listOf(
createArchiveMediaItem(seed = 1, cdn = 1),
createArchiveMediaItem(seed = 2, cdn = 2)
)
val remoteData = listOf(
createArchiveMediaObject(seed = 1, cdn = 1),
createArchiveMediaObject(seed = 2, cdn = 2)
)
SignalDatabase.backupMediaSnapshots.writePendingMediaObjects(localData.asSequence(), 1L)
SignalDatabase.backupMediaSnapshots.commitPendingRows()
val notFound = SignalDatabase.backupMediaSnapshots.getMediaObjectsThatCantBeFound(remoteData)
assertThat(notFound.size).isEqualTo(0)
}
@Test
fun getMediaObjectsThatCantBeFound_oneMissing() {
val localData = listOf(
createArchiveMediaItem(seed = 1, cdn = 1),
createArchiveMediaItem(seed = 2, cdn = 2)
)
val remoteData = listOf(
createArchiveMediaObject(seed = 1, cdn = 1),
createArchiveMediaObject(seed = 3, cdn = 2)
)
SignalDatabase.backupMediaSnapshots.writePendingMediaObjects(localData.asSequence(), 1L)
SignalDatabase.backupMediaSnapshots.commitPendingRows()
val notFound = SignalDatabase.backupMediaSnapshots.getMediaObjectsThatCantBeFound(remoteData)
assertThat(notFound.size).isEqualTo(1)
assertThat(notFound.first().cdn).isEqualTo(2)
}
private fun getTotalItemCount(): Int {
@@ -112,8 +197,24 @@ class BackupMediaSnapshotTableTest {
.readToSingleInt(-1)
}
private fun generateArchiveObjectSequence(count: Int = SEQUENCE_COUNT): Sequence<ArchivedMediaObject> {
private fun generateArchiveMediaItemSequence(count: Int = SEQUENCE_COUNT): Sequence<ArchiveMediaItem> {
return generateSequence(0) { seed -> if (seed < (count - 1)) seed + 1 else null }
.map { ArchivedMediaObject(mediaId = "media_id_$it", 0) }
.map { createArchiveMediaItem(it) }
}
private fun createArchiveMediaItem(seed: Int, cdn: Int = 0): ArchiveMediaItem {
return ArchiveMediaItem(
mediaId = "media_id_$seed",
thumbnailMediaId = "thumbnail_media_id_$seed",
cdn = cdn,
digest = Util.toByteArray(seed)
)
}
private fun createArchiveMediaObject(seed: Int, cdn: Int = 0): ArchivedMediaObject {
return ArchivedMediaObject(
mediaId = "media_id_$seed",
cdn = cdn
)
}
}

3
app/src/androidTest/java/org/thoughtcrime/securesms/messages/SyncMessageProcessorTest_synchronizeDeleteForMe.kt
View File

@@ -677,7 +677,6 @@ class SyncMessageProcessorTest_synchronizeDeleteForMe {
mmsId = this.mmsId,
hasData = this.hasData,
hasThumbnail = false,
hasArchiveThumbnail = false,
contentType = this.contentType,
transferProgress = this.transferState,
size = this.size,
@@ -705,8 +704,6 @@ class SyncMessageProcessorTest_synchronizeDeleteForMe {
uploadTimestamp = this.uploadTimestamp,
dataHash = this.dataHash,
archiveCdn = this.archiveCdn,
archiveMediaName = this.archiveMediaName,
archiveMediaId = this.archiveMediaId,
thumbnailRestoreState = this.thumbnailRestoreState,
archiveTransferState = this.archiveTransferState,
uuid = uuid

21
app/src/main/java/org/thoughtcrime/securesms/attachments/DatabaseAttachment.kt
View File

@@ -29,19 +29,12 @@ class DatabaseAttachment : Attachment {
@JvmField
val archiveCdn: Int
@JvmField
val archiveMediaName: String?
@JvmField
val archiveMediaId: String?
@JvmField
val thumbnailRestoreState: AttachmentTable.ThumbnailRestoreState
@JvmField
val archiveTransferState: AttachmentTable.ArchiveTransferState
private val hasArchiveThumbnail: Boolean
private val hasThumbnail: Boolean
val displayOrder: Int
@@ -50,7 +43,6 @@ class DatabaseAttachment : Attachment {
mmsId: Long,
hasData: Boolean,
hasThumbnail: Boolean,
hasArchiveThumbnail: Boolean,
contentType: String?,
transferProgress: Int,
size: Long,
@@ -78,8 +70,6 @@ class DatabaseAttachment : Attachment {
uploadTimestamp: Long,
dataHash: String?,
archiveCdn: Int,
archiveMediaName: String?,
archiveMediaId: String?,
thumbnailRestoreState: AttachmentTable.ThumbnailRestoreState,
archiveTransferState: AttachmentTable.ArchiveTransferState,
uuid: UUID?
@@ -114,11 +104,8 @@ class DatabaseAttachment : Attachment {
this.hasData = hasData
this.dataHash = dataHash
this.hasThumbnail = hasThumbnail
this.hasArchiveThumbnail = hasArchiveThumbnail
this.displayOrder = displayOrder
this.archiveCdn = archiveCdn
this.archiveMediaName = archiveMediaName
this.archiveMediaId = archiveMediaId
this.thumbnailRestoreState = thumbnailRestoreState
this.archiveTransferState = archiveTransferState
}
@@ -131,9 +118,6 @@ class DatabaseAttachment : Attachment {
mmsId = parcel.readLong()
displayOrder = parcel.readInt()
archiveCdn = parcel.readInt()
archiveMediaName = parcel.readString()
archiveMediaId = parcel.readString()
hasArchiveThumbnail = ParcelUtil.readBoolean(parcel)
thumbnailRestoreState = AttachmentTable.ThumbnailRestoreState.deserialize(parcel.readInt())
archiveTransferState = AttachmentTable.ArchiveTransferState.deserialize(parcel.readInt())
}
@@ -147,9 +131,6 @@ class DatabaseAttachment : Attachment {
dest.writeLong(mmsId)
dest.writeInt(displayOrder)
dest.writeInt(archiveCdn)
dest.writeString(archiveMediaName)
dest.writeString(archiveMediaId)
ParcelUtil.writeBoolean(dest, hasArchiveThumbnail)
dest.writeInt(thumbnailRestoreState.value)
dest.writeInt(archiveTransferState.value)
}
@@ -169,7 +150,7 @@ class DatabaseAttachment : Attachment {
}
override val thumbnailUri: Uri?
get() = if (hasArchiveThumbnail) {
get() = if (thumbnailRestoreState == AttachmentTable.ThumbnailRestoreState.FINISHED) {
PartAuthority.getAttachmentThumbnailUri(attachmentId)
} else {
null

45
app/src/main/java/org/thoughtcrime/securesms/backup/v2/BackupRepository.kt
View File

@@ -28,7 +28,7 @@ import org.signal.core.util.getAllTriggerDefinitions
import org.signal.core.util.getForeignKeyViolations
import org.signal.core.util.logging.Log
import org.signal.core.util.requireInt
import org.signal.core.util.requireNonNullString
import org.signal.core.util.requireNonNullBlob
import org.signal.core.util.stream.NonClosingOutputStream
import org.signal.core.util.urlEncode
import org.signal.core.util.withinTransaction
@@ -59,6 +59,7 @@ import org.thoughtcrime.securesms.components.settings.app.subscription.Recurring
import org.thoughtcrime.securesms.crypto.AttachmentSecretProvider
import org.thoughtcrime.securesms.crypto.DatabaseSecretProvider
import org.thoughtcrime.securesms.database.AttachmentTable
import org.thoughtcrime.securesms.database.BackupMediaSnapshotTable.ArchiveMediaItem
import org.thoughtcrime.securesms.database.KeyValueDatabase
import org.thoughtcrime.securesms.database.SearchTable
import org.thoughtcrime.securesms.database.SignalDatabase
@@ -357,6 +358,7 @@ object BackupRepository {
Log.d(TAG, "Disabling backups.")
SignalStore.backup.disableBackups()
SignalDatabase.attachments.clearAllArchiveData()
true
} catch (e: Exception) {
Log.w(TAG, "Failed to turn off backups.", e)
@@ -1100,7 +1102,7 @@ object BackupRepository {
fun copyThumbnailToArchive(thumbnailAttachment: Attachment, parentAttachment: DatabaseAttachment): NetworkResult<ArchiveMediaResponse> {
return initBackupAndFetchAuth()
.then { credential ->
val request = thumbnailAttachment.toArchiveMediaRequest(parentAttachment.getThumbnailMediaName(), credential.mediaBackupAccess.backupKey)
val request = thumbnailAttachment.toArchiveMediaRequest(parentAttachment.requireThumbnailMediaName(), credential.mediaBackupAccess.backupKey)
SignalNetwork.archive.copyAttachmentToArchive(
aci = SignalStore.account.requireAci(),
@@ -1116,7 +1118,7 @@ object BackupRepository {
fun copyAttachmentToArchive(attachment: DatabaseAttachment): NetworkResult<Unit> {
return initBackupAndFetchAuth()
.then { credential ->
val mediaName = attachment.getMediaName()
val mediaName = attachment.requireMediaName()
val request = attachment.toArchiveMediaRequest(mediaName, credential.mediaBackupAccess.backupKey)
SignalNetwork.archive
.copyAttachmentToArchive(
@@ -1124,12 +1126,9 @@ object BackupRepository {
archiveServiceAccess = credential.mediaBackupAccess,
item = request
)
.map { credential to Triple(mediaName, request.mediaId, it) }
}
.map { (credential, triple) ->
val (mediaName, mediaId, response) = triple
val thumbnailId = credential.mediaBackupAccess.backupKey.deriveMediaId(attachment.getThumbnailMediaName()).encode()
SignalDatabase.attachments.setArchiveData(attachmentId = attachment.attachmentId, archiveCdn = response.cdn, archiveMediaName = mediaName.name, archiveMediaId = mediaId, archiveThumbnailMediaId = thumbnailId)
.map { response ->
SignalDatabase.attachments.setArchiveCdn(attachmentId = attachment.attachmentId, archiveCdn = response.cdn)
}
.also { Log.i(TAG, "archiveMediaResult: $it") }
}
@@ -1142,7 +1141,7 @@ object BackupRepository {
val attachmentIdToMediaName = mutableMapOf<AttachmentId, String>()
databaseAttachments.forEach {
val mediaName = it.getMediaName()
val mediaName = it.requireMediaName()
val request = it.toArchiveMediaRequest(mediaName, credential.mediaBackupAccess.backupKey)
requests += request
mediaIdToAttachmentId[request.mediaId] = it.attachmentId
@@ -1164,7 +1163,7 @@ object BackupRepository {
val attachmentId = result.mediaIdToAttachmentId(it.mediaId)
val mediaName = result.attachmentIdToMediaName(attachmentId)
val thumbnailId = credential.mediaBackupAccess.backupKey.deriveMediaId(MediaName.forThumbnailFromMediaName(mediaName = mediaName)).encode()
SignalDatabase.attachments.setArchiveData(attachmentId = attachmentId, archiveCdn = it.cdn!!, archiveMediaName = mediaName, archiveMediaId = it.mediaId, thumbnailId)
SignalDatabase.attachments.setArchiveCdn(attachmentId = attachmentId, archiveCdn = it.cdn!!)
}
result
}
@@ -1172,12 +1171,14 @@ object BackupRepository {
}
fun deleteArchivedMedia(attachments: List<DatabaseAttachment>): NetworkResult<Unit> {
val mediaRootBackupKey = SignalStore.backup.mediaRootBackupKey
val mediaToDelete = attachments
.filter { it.archiveMediaId != null }
.filter { it.archiveTransferState == AttachmentTable.ArchiveTransferState.FINISHED }
.map {
DeleteArchivedMediaRequest.ArchivedMediaObject(
cdn = it.archiveCdn,
mediaId = it.archiveMediaId!!
mediaId = it.requireMediaName().toMediaId(mediaRootBackupKey).encode()
)
}
@@ -1538,14 +1539,6 @@ object BackupRepository {
val profileKey: ProfileKey
)
fun DatabaseAttachment.getMediaName(): MediaName {
return MediaName.fromDigest(remoteDigest!!)
}
fun DatabaseAttachment.getThumbnailMediaName(): MediaName {
return MediaName.fromDigestForThumbnail(remoteDigest!!)
}
private fun Attachment.toArchiveMediaRequest(mediaName: MediaName, mediaRootBackupKey: MediaRootBackupKey): ArchiveMediaRequest {
val mediaSecrets = mediaRootBackupKey.deriveMediaSecrets(mediaName)
@@ -1646,7 +1639,7 @@ sealed class ImportResult {
* // Cursor is closed after use block.
* ```
*/
class ArchivedMediaObjectIterator(private val cursor: Cursor) : Iterator<ArchivedMediaObject> {
class ArchiveMediaItemIterator(private val cursor: Cursor) : Iterator<ArchiveMediaItem> {
init {
cursor.moveToFirst()
@@ -1654,10 +1647,14 @@ class ArchivedMediaObjectIterator(private val cursor: Cursor) : Iterator<Archive
override fun hasNext(): Boolean = !cursor.isAfterLast
override fun next(): ArchivedMediaObject {
val mediaId = cursor.requireNonNullString(AttachmentTable.ARCHIVE_MEDIA_ID)
override fun next(): ArchiveMediaItem {
val digest = cursor.requireNonNullBlob(AttachmentTable.REMOTE_DIGEST)
val cdn = cursor.requireInt(AttachmentTable.ARCHIVE_CDN)
val mediaId = MediaName.fromDigest(digest).toMediaId(SignalStore.backup.mediaRootBackupKey).encode()
val thumbnailMediaId = MediaName.fromDigestForThumbnail(digest).toMediaId(SignalStore.backup.mediaRootBackupKey).encode()
cursor.moveToNext()
return ArchivedMediaObject(mediaId, cdn)
return ArchiveMediaItem(mediaId, thumbnailMediaId, cdn, digest)
}
}

49
app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/DatabaseAttachmentArchiveExtensions.kt → app/src/main/java/org/thoughtcrime/securesms/backup/v2/DatabaseAttachmentArchiveUtil.kt
View File

@@ -1,16 +1,14 @@
/*
* Copyright 2024 Signal Messenger, LLC
* Copyright 2025 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.thoughtcrime.securesms.backup.v2.database
package org.thoughtcrime.securesms.backup.v2
import android.text.TextUtils
import org.signal.core.util.Base64
import org.thoughtcrime.securesms.attachments.DatabaseAttachment
import org.thoughtcrime.securesms.attachments.InvalidAttachmentException
import org.thoughtcrime.securesms.backup.v2.BackupRepository
import org.thoughtcrime.securesms.backup.v2.BackupRepository.getThumbnailMediaName
import org.thoughtcrime.securesms.keyvalue.SignalStore
import org.thoughtcrime.securesms.util.Util
import org.whispersystems.signalservice.api.backup.MediaName
@@ -19,6 +17,43 @@ import org.whispersystems.signalservice.api.messages.SignalServiceAttachmentRemo
import java.io.IOException
import java.util.Optional
object DatabaseAttachmentArchiveUtil {
@JvmStatic
fun requireMediaName(attachment: DatabaseAttachment): MediaName {
return MediaName.fromDigest(attachment.remoteDigest!!)
}
/**
* For java, since it struggles with value classes.
*/
@JvmStatic
fun requireMediaNameAsString(attachment: DatabaseAttachment): String {
return MediaName.fromDigest(attachment.remoteDigest!!).name
}
@JvmStatic
fun getMediaName(attachment: DatabaseAttachment): MediaName? {
return attachment.remoteDigest?.let { MediaName.fromDigest(it) }
}
@JvmStatic
fun requireThumbnailMediaName(attachment: DatabaseAttachment): MediaName {
return MediaName.fromDigestForThumbnail(attachment.remoteDigest!!)
}
}
fun DatabaseAttachment.requireMediaName(): MediaName {
return DatabaseAttachmentArchiveUtil.requireMediaName(this)
}
fun DatabaseAttachment.getMediaName(): MediaName? {
return DatabaseAttachmentArchiveUtil.getMediaName(this)
}
fun DatabaseAttachment.requireThumbnailMediaName(): MediaName {
return DatabaseAttachmentArchiveUtil.requireThumbnailMediaName(this)
}
/**
* Creates a [SignalServiceAttachmentPointer] for the archived attachment of the given [DatabaseAttachment].
*/
@@ -39,7 +74,7 @@ fun DatabaseAttachment.createArchiveAttachmentPointer(useArchiveCdn: Boolean): S
val id = SignalServiceAttachmentRemoteId.Backup(
mediaCdnPath = mediaCdnPath,
mediaId = mediaRootBackupKey.deriveMediaId(MediaName(archiveMediaName!!)).encode()
mediaId = this.requireMediaName().toMediaId(mediaRootBackupKey).encode()
)
id to archiveCdn
@@ -93,8 +128,8 @@ fun DatabaseAttachment.createArchiveThumbnailPointer(): SignalServiceAttachmentP
val mediaRootBackupKey = SignalStore.backup.mediaRootBackupKey
val mediaCdnPath = BackupRepository.getArchivedMediaCdnPath().successOrThrow()
return try {
val key = mediaRootBackupKey.deriveThumbnailTransitKey(getThumbnailMediaName())
val mediaId = mediaRootBackupKey.deriveMediaId(getThumbnailMediaName()).encode()
val key = mediaRootBackupKey.deriveThumbnailTransitKey(requireThumbnailMediaName())
val mediaId = mediaRootBackupKey.deriveMediaId(requireThumbnailMediaName()).encode()
SignalServiceAttachmentPointer(
cdnNumber = archiveCdn,
remoteId = SignalServiceAttachmentRemoteId.Backup(

2
app/src/main/java/org/thoughtcrime/securesms/backup/v2/exporters/ChatItemArchiveExporter.kt
View File

@@ -859,7 +859,7 @@ private fun LinkPreview.toRemoteLinkPreview(mediaArchiveEnabled: Boolean): org.t
}
private fun BackupMessageRecord.toRemoteViewOnceMessage(mediaArchiveEnabled: Boolean, reactionRecords: List<ReactionRecord>?, attachments: List<DatabaseAttachment>?): ViewOnceMessage {
val attachment: DatabaseAttachment? = attachments?.firstOrNull()?.takeUnless { !it.hasData && it.size == 0L && it.archiveMediaId == null && it.width == 0 && it.height == 0 && it.blurHash == null }
val attachment: DatabaseAttachment? = attachments?.firstOrNull()?.takeUnless { !it.hasData && it.size == 0L && it.remoteDigest == null && it.width == 0 && it.height == 0 && it.blurHash == null }
return ViewOnceMessage(
attachment = attachment?.toRemoteMessageAttachment(mediaArchiveEnabled),

8
app/src/main/java/org/thoughtcrime/securesms/backup/v2/util/ArchiveConverterExtensions.kt
View File

@@ -8,6 +8,7 @@ package org.thoughtcrime.securesms.backup.v2.util
import okio.ByteString
import okio.ByteString.Companion.toByteString
import org.signal.core.util.Base64
import org.signal.core.util.emptyIfNull
import org.signal.core.util.nullIfBlank
import org.signal.core.util.orNull
import org.thoughtcrime.securesms.attachments.ArchivedAttachment
@@ -16,8 +17,8 @@ import org.thoughtcrime.securesms.attachments.Cdn
import org.thoughtcrime.securesms.attachments.DatabaseAttachment
import org.thoughtcrime.securesms.attachments.PointerAttachment
import org.thoughtcrime.securesms.attachments.TombstoneAttachment
import org.thoughtcrime.securesms.backup.v2.BackupRepository.getMediaName
import org.thoughtcrime.securesms.backup.v2.ImportState
import org.thoughtcrime.securesms.backup.v2.getMediaName
import org.thoughtcrime.securesms.backup.v2.proto.FilePointer
import org.thoughtcrime.securesms.conversation.colors.AvatarColor
import org.thoughtcrime.securesms.database.AttachmentTable
@@ -150,10 +151,11 @@ fun DatabaseAttachment.toRemoteFilePointer(mediaArchiveEnabled: Boolean, content
if (mediaArchiveEnabled && !pending) {
val transitCdnKey = this.remoteLocation?.nullIfBlank()
val transitCdnNumber = this.cdn.cdnNumber.takeIf { transitCdnKey != null }
val archiveMediaName = this.getMediaName()?.toString()
builder.backupLocator = FilePointer.BackupLocator(
mediaName = this.archiveMediaName ?: this.getMediaName().toString(),
cdnNumber = this.archiveCdn.takeIf { this.archiveMediaName != null },
mediaName = archiveMediaName.emptyIfNull(),
cdnNumber = this.archiveCdn.takeIf { archiveMediaName != null },
key = Base64.decode(remoteKey).toByteString(),
size = this.size.toInt(),
digest = this.remoteDigest.toByteString(),

20
app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/backup/InternalBackupPlaygroundViewModel.kt
View File

@@ -41,6 +41,7 @@ import org.thoughtcrime.securesms.backup.v2.local.LocalArchiver
import org.thoughtcrime.securesms.backup.v2.local.LocalArchiver.FailureCause
import org.thoughtcrime.securesms.backup.v2.local.SnapshotFileSystem
import org.thoughtcrime.securesms.backup.v2.stream.EncryptedBackupReader.Companion.MAC_SIZE
import org.thoughtcrime.securesms.database.AttachmentTable
import org.thoughtcrime.securesms.database.MessageType
import org.thoughtcrime.securesms.database.SignalDatabase
import org.thoughtcrime.securesms.dependencies.AppDependencies
@@ -53,13 +54,11 @@ import org.thoughtcrime.securesms.jobs.CopyAttachmentToArchiveJob
import org.thoughtcrime.securesms.jobs.RestoreAttachmentJob
import org.thoughtcrime.securesms.jobs.RestoreAttachmentThumbnailJob
import org.thoughtcrime.securesms.jobs.RestoreLocalAttachmentJob
import org.thoughtcrime.securesms.jobs.SyncArchivedMediaJob
import org.thoughtcrime.securesms.keyvalue.SignalStore
import org.thoughtcrime.securesms.mms.IncomingMessage
import org.thoughtcrime.securesms.providers.BlobProvider
import org.thoughtcrime.securesms.recipients.Recipient
import org.whispersystems.signalservice.api.NetworkResult
import org.whispersystems.signalservice.api.backup.MediaName
import org.whispersystems.signalservice.api.backup.MessageBackupKey
import org.whispersystems.signalservice.api.push.ServiceId.ACI
import java.io.FileOutputStream
@@ -351,7 +350,6 @@ class InternalBackupPlaygroundViewModel : ViewModel() {
AppDependencies
.jobManager
.startChain(BackupRestoreJob())
.then(SyncArchivedMediaJob())
.then(BackupRestoreMediaJob())
.enqueueAndBlockUntilCompletion(120.seconds.inWholeMilliseconds)
}
@@ -585,24 +583,16 @@ class InternalBackupPlaygroundViewModel : ViewModel() {
attachments: List<BackupAttachment> = this.attachments,
inProgress: Set<AttachmentId> = this.inProgressMediaIds
): MediaState {
val backupKey = SignalStore.backup.messageBackupKey
val mediaRootBackupKey = SignalStore.backup.mediaRootBackupKey
val updatedAttachments = attachments.map {
val state = if (inProgress.contains(it.dbAttachment.attachmentId)) {
BackupAttachment.State.IN_PROGRESS
} else if (it.dbAttachment.archiveMediaName != null) {
if (it.dbAttachment.remoteDigest != null) {
val mediaId = mediaRootBackupKey.deriveMediaId(MediaName(it.dbAttachment.archiveMediaName)).encode()
if (it.dbAttachment.archiveMediaId == mediaId) {
BackupAttachment.State.UPLOADED_FINAL
} else {
BackupAttachment.State.UPLOADED_UNDOWNLOADED
}
} else if (it.dbAttachment.archiveTransferState == AttachmentTable.ArchiveTransferState.FINISHED) {
if (it.dbAttachment.transferState == AttachmentTable.TRANSFER_PROGRESS_DONE) {
BackupAttachment.State.UPLOADED_FINAL
} else {
BackupAttachment.State.UPLOADED_UNDOWNLOADED
}
} else if (it.dbAttachment.dataHash == null) {
} else if (it.dbAttachment.remoteLocation != null) {
BackupAttachment.State.ATTACHMENT_CDN
} else {
BackupAttachment.State.LOCAL_ONLY

111
app/src/main/java/org/thoughtcrime/securesms/database/AttachmentTable.kt
View File

@@ -64,6 +64,7 @@ import org.signal.core.util.stream.LimitedInputStream
import org.signal.core.util.stream.NullOutputStream
import org.signal.core.util.toInt
import org.signal.core.util.update
import org.signal.core.util.updateAll
import org.signal.core.util.withinTransaction
import org.thoughtcrime.securesms.attachments.ArchivedAttachment
import org.thoughtcrime.securesms.attachments.Attachment
@@ -97,7 +98,6 @@ import org.thoughtcrime.securesms.util.StorageUtil
import org.thoughtcrime.securesms.util.Util
import org.thoughtcrime.securesms.video.EncryptedMediaDataSource
import org.whispersystems.signalservice.api.attachment.AttachmentUploadResult
import org.whispersystems.signalservice.api.backup.MediaId
import org.whispersystems.signalservice.api.crypto.AttachmentCipherOutputStream
import org.whispersystems.signalservice.api.util.UuidUtil
import org.whispersystems.signalservice.internal.crypto.PaddingInputStream
@@ -164,9 +164,6 @@ class AttachmentTable(
const val DISPLAY_ORDER = "display_order"
const val UPLOAD_TIMESTAMP = "upload_timestamp"
const val ARCHIVE_CDN = "archive_cdn"
const val ARCHIVE_MEDIA_NAME = "archive_media_name"
const val ARCHIVE_MEDIA_ID = "archive_media_id"
const val ARCHIVE_THUMBNAIL_MEDIA_ID = "archive_thumbnail_media_id"
const val ARCHIVE_TRANSFER_FILE = "archive_transfer_file"
const val ARCHIVE_TRANSFER_STATE = "archive_transfer_state"
const val THUMBNAIL_RESTORE_STATE = "thumbnail_restore_state"
@@ -224,8 +221,6 @@ class AttachmentTable(
DATA_HASH_START,
DATA_HASH_END,
ARCHIVE_CDN,
ARCHIVE_MEDIA_NAME,
ARCHIVE_MEDIA_ID,
ARCHIVE_TRANSFER_FILE,
THUMBNAIL_FILE,
THUMBNAIL_RESTORE_STATE,
@@ -270,11 +265,8 @@ class AttachmentTable(
$DATA_HASH_START TEXT DEFAULT NULL,
$DATA_HASH_END TEXT DEFAULT NULL,
$ARCHIVE_CDN INTEGER DEFAULT 0,
$ARCHIVE_MEDIA_NAME TEXT DEFAULT NULL,
$ARCHIVE_MEDIA_ID TEXT DEFAULT NULL,
$ARCHIVE_TRANSFER_FILE TEXT DEFAULT NULL,
$ARCHIVE_TRANSFER_STATE INTEGER DEFAULT ${ArchiveTransferState.NONE.value},
$ARCHIVE_THUMBNAIL_MEDIA_ID TEXT DEFAULT NULL,
$THUMBNAIL_FILE TEXT DEFAULT NULL,
$THUMBNAIL_RANDOM BLOB DEFAULT NULL,
$THUMBNAIL_RESTORE_STATE INTEGER DEFAULT ${ThumbnailRestoreState.NONE.value},
@@ -294,8 +286,8 @@ class AttachmentTable(
"CREATE INDEX IF NOT EXISTS attachment_data_hash_start_index ON $TABLE_NAME ($DATA_HASH_START);",
"CREATE INDEX IF NOT EXISTS attachment_data_hash_end_index ON $TABLE_NAME ($DATA_HASH_END);",
"CREATE INDEX IF NOT EXISTS $DATA_FILE_INDEX ON $TABLE_NAME ($DATA_FILE);",
"CREATE INDEX IF NOT EXISTS attachment_archive_media_id_index ON $TABLE_NAME ($ARCHIVE_MEDIA_ID);",
"CREATE INDEX IF NOT EXISTS attachment_archive_transfer_state ON $TABLE_NAME ($ARCHIVE_TRANSFER_STATE);"
"CREATE INDEX IF NOT EXISTS attachment_archive_transfer_state ON $TABLE_NAME ($ARCHIVE_TRANSFER_STATE);",
"CREATE INDEX IF NOT EXISTS attachment_remote_digest_index ON $TABLE_NAME ($REMOTE_DIGEST);"
)
@JvmStatic
@@ -403,11 +395,15 @@ class AttachmentTable(
}
}
fun getMediaIdCursor(): Cursor {
/**
* Returns a cursor (with just the digest+archive_cdn) for all attachments that are eligible for archive upload.
* In practice, this means that the attachments have a digest and have not hit a permanent archive upload failure.
*/
fun getAttachmentsEligibleForArchiveUpload(): Cursor {
return readableDatabase
.select(ARCHIVE_MEDIA_ID, ARCHIVE_CDN)
.select(REMOTE_DIGEST, ARCHIVE_CDN)
.from(TABLE_NAME)
.where("$ARCHIVE_MEDIA_ID IS NOT NULL")
.where("$REMOTE_DIGEST IS NOT NULL AND $ARCHIVE_TRANSFER_STATE != ${ArchiveTransferState.PERMANENT_FAILURE.value}")
.run()
}
@@ -742,9 +738,12 @@ class AttachmentTable(
"""
SELECT SUM($DATA_SIZE)
FROM (
SELECT DISTINCT $ARCHIVE_MEDIA_ID, $DATA_SIZE
SELECT DISTINCT $REMOTE_DIGEST, $DATA_SIZE
FROM $TABLE_NAME
WHERE $ARCHIVE_TRANSFER_STATE NOT IN (${ArchiveTransferState.FINISHED.value}, ${ArchiveTransferState.PERMANENT_FAILURE.value})
WHERE
$DATA_FILE NOT NULL AND
$REMOTE_DIGEST NOT NULL AND
$ARCHIVE_TRANSFER_STATE NOT IN (${ArchiveTransferState.FINISHED.value}, ${ArchiveTransferState.PERMANENT_FAILURE.value})
)
""".trimIndent()
)
@@ -1158,7 +1157,7 @@ class AttachmentTable(
// We don't look at hash_start here because that could result in us matching on a file that got compressed down to something smaller, effectively lowering
// the quality of the attachment we received.
val hashMatch: DataFileInfo? = readableDatabase
.select(ID, DATA_FILE, DATA_SIZE, DATA_RANDOM, DATA_HASH_START, DATA_HASH_END, TRANSFORM_PROPERTIES, UPLOAD_TIMESTAMP, ARCHIVE_CDN, ARCHIVE_MEDIA_NAME, ARCHIVE_MEDIA_ID)
.select(ID, DATA_FILE, DATA_SIZE, DATA_RANDOM, DATA_HASH_START, DATA_HASH_END, TRANSFORM_PROPERTIES, UPLOAD_TIMESTAMP, ARCHIVE_CDN)
.from(TABLE_NAME)
.where("$DATA_HASH_END = ? AND $DATA_HASH_END NOT NULL AND $TRANSFER_STATE = $TRANSFER_PROGRESS_DONE AND $DATA_FILE NOT NULL", fileWriteResult.hash)
.run()
@@ -1175,8 +1174,6 @@ class AttachmentTable(
values.put(DATA_HASH_START, hashMatch.hashEnd)
values.put(DATA_HASH_END, hashMatch.hashEnd)
values.put(ARCHIVE_CDN, hashMatch.archiveCdn)
values.put(ARCHIVE_MEDIA_NAME, hashMatch.archiveMediaName)
values.put(ARCHIVE_MEDIA_ID, hashMatch.archiveMediaId)
} else {
values.put(DATA_FILE, fileWriteResult.file.absolutePath)
values.put(DATA_SIZE, fileWriteResult.length)
@@ -1252,7 +1249,7 @@ class AttachmentTable(
}
@Throws(IOException::class)
fun finalizeAttachmentThumbnailAfterDownload(attachmentId: AttachmentId, archiveMediaId: String, inputStream: InputStream, transferFile: File) {
fun finalizeAttachmentThumbnailAfterDownload(attachmentId: AttachmentId, digest: ByteArray, inputStream: InputStream, transferFile: File) {
Log.i(TAG, "[finalizeAttachmentThumbnailAfterDownload] Finalizing downloaded data for $attachmentId.")
val fileWriteResult: DataFileWriteResult = writeToDataFile(newDataFile(context), inputStream, TransformProperties.empty())
@@ -1265,7 +1262,7 @@ class AttachmentTable(
db.update(TABLE_NAME)
.values(values)
.where("$ARCHIVE_MEDIA_ID = ?", archiveMediaId)
.where("$REMOTE_DIGEST = ?", digest)
.run()
}
@@ -1277,10 +1274,12 @@ class AttachmentTable(
}
}
/**
* Updates the state around archive thumbnail uploads, and ensures that all attachments sharing the same digest remain in sync.
*/
fun finalizeAttachmentThumbnailAfterUpload(
attachmentId: AttachmentId,
archiveMediaId: String,
archiveThumbnailMediaId: MediaId,
attachmentDigest: ByteArray,
data: ByteArray
) {
Log.i(TAG, "[finalizeAttachmentThumbnailAfterUpload] Finalizing archive data for $attachmentId thumbnail.")
@@ -1290,13 +1289,12 @@ class AttachmentTable(
val values = contentValuesOf(
THUMBNAIL_FILE to fileWriteResult.file.absolutePath,
THUMBNAIL_RANDOM to fileWriteResult.random,
THUMBNAIL_RESTORE_STATE to ThumbnailRestoreState.FINISHED.value,
ARCHIVE_THUMBNAIL_MEDIA_ID to archiveThumbnailMediaId.encode()
THUMBNAIL_RESTORE_STATE to ThumbnailRestoreState.FINISHED.value
)
db.update(TABLE_NAME)
.values(values)
.where("$ARCHIVE_MEDIA_ID = ? OR $ID = ?", archiveMediaId, attachmentId)
.where("$ID = ? OR $REMOTE_DIGEST = ?", attachmentId, attachmentDigest)
.run()
}
}
@@ -1601,10 +1599,7 @@ class AttachmentTable(
$DATA_RANDOM,
$DATA_HASH_START,
$DATA_HASH_END,
$ARCHIVE_MEDIA_ID,
$ARCHIVE_MEDIA_NAME,
$ARCHIVE_CDN,
$ARCHIVE_THUMBNAIL_MEDIA_ID,
$THUMBNAIL_RESTORE_STATE
)
SELECT
@@ -1631,10 +1626,7 @@ class AttachmentTable(
$DATA_RANDOM,
$DATA_HASH_START,
$DATA_HASH_END,
"${attachment.archiveMediaId}",
"${attachment.archiveMediaName}",
${attachment.archiveCdn},
$ARCHIVE_THUMBNAIL_MEDIA_ID,
${if (forThumbnail) ThumbnailRestoreState.NEEDS_RESTORE.value else ThumbnailRestoreState.NONE.value}
FROM $TABLE_NAME
WHERE $ID = ${attachment.attachmentId.id}
@@ -1733,7 +1725,7 @@ class AttachmentTable(
fun getDataFileInfo(attachmentId: AttachmentId): DataFileInfo? {
return readableDatabase
.select(ID, DATA_FILE, DATA_SIZE, DATA_RANDOM, DATA_HASH_START, DATA_HASH_END, TRANSFORM_PROPERTIES, UPLOAD_TIMESTAMP, ARCHIVE_CDN, ARCHIVE_MEDIA_NAME, ARCHIVE_MEDIA_ID)
.select(ID, DATA_FILE, DATA_SIZE, DATA_RANDOM, DATA_HASH_START, DATA_HASH_END, TRANSFORM_PROPERTIES, UPLOAD_TIMESTAMP, ARCHIVE_CDN)
.from(TABLE_NAME)
.where("$ID = ?", attachmentId.id)
.run()
@@ -1934,9 +1926,6 @@ class AttachmentTable(
uploadTimestamp = jsonObject.getLong(UPLOAD_TIMESTAMP),
dataHash = jsonObject.getString(DATA_HASH_END),
archiveCdn = jsonObject.getInt(ARCHIVE_CDN),
archiveMediaName = jsonObject.getString(ARCHIVE_MEDIA_NAME),
archiveMediaId = jsonObject.getString(ARCHIVE_MEDIA_ID),
hasArchiveThumbnail = !TextUtils.isEmpty(jsonObject.getString(THUMBNAIL_FILE)),
thumbnailRestoreState = ThumbnailRestoreState.deserialize(jsonObject.getInt(THUMBNAIL_RESTORE_STATE)),
archiveTransferState = ArchiveTransferState.deserialize(jsonObject.getInt(ARCHIVE_TRANSFER_STATE)),
uuid = UuidUtil.parseOrNull(jsonObject.getString(ATTACHMENT_UUID))
@@ -1982,7 +1971,7 @@ class AttachmentTable(
/**
* Sets the archive data for the specific attachment, as well as for any attachments that use the same underlying file.
*/
fun setArchiveData(attachmentId: AttachmentId, archiveCdn: Int, archiveMediaName: String, archiveMediaId: String, archiveThumbnailMediaId: String) {
fun setArchiveCdn(attachmentId: AttachmentId, archiveCdn: Int) {
writableDatabase.withinTransaction { db ->
val dataFile = db
.select(DATA_FILE)
@@ -1999,9 +1988,6 @@ class AttachmentTable(
db.update(TABLE_NAME)
.values(
ARCHIVE_CDN to archiveCdn,
ARCHIVE_MEDIA_ID to archiveMediaId,
ARCHIVE_MEDIA_NAME to archiveMediaName,
ARCHIVE_THUMBNAIL_MEDIA_ID to archiveThumbnailMediaId,
ARCHIVE_TRANSFER_STATE to ArchiveTransferState.FINISHED.value
)
.where("$DATA_FILE = ?", dataFile)
@@ -2009,14 +1995,15 @@ class AttachmentTable(
}
}
fun updateArchiveCdnByMediaId(archiveMediaId: String, archiveCdn: Int): Int {
return writableDatabase.rawQuery(
"UPDATE $TABLE_NAME SET " +
"$ARCHIVE_CDN = CASE WHEN $ARCHIVE_MEDIA_ID = ? THEN ? ELSE $ARCHIVE_CDN END " +
"WHERE $ARCHIVE_MEDIA_ID = ? OR $ARCHIVE_THUMBNAIL_MEDIA_ID = ? " +
"RETURNING $ARCHIVE_CDN",
SqlUtil.buildArgs(archiveMediaId, archiveCdn, archiveMediaId, archiveMediaId)
).count
/**
* Updates all attachments that share the same digest with the given archive CDN.
*/
fun setArchiveCdnByDigest(digest: ByteArray, archiveCdn: Int) {
writableDatabase
.update(TABLE_NAME)
.values(ARCHIVE_CDN to archiveCdn)
.where("$REMOTE_DIGEST = ?", digest)
.run()
}
fun clearArchiveData(attachmentIds: List<AttachmentId>) {
@@ -2025,9 +2012,7 @@ class AttachmentTable(
writableDatabase
.update(TABLE_NAME)
.values(
ARCHIVE_CDN to 0,
ARCHIVE_MEDIA_ID to null,
ARCHIVE_MEDIA_NAME to null
ARCHIVE_CDN to 0
)
.where(query.where, query.whereArgs)
.run()
@@ -2036,13 +2021,11 @@ class AttachmentTable(
fun clearAllArchiveData() {
writableDatabase
.update(TABLE_NAME)
.updateAll(TABLE_NAME)
.values(
ARCHIVE_CDN to 0,
ARCHIVE_MEDIA_ID to null,
ARCHIVE_MEDIA_NAME to null
ARCHIVE_TRANSFER_STATE to ArchiveTransferState.NONE.value
)
.where("$ARCHIVE_CDN > 0 OR $ARCHIVE_MEDIA_ID IS NOT NULL OR $ARCHIVE_MEDIA_NAME IS NOT NULL")
.run()
}
@@ -2332,9 +2315,6 @@ class AttachmentTable(
put(CAPTION, attachment.caption)
put(UPLOAD_TIMESTAMP, attachment.uploadTimestamp)
put(ARCHIVE_CDN, attachment.archiveCdn)
put(ARCHIVE_MEDIA_NAME, attachment.archiveMediaName)
put(ARCHIVE_MEDIA_ID, attachment.archiveMediaId)
put(ARCHIVE_THUMBNAIL_MEDIA_ID, attachment.archiveThumbnailMediaId)
put(ARCHIVE_TRANSFER_STATE, ArchiveTransferState.FINISHED.value)
put(THUMBNAIL_RESTORE_STATE, ThumbnailRestoreState.NEEDS_RESTORE.value)
put(ATTACHMENT_UUID, attachment.uuid?.toString())
@@ -2399,7 +2379,7 @@ class AttachmentTable(
// First we'll check if our file hash matches the starting or ending hash of any other attachments and has compatible transform properties.
// We'll prefer the match with the most recent upload timestamp.
val hashMatch: DataFileInfo? = readableDatabase
.select(ID, DATA_FILE, DATA_SIZE, DATA_RANDOM, DATA_HASH_START, DATA_HASH_END, TRANSFORM_PROPERTIES, UPLOAD_TIMESTAMP, ARCHIVE_CDN, ARCHIVE_MEDIA_NAME, ARCHIVE_MEDIA_ID)
.select(ID, DATA_FILE, DATA_SIZE, DATA_RANDOM, DATA_HASH_START, DATA_HASH_END, TRANSFORM_PROPERTIES, UPLOAD_TIMESTAMP, ARCHIVE_CDN)
.from(TABLE_NAME)
.where("$DATA_FILE NOT NULL AND ($DATA_HASH_START = ? OR $DATA_HASH_END = ?)", fileWriteResult.hash, fileWriteResult.hash)
.run()
@@ -2435,8 +2415,6 @@ class AttachmentTable(
contentValues.put(DATA_HASH_START, fileWriteResult.hash)
contentValues.put(DATA_HASH_END, hashMatch.hashEnd)
contentValues.put(ARCHIVE_CDN, hashMatch.archiveCdn)
contentValues.put(ARCHIVE_MEDIA_NAME, hashMatch.archiveMediaName)
contentValues.put(ARCHIVE_MEDIA_ID, hashMatch.archiveMediaId)
if (hashMatch.transformProperties.skipTransform) {
Log.i(TAG, "[insertAttachmentWithData] The hash match has a DATA_HASH_END and skipTransform=true, so skipping transform of the new file as well. (MessageId: $messageId, ${attachment.uri})")
@@ -2597,9 +2575,6 @@ class AttachmentTable(
uploadTimestamp = cursor.requireLong(UPLOAD_TIMESTAMP),
dataHash = cursor.requireString(DATA_HASH_END),
archiveCdn = cursor.requireInt(ARCHIVE_CDN),
archiveMediaName = cursor.requireString(ARCHIVE_MEDIA_NAME),
archiveMediaId = cursor.requireString(ARCHIVE_MEDIA_ID),
hasArchiveThumbnail = !cursor.isNull(THUMBNAIL_FILE),
thumbnailRestoreState = ThumbnailRestoreState.deserialize(cursor.requireInt(THUMBNAIL_RESTORE_STATE)),
archiveTransferState = ArchiveTransferState.deserialize(cursor.requireInt(ARCHIVE_TRANSFER_STATE)),
uuid = UuidUtil.parseOrNull(cursor.requireString(ATTACHMENT_UUID))
@@ -2627,9 +2602,7 @@ class AttachmentTable(
hashEnd = this.requireString(DATA_HASH_END),
transformProperties = TransformProperties.parse(this.requireString(TRANSFORM_PROPERTIES)),
uploadTimestamp = this.requireLong(UPLOAD_TIMESTAMP),
archiveCdn = this.requireInt(ARCHIVE_CDN),
archiveMediaName = this.requireString(ARCHIVE_MEDIA_NAME),
archiveMediaId = this.requireString(ARCHIVE_MEDIA_ID)
archiveCdn = this.requireInt(ARCHIVE_CDN)
)
}
@@ -2693,9 +2666,7 @@ class AttachmentTable(
val hashEnd: String?,
val transformProperties: TransformProperties,
val uploadTimestamp: Long,
val archiveCdn: Int,
val archiveMediaName: String?,
val archiveMediaId: String?
val archiveCdn: Int
)
@VisibleForTesting

161
app/src/main/java/org/thoughtcrime/securesms/database/BackupMediaSnapshotTable.kt
View File

@@ -10,15 +10,20 @@ import androidx.annotation.VisibleForTesting
import androidx.core.content.contentValuesOf
import org.signal.core.util.SqlUtil
import org.signal.core.util.delete
import org.signal.core.util.exists
import org.signal.core.util.readToList
import org.signal.core.util.readToSet
import org.signal.core.util.requireInt
import org.signal.core.util.requireNonNullBlob
import org.signal.core.util.requireNonNullString
import org.signal.core.util.select
import org.signal.core.util.toInt
import org.thoughtcrime.securesms.backup.v2.ArchivedMediaObject
/**
* Helper table for attachment deletion sync
* When we delete attachments locally, we can't immediately delete them from the archive CDN. This is because there is still a backup that exists that
* references that attachment -- at least until a new backup is made.
*
* So, this table maintains a snapshot of the media present in the last backup, so that we know what we can and can't delete from the archive CDN.
*/
class BackupMediaSnapshotTable(context: Context, database: SignalDatabase) : DatabaseTable(context, database) {
companion object {
@@ -50,72 +55,160 @@ class BackupMediaSnapshotTable(context: Context, database: SignalDatabase) : Dat
@VisibleForTesting
const val PENDING_SYNC_TIME = "pending_sync_time"
/**
* Whether or not this entry is for a thumbnail.
*/
const val IS_THUMBNAIL = "is_thumbnail"
/**
* The remote digest for the media object. This is used to find matching attachments in the attachment table when necessary.
*/
const val REMOTE_DIGEST = "remote_digest"
val CREATE_TABLE = """
CREATE TABLE $TABLE_NAME (
$ID INTEGER PRIMARY KEY,
$MEDIA_ID TEXT UNIQUE,
$CDN INTEGER,
$LAST_SYNC_TIME INTEGER DEFAULT 0,
$PENDING_SYNC_TIME INTEGER
$PENDING_SYNC_TIME INTEGER,
$IS_THUMBNAIL INTEGER DEFAULT 0,
$REMOTE_DIGEST BLOB NOT NULL
)
""".trimIndent()
private const val ON_MEDIA_ID_CONFLICT = """
ON CONFLICT($MEDIA_ID) DO UPDATE SET
$PENDING_SYNC_TIME = EXCLUDED.$PENDING_SYNC_TIME,
$CDN = EXCLUDED.$CDN
"""
}
/**
* Creates the temporary table if it doesn't exist, clears it, then inserts the media objects into it.
* Writes the set of media items that are slated to be referenced in the next backup, updating their pending sync time.
* Will insert multiple rows per object -- one for the main item, and one for the thumbnail.
*/
fun writePendingMediaObjects(mediaObjects: Sequence<ArchivedMediaObject>, pendingSyncTime: Long) {
mediaObjects.chunked(999)
fun writePendingMediaObjects(mediaObjects: Sequence<ArchiveMediaItem>, pendingSyncTime: Long) {
mediaObjects
.chunked(SqlUtil.MAX_QUERY_ARGS)
.forEach { chunk ->
writePendingMediaObjectsChunk(chunk, pendingSyncTime)
}
}
writePendingMediaObjectsChunk(
chunk.map { MediaEntry(it.mediaId, it.cdn, it.digest, isThumbnail = false) },
pendingSyncTime
)
private fun writePendingMediaObjectsChunk(chunk: List<ArchivedMediaObject>, pendingSyncTime: Long) {
SqlUtil.buildBulkInsert(
TABLE_NAME,
arrayOf(MEDIA_ID, CDN, PENDING_SYNC_TIME),
chunk.map {
contentValuesOf(MEDIA_ID to it.mediaId, CDN to it.cdn, PENDING_SYNC_TIME to pendingSyncTime)
writePendingMediaObjectsChunk(
chunk.map { MediaEntry(it.thumbnailMediaId, it.cdn, it.digest, isThumbnail = true) },
pendingSyncTime
)
}
).forEach {
writableDatabase.execSQL("${it.where} $ON_MEDIA_ID_CONFLICT", it.whereArgs)
}
}
/**
* Copies all entries from the temporary table to the persistent table, then deletes the temporary table.
* Commits the pending sync time to the last sync time. This is called once a backup has been successfully uploaded.
*/
fun commitPendingRows() {
writableDatabase.execSQL("UPDATE $TABLE_NAME SET $LAST_SYNC_TIME = $PENDING_SYNC_TIME")
}
fun getPageOfOldMediaObjects(currentSyncTime: Long, pageSize: Int): List<ArchivedMediaObject> {
fun getPageOfOldMediaObjects(currentSyncTime: Long, pageSize: Int): Set<ArchivedMediaObject> {
return readableDatabase.select(MEDIA_ID, CDN)
.from(TABLE_NAME)
.where("$LAST_SYNC_TIME < ? AND $LAST_SYNC_TIME = $PENDING_SYNC_TIME", currentSyncTime)
.limit(pageSize)
.run()
.readToList {
.readToSet {
ArchivedMediaObject(mediaId = it.requireNonNullString(MEDIA_ID), cdn = it.requireInt(CDN))
}
}
fun deleteMediaObjects(mediaObjects: List<ArchivedMediaObject>) {
SqlUtil.buildCollectionQuery(MEDIA_ID, mediaObjects.map { it.mediaId }).forEach {
writableDatabase.delete(TABLE_NAME)
.where(it.where, it.whereArgs)
.run()
fun deleteMediaObjects(mediaObjects: Collection<ArchivedMediaObject>) {
val query = SqlUtil.buildFastCollectionQuery(MEDIA_ID, mediaObjects.map { it.mediaId })
writableDatabase.delete(TABLE_NAME)
.where(query.where, query.whereArgs)
.run()
}
/**
* Given a list of media objects, find the ones that we have no knowledge of in our local store.
*/
fun getMediaObjectsThatCantBeFound(objects: List<ArchivedMediaObject>): Set<ArchivedMediaObject> {
val query = SqlUtil.buildSingleCollectionQuery(
column = MEDIA_ID,
values = objects.map { it.mediaId },
collectionOperator = SqlUtil.CollectionOperator.NOT_IN,
prefix = "$IS_THUMBNAIL = 0 AND "
)
return readableDatabase
.select(MEDIA_ID, CDN)
.from(TABLE_NAME)
.where(query.where, query.whereArgs)
.run()
.readToSet {
ArchivedMediaObject(
mediaId = it.requireNonNullString(MEDIA_ID),
cdn = it.requireInt(CDN)
)
}
}
/**
* Given a list of media objects, find the ones that we have no knowledge of in our local store.
*/
fun getMediaObjectsWithNonMatchingCdn(objects: List<ArchivedMediaObject>): List<CdnMismatchResult> {
val inputValues = objects.joinToString(separator = ", ") { "('${it.mediaId}', ${it.cdn})" }
return readableDatabase.rawQuery(
"""
WITH input_pairs($MEDIA_ID, $CDN) AS (VALUES $inputValues)
SELECT a.$REMOTE_DIGEST, b.$CDN
FROM $TABLE_NAME a
JOIN input_pairs b ON a.$MEDIA_ID = b.$MEDIA_ID
WHERE a.$CDN != b.$CDN AND a.$IS_THUMBNAIL = 0
"""
).readToList { cursor ->
CdnMismatchResult(
digest = cursor.requireNonNullBlob(REMOTE_DIGEST),
cdn = cursor.requireInt(CDN)
)
}
}
fun hasOldMediaObjects(currentSyncTime: Long): Boolean {
return readableDatabase.exists(TABLE_NAME).where("$LAST_SYNC_TIME > ? AND $LAST_SYNC_TIME = $PENDING_SYNC_TIME", currentSyncTime).run()
private fun writePendingMediaObjectsChunk(chunk: List<MediaEntry>, pendingSyncTime: Long) {
val values = chunk.map {
contentValuesOf(
MEDIA_ID to it.mediaId,
CDN to it.cdn,
REMOTE_DIGEST to it.digest,
IS_THUMBNAIL to it.isThumbnail.toInt(),
PENDING_SYNC_TIME to pendingSyncTime
)
}
val query = SqlUtil.buildSingleBulkInsert(TABLE_NAME, arrayOf(MEDIA_ID, CDN, REMOTE_DIGEST, IS_THUMBNAIL, PENDING_SYNC_TIME), values)
writableDatabase.execSQL(
"""
${query.where}
ON CONFLICT($MEDIA_ID) DO UPDATE SET
$PENDING_SYNC_TIME = EXCLUDED.$PENDING_SYNC_TIME,
$CDN = EXCLUDED.$CDN
""",
query.whereArgs
)
}
class ArchiveMediaItem(
val mediaId: String,
val thumbnailMediaId: String,
val cdn: Int,
val digest: ByteArray
)
class CdnMismatchResult(
val digest: ByteArray,
val cdn: Int
)
private data class MediaEntry(
val mediaId: String,
val cdn: Int,
val digest: ByteArray,
val isThumbnail: Boolean
)
}

2
app/src/main/java/org/thoughtcrime/securesms/database/MediaTable.kt
View File

@@ -53,8 +53,6 @@ class MediaTable internal constructor(context: Context?, databaseHelper: SignalD
${AttachmentTable.TABLE_NAME}.${AttachmentTable.REMOTE_INCREMENTAL_DIGEST_CHUNK_SIZE},
${AttachmentTable.TABLE_NAME}.${AttachmentTable.DATA_HASH_END},
${AttachmentTable.TABLE_NAME}.${AttachmentTable.ARCHIVE_CDN},
${AttachmentTable.TABLE_NAME}.${AttachmentTable.ARCHIVE_MEDIA_NAME},
${AttachmentTable.TABLE_NAME}.${AttachmentTable.ARCHIVE_MEDIA_ID},
${AttachmentTable.TABLE_NAME}.${AttachmentTable.THUMBNAIL_RESTORE_STATE},
${AttachmentTable.TABLE_NAME}.${AttachmentTable.ARCHIVE_TRANSFER_STATE},
${AttachmentTable.TABLE_NAME}.${AttachmentTable.ATTACHMENT_UUID},

2
app/src/main/java/org/thoughtcrime/securesms/database/MessageTable.kt
View File

@@ -389,8 +389,6 @@ open class MessageTable(context: Context?, databaseHelper: SignalDatabase) : Dat
'${AttachmentTable.UPLOAD_TIMESTAMP}', ${AttachmentTable.TABLE_NAME}.${AttachmentTable.UPLOAD_TIMESTAMP},
'${AttachmentTable.DATA_HASH_END}', ${AttachmentTable.TABLE_NAME}.${AttachmentTable.DATA_HASH_END},
'${AttachmentTable.ARCHIVE_CDN}', ${AttachmentTable.TABLE_NAME}.${AttachmentTable.ARCHIVE_CDN},
'${AttachmentTable.ARCHIVE_MEDIA_NAME}', ${AttachmentTable.TABLE_NAME}.${AttachmentTable.ARCHIVE_MEDIA_NAME},
'${AttachmentTable.ARCHIVE_MEDIA_ID}', ${AttachmentTable.TABLE_NAME}.${AttachmentTable.ARCHIVE_MEDIA_ID},
'${AttachmentTable.THUMBNAIL_RESTORE_STATE}', ${AttachmentTable.TABLE_NAME}.${AttachmentTable.THUMBNAIL_RESTORE_STATE},
'${AttachmentTable.ARCHIVE_TRANSFER_STATE}', ${AttachmentTable.TABLE_NAME}.${AttachmentTable.ARCHIVE_TRANSFER_STATE},
'${AttachmentTable.ATTACHMENT_UUID}', ${AttachmentTable.TABLE_NAME}.${AttachmentTable.ATTACHMENT_UUID}

6
app/src/main/java/org/thoughtcrime/securesms/database/helpers/SignalDatabaseMigrations.kt
View File

@@ -123,6 +123,7 @@ import org.thoughtcrime.securesms.database.helpers.migration.V265_FixFtsTriggers
import org.thoughtcrime.securesms.database.helpers.migration.V266_UniqueThreadPinOrder
import org.thoughtcrime.securesms.database.helpers.migration.V267_FixGroupInvitationDeclinedUpdate
import org.thoughtcrime.securesms.database.helpers.migration.V268_FixInAppPaymentsErrorStateConsistency
import org.thoughtcrime.securesms.database.helpers.migration.V269_BackupMediaSnapshotChanges
import org.thoughtcrime.securesms.database.SQLiteDatabase as SignalSqliteDatabase
/**
@@ -251,10 +252,11 @@ object SignalDatabaseMigrations {
265 to V265_FixFtsTriggers,
266 to V266_UniqueThreadPinOrder,
267 to V267_FixGroupInvitationDeclinedUpdate,
268 to V268_FixInAppPaymentsErrorStateConsistency
268 to V268_FixInAppPaymentsErrorStateConsistency,
269 to V269_BackupMediaSnapshotChanges
)
const val DATABASE_VERSION = 268
const val DATABASE_VERSION = 269
@JvmStatic
fun migrate(context: Application, db: SignalSqliteDatabase, oldVersion: Int, newVersion: Int) {

26
app/src/main/java/org/thoughtcrime/securesms/database/helpers/migration/V269_BackupMediaSnapshotChanges.kt Normal file
View File

@@ -0,0 +1,26 @@
/*
* Copyright 2025 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.thoughtcrime.securesms.database.helpers.migration
import android.app.Application
import org.thoughtcrime.securesms.database.SQLiteDatabase
/**
* We made a change to stop storing mediaId/names in favor of computing them on-the-fly.
* So, this change removes those columns and adds some plumbing elsewhere that we need to keep things glued together correctly.
*/
object V269_BackupMediaSnapshotChanges : SignalDatabaseMigration {
override fun migrate(context: Application, db: SQLiteDatabase, oldVersion: Int, newVersion: Int) {
db.execSQL("DROP INDEX attachment_archive_media_id_index")
db.execSQL("ALTER TABLE attachment DROP COLUMN archive_media_id")
db.execSQL("ALTER TABLE attachment DROP COLUMN archive_media_name")
db.execSQL("ALTER TABLE attachment DROP COLUMN archive_thumbnail_media_id")
db.execSQL("CREATE INDEX IF NOT EXISTS attachment_remote_digest_index ON attachment (remote_digest);")
db.execSQL("ALTER TABLE backup_media_snapshot ADD COLUMN is_thumbnail INTEGER DEFAULT 0")
db.execSQL("ALTER TABLE backup_media_snapshot ADD COLUMN remote_digest BLOB NOT NULL")
}
}

16
app/src/main/java/org/thoughtcrime/securesms/jobs/ArchiveThumbnailUploadJob.kt
View File

@@ -13,8 +13,7 @@ import org.thoughtcrime.securesms.attachments.AttachmentId
import org.thoughtcrime.securesms.attachments.DatabaseAttachment
import org.thoughtcrime.securesms.attachments.PointerAttachment
import org.thoughtcrime.securesms.backup.v2.BackupRepository
import org.thoughtcrime.securesms.backup.v2.BackupRepository.getMediaName
import org.thoughtcrime.securesms.backup.v2.BackupRepository.getThumbnailMediaName
import org.thoughtcrime.securesms.backup.v2.requireThumbnailMediaName
import org.thoughtcrime.securesms.database.SignalDatabase
import org.thoughtcrime.securesms.dependencies.AppDependencies
import org.thoughtcrime.securesms.jobmanager.Job
@@ -98,7 +97,7 @@ class ArchiveThumbnailUploadJob private constructor(
.getAttachmentUploadForm()
.then { form ->
SignalNetwork.attachments.getResumableUploadSpec(
key = mediaRootBackupKey.deriveThumbnailTransitKey(attachment.getThumbnailMediaName()),
key = mediaRootBackupKey.deriveThumbnailTransitKey(attachment.requireThumbnailMediaName()),
iv = attachment.remoteIv!!,
uploadForm = form
)
@@ -133,15 +132,18 @@ class ArchiveThumbnailUploadJob private constructor(
return Result.retry(defaultBackoff())
}
val mediaSecrets = mediaRootBackupKey.deriveMediaSecrets(attachment.getThumbnailMediaName())
val mediaSecrets = mediaRootBackupKey.deriveMediaSecrets(attachment.requireThumbnailMediaName())
return when (val result = BackupRepository.copyThumbnailToArchive(attachmentPointer, attachment)) {
is NetworkResult.Success -> {
// save attachment thumbnail
val archiveMediaId = attachment.archiveMediaId ?: mediaRootBackupKey.deriveMediaId(attachment.getMediaName()).encode()
SignalDatabase.attachments.finalizeAttachmentThumbnailAfterUpload(attachmentId, archiveMediaId, mediaSecrets.id, thumbnailResult.data)
SignalDatabase.attachments.finalizeAttachmentThumbnailAfterUpload(
attachmentId = attachmentId,
attachmentDigest = attachment.remoteDigest!!,
data = thumbnailResult.data
)
Log.d(TAG, "Successfully archived thumbnail for $attachmentId mediaName=${attachment.getThumbnailMediaName()}")
Log.d(TAG, "Successfully archived thumbnail for $attachmentId mediaName=${attachment.requireThumbnailMediaName()}")
Result.success()
}
is NetworkResult.NetworkError -> {

14
app/src/main/java/org/thoughtcrime/securesms/jobs/AttachmentDownloadJob.kt
View File

@@ -94,12 +94,12 @@ class AttachmentDownloadJob private constructor(
AttachmentTable.TRANSFER_PROGRESS_PENDING,
AttachmentTable.TRANSFER_PROGRESS_FAILED -> {
if (SignalStore.backup.backsUpMedia && databaseAttachment.remoteLocation == null) {
if (databaseAttachment.archiveMediaName.isNullOrEmpty()) {
Log.w(TAG, "No remote location or archive media name, can't download")
null
} else {
if (databaseAttachment.archiveTransferState == AttachmentTable.ArchiveTransferState.FINISHED) {
Log.i(TAG, "Trying to restore attachment from archive cdn")
RestoreAttachmentJob.restoreAttachment(databaseAttachment)
} else {
Log.w(TAG, "No remote location, and the archive transfer state is unfinished. Can't download.")
null
}
} else {
val downloadJob = AttachmentDownloadJob(
@@ -200,8 +200,8 @@ class AttachmentDownloadJob private constructor(
}
if (SignalStore.backup.backsUpMedia && attachment.remoteLocation == null) {
if (attachment.archiveMediaName.isNullOrEmpty()) {
throw InvalidAttachmentException("No remote location or archive media name")
if (attachment.archiveTransferState != AttachmentTable.ArchiveTransferState.FINISHED) {
throw InvalidAttachmentException("No remote location, and the archive transfer state is unfinished. Can't download.")
}
Log.i(TAG, "Trying to restore attachment from archive cdn instead")
@@ -310,7 +310,7 @@ class AttachmentDownloadJob private constructor(
Log.w(TAG, "Experienced exception while trying to download an attachment.", e)
markFailed(messageId, attachmentId)
} catch (e: NonSuccessfulResponseCodeException) {
if (SignalStore.backup.backsUpMedia && e.code == 404 && attachment.archiveMediaName?.isNotEmpty() == true) {
if (SignalStore.backup.backsUpMedia && e.code == 404 && attachment.archiveTransferState === AttachmentTable.ArchiveTransferState.FINISHED) {
Log.i(TAG, "Retrying download from archive CDN")
RestoreAttachmentJob.restoreAttachment(attachment)
return false

199
app/src/main/java/org/thoughtcrime/securesms/jobs/BackupMediaSnapshotSyncJob.kt
View File

@@ -6,19 +6,33 @@
package org.thoughtcrime.securesms.jobs
import org.signal.core.util.logging.Log
import org.signal.core.util.nullIfBlank
import org.thoughtcrime.securesms.backup.v2.ArchivedMediaObject
import org.thoughtcrime.securesms.backup.v2.BackupRepository
import org.thoughtcrime.securesms.database.SignalDatabase
import org.thoughtcrime.securesms.dependencies.AppDependencies
import org.thoughtcrime.securesms.jobmanager.Job
import org.thoughtcrime.securesms.jobmanager.impl.NetworkConstraint
import org.thoughtcrime.securesms.jobs.protos.BackupMediaSnapshotSyncJobData
import org.thoughtcrime.securesms.keyvalue.SignalStore
import org.whispersystems.signalservice.api.NetworkResult
import org.whispersystems.signalservice.api.archive.ArchiveGetMediaItemsResponse
import java.lang.RuntimeException
import kotlin.time.Duration.Companion.days
import kotlin.time.Duration.Companion.hours
/**
* Synchronizes the server media via bulk deletions of old attachments not present
* in the user's current backup.
* When we delete attachments locally, we can't immediately delete them from the archive CDN. This is because there is still a backup that exists that
* references that attachment -- at least until a new backup is made.
*
* This job uses data we store locally in [org.thoughtcrime.securesms.database.BackupMediaSnapshotTable] to determine which media objects can be safely
* deleted from the archive CDN, and then deletes them.
*/
class BackupMediaSnapshotSyncJob private constructor(private val syncTime: Long, parameters: Parameters) : Job(parameters) {
class BackupMediaSnapshotSyncJob private constructor(
private val syncTime: Long,
private var serverCursor: String?,
parameters: Parameters
) : Job(parameters) {
companion object {
@@ -26,51 +40,186 @@ class BackupMediaSnapshotSyncJob private constructor(private val syncTime: Long,
const val KEY = "BackupMediaSnapshotSyncJob"
private const val PAGE_SIZE = 500
private const val REMOTE_DELETE_BATCH_SIZE = 500
private val BACKUP_MEDIA_SYNC_INTERVAL = 7.days.inWholeMilliseconds
fun enqueue(backupSnapshotId: Long) {
fun enqueue(syncTime: Long) {
AppDependencies.jobManager.add(
BackupMediaSnapshotSyncJob(
backupSnapshotId,
Parameters.Builder()
syncTime = syncTime,
serverCursor = null,
parameters = Parameters.Builder()
.addConstraint(NetworkConstraint.KEY)
.setMaxInstancesForFactory(1)
.setQueue("BackupMediaSnapshotSyncJob")
.setMaxAttempts(Parameters.UNLIMITED)
.setLifespan(12.hours.inWholeMilliseconds)
.build()
)
)
}
}
override fun serialize(): ByteArray = BackupMediaSnapshotSyncJobData(syncTime).encode()
override fun serialize(): ByteArray = BackupMediaSnapshotSyncJobData(syncTime, serverCursor ?: "").encode()
override fun getFactoryKey(): String = KEY
override fun run(): Result {
while (SignalDatabase.backupMediaSnapshots.hasOldMediaObjects(syncTime)) {
val mediaObjects = SignalDatabase.backupMediaSnapshots.getPageOfOldMediaObjects(syncTime, PAGE_SIZE)
when (val networkResult = BackupRepository.deleteAbandonedMediaObjects(mediaObjects)) {
is NetworkResult.Success -> {
SignalDatabase.backupMediaSnapshots.deleteMediaObjects(mediaObjects)
}
else -> {
Log.w(TAG, "Failed to delete media objects.", networkResult.getCause())
return Result.failure()
}
}
if (serverCursor == null) {
removeLocallyDeletedAttachmentsFromCdn()?.let { result -> return result }
} else {
Log.d(TAG, "Already deleted old attachments from CDN. Skipping to syncing with remote.")
}
return Result.success()
val timeSinceLastRemoteSync = System.currentTimeMillis() - SignalStore.backup.lastMediaSyncTime
if (serverCursor == null && timeSinceLastRemoteSync > 0 && timeSinceLastRemoteSync < BACKUP_MEDIA_SYNC_INTERVAL) {
Log.d(TAG, "No need to do a remote sync yet. Time since last sync: $timeSinceLastRemoteSync ms")
return Result.success()
}
return syncDataFromCdn() ?: Result.success()
}
override fun onFailure() = Unit
/**
* Looks through our local snapshot of what attachments we put in the last backup file, and uses that to delete any old attachments from the archive CDN
* that we no longer need.
*/
private fun removeLocallyDeletedAttachmentsFromCdn(): Result? {
var mediaObjects = SignalDatabase.backupMediaSnapshots.getPageOfOldMediaObjects(syncTime, REMOTE_DELETE_BATCH_SIZE)
while (mediaObjects.isNotEmpty()) {
deleteMediaObjectsFromCdn(mediaObjects)?.let { result -> return result }
SignalDatabase.backupMediaSnapshots.deleteMediaObjects(mediaObjects)
mediaObjects = SignalDatabase.backupMediaSnapshots.getPageOfOldMediaObjects(syncTime, REMOTE_DELETE_BATCH_SIZE)
}
return null
}
/**
* Fetches all attachment metadata from the archive CDN and ensures that our local store is in sync with it.
*
* Specifically, we make sure that:
* (1) We delete any attachments from the CDN that we have no knowledge of in any backup.
* (2) We ensure that our local store has the correct CDN for any attachments on the CDN (they should only really fall out of sync when you restore a backup
* that was made before all of the attachments had been uploaded).
*/
private fun syncDataFromCdn(): Result? {
val attachmentsToDelete = HashSet<ArchivedMediaObject>()
var cursor: String? = serverCursor
do {
val (archivedItemPage, jobResult) = getRemoteArchiveItemPage(cursor)
if (jobResult != null) {
return jobResult
}
check(archivedItemPage != null)
cursor = archivedItemPage.cursor
attachmentsToDelete += syncCdnPage(archivedItemPage)
if (attachmentsToDelete.size >= REMOTE_DELETE_BATCH_SIZE) {
deleteMediaObjectsFromCdn(attachmentsToDelete)?.let { result -> return result }
attachmentsToDelete.clear()
}
// We don't persist attachmentsToDelete, so we can only update the persisted serverCursor if there's no pending deletes
if (attachmentsToDelete.isEmpty()) {
serverCursor = archivedItemPage.cursor
}
} while (cursor != null)
if (attachmentsToDelete.isNotEmpty()) {
deleteMediaObjectsFromCdn(attachmentsToDelete)?.let { result -> return result }
}
SignalStore.backup.lastMediaSyncTime = System.currentTimeMillis()
return null
}
/**
* Update CDNs of archived media items. Returns set of objects that don't match
* to a local attachment DB row.
*/
private fun syncCdnPage(archivedItemPage: ArchiveGetMediaItemsResponse): Set<ArchivedMediaObject> {
val mediaObjects = archivedItemPage.storedMediaObjects.map {
ArchivedMediaObject(
mediaId = it.mediaId,
cdn = it.cdn
)
}
val notFoundMediaObjects = SignalDatabase.backupMediaSnapshots.getMediaObjectsThatCantBeFound(mediaObjects)
val remainingObjects = mediaObjects - notFoundMediaObjects
val cdnMismatches = SignalDatabase.backupMediaSnapshots.getMediaObjectsWithNonMatchingCdn(remainingObjects)
if (cdnMismatches.isNotEmpty()) {
Log.w(TAG, "Found ${cdnMismatches.size} items with CDNs that differ from what we have locally. Updating our local store.")
for (mismatch in cdnMismatches) {
SignalDatabase.attachments.setArchiveCdnByDigest(mismatch.digest, mismatch.cdn)
}
}
return notFoundMediaObjects
}
private fun getRemoteArchiveItemPage(cursor: String?): Pair<ArchiveGetMediaItemsResponse?, Result?> {
return when (val result = BackupRepository.listRemoteMediaObjects(100, cursor)) {
is NetworkResult.Success -> result.result to null
is NetworkResult.NetworkError -> return null to Result.retry(defaultBackoff())
is NetworkResult.StatusCodeError -> {
if (result.code == 429) {
Log.w(TAG, "Rate limited while attempting to list media objects. Retrying later.")
return null to Result.retry(result.retryAfter()?.inWholeMilliseconds ?: defaultBackoff())
} else {
Log.w(TAG, "Failed to list remote media objects with code: ${result.code}. Unable to proceed.", result.getCause())
return null to Result.failure()
}
}
is NetworkResult.ApplicationError -> {
Log.w(TAG, "Failed to list remote media objects due to a crash.", result.getCause())
return null to Result.fatalFailure(RuntimeException(result.getCause()))
}
}
}
private fun deleteMediaObjectsFromCdn(attachmentsToDelete: Set<ArchivedMediaObject>): Result? {
when (val result = BackupRepository.deleteAbandonedMediaObjects(attachmentsToDelete)) {
is NetworkResult.Success -> {
Log.i(TAG, "Successfully deleted ${attachmentsToDelete.size} attachments off of the CDN.")
}
is NetworkResult.NetworkError -> {
return Result.retry(defaultBackoff())
}
is NetworkResult.StatusCodeError -> {
if (result.code == 429) {
Log.w(TAG, "Rate limited while attempting to delete media objects. Retrying later.")
return Result.retry(result.retryAfter()?.inWholeMilliseconds ?: defaultBackoff())
} else {
Log.w(TAG, "Failed to delete attachments from CDN with code: ${result.code}. Not failing job, just skipping and trying next page.", result.getCause())
}
}
else -> {
Log.w(TAG, "Crash when trying to delete attachments from the CDN", result.getCause())
return Result.fatalFailure(RuntimeException(result.getCause()))
}
}
return null
}
class Factory : Job.Factory<BackupMediaSnapshotSyncJob> {
override fun create(parameters: Parameters, serializedData: ByteArray?): BackupMediaSnapshotSyncJob {
val syncTime: Long = BackupMediaSnapshotSyncJobData.ADAPTER.decode(serializedData!!).syncTime
val data = BackupMediaSnapshotSyncJobData.ADAPTER.decode(serializedData!!)
return BackupMediaSnapshotSyncJob(syncTime, parameters)
return BackupMediaSnapshotSyncJob(
syncTime = data.syncTime,
serverCursor = data.serverCursor.nullIfBlank(),
parameters = parameters
)
}
}
}

12
app/src/main/java/org/thoughtcrime/securesms/jobs/BackupMessagesJob.kt
View File

@@ -10,8 +10,8 @@ import org.signal.core.util.isNotNullOrBlank
import org.signal.core.util.logging.Log
import org.signal.protos.resumableuploads.ResumableUpload
import org.thoughtcrime.securesms.backup.ArchiveUploadProgress
import org.thoughtcrime.securesms.backup.v2.ArchiveMediaItemIterator
import org.thoughtcrime.securesms.backup.v2.ArchiveValidator
import org.thoughtcrime.securesms.backup.v2.ArchivedMediaObjectIterator
import org.thoughtcrime.securesms.backup.v2.BackupRepository
import org.thoughtcrime.securesms.backup.v2.ResumableMessagesBackupUploadSpec
import org.thoughtcrime.securesms.database.SignalDatabase
@@ -52,15 +52,11 @@ class BackupMessagesJob private constructor(
* Pruning abandoned remote media is relatively expensive, so we should
* not do this every time we backup.
*/
fun enqueue(pruneAbandonedRemoteMedia: Boolean = false) {
fun enqueue() {
val jobManager = AppDependencies.jobManager
val chain = jobManager.startChain(BackupMessagesJob())
if (pruneAbandonedRemoteMedia) {
chain.then(SyncArchivedMediaJob())
}
if (SignalStore.backup.optimizeStorage && SignalStore.backup.backsUpMedia) {
chain.then(OptimizeMediaJob())
}
@@ -272,9 +268,9 @@ class BackupMessagesJob private constructor(
private fun writeMediaCursorToTemporaryTable(db: SignalDatabase, mediaBackupEnabled: Boolean, currentTime: Long) {
if (mediaBackupEnabled) {
db.attachmentTable.getMediaIdCursor().use {
db.attachmentTable.getAttachmentsEligibleForArchiveUpload().use {
SignalDatabase.backupMediaSnapshots.writePendingMediaObjects(
mediaObjects = ArchivedMediaObjectIterator(it).asSequence(),
mediaObjects = ArchiveMediaItemIterator(it).asSequence(),
pendingSyncTime = currentTime
)
}

1
app/src/main/java/org/thoughtcrime/securesms/jobs/JobManagerFactories.java
View File

@@ -263,7 +263,6 @@ public final class JobManagerFactories {
put(SubmitRateLimitPushChallengeJob.KEY, new SubmitRateLimitPushChallengeJob.Factory());
put(Svr2MirrorJob.KEY, new Svr2MirrorJob.Factory());
put(Svr3MirrorJob.KEY, new Svr3MirrorJob.Factory());
put(SyncArchivedMediaJob.KEY, new SyncArchivedMediaJob.Factory());
put(ThreadUpdateJob.KEY, new ThreadUpdateJob.Factory());
put(TrimThreadJob.KEY, new TrimThreadJob.Factory());
put(TypingSendJob.KEY, new TypingSendJob.Factory());

10
app/src/main/java/org/thoughtcrime/securesms/jobs/RestoreAttachmentJob.kt
View File

@@ -19,7 +19,8 @@ import org.thoughtcrime.securesms.attachments.AttachmentId
import org.thoughtcrime.securesms.attachments.DatabaseAttachment
import org.thoughtcrime.securesms.attachments.InvalidAttachmentException
import org.thoughtcrime.securesms.backup.v2.BackupRepository
import org.thoughtcrime.securesms.backup.v2.database.createArchiveAttachmentPointer
import org.thoughtcrime.securesms.backup.v2.createArchiveAttachmentPointer
import org.thoughtcrime.securesms.backup.v2.requireMediaName
import org.thoughtcrime.securesms.database.AttachmentTable
import org.thoughtcrime.securesms.database.SignalDatabase
import org.thoughtcrime.securesms.dependencies.AppDependencies
@@ -36,7 +37,6 @@ import org.thoughtcrime.securesms.notifications.NotificationChannels
import org.thoughtcrime.securesms.notifications.NotificationIds
import org.thoughtcrime.securesms.transport.RetryLaterException
import org.thoughtcrime.securesms.util.RemoteConfig
import org.whispersystems.signalservice.api.backup.MediaName
import org.whispersystems.signalservice.api.messages.SignalServiceAttachment
import org.whispersystems.signalservice.api.push.exceptions.MissingConfigurationException
import org.whispersystems.signalservice.api.push.exceptions.NonSuccessfulResponseCodeException
@@ -211,8 +211,8 @@ class RestoreAttachmentJob private constructor(
}
useArchiveCdn = if (SignalStore.backup.backsUpMedia && !forceTransitTier) {
if (attachment.archiveMediaName.isNullOrEmpty()) {
throw InvalidAttachmentException("Invalid attachment configuration")
if (attachment.archiveTransferState != AttachmentTable.ArchiveTransferState.FINISHED) {
throw InvalidAttachmentException("Invalid attachment configuration! backsUpMedia: ${SignalStore.backup.backsUpMedia}, forceTransitTier: $forceTransitTier, archiveTransferState: ${attachment.archiveTransferState}")
}
true
} else {
@@ -238,7 +238,7 @@ class RestoreAttachmentJob private constructor(
messageReceiver
.retrieveArchivedAttachment(
SignalStore.backup.mediaRootBackupKey.deriveMediaSecrets(MediaName(attachment.archiveMediaName!!)),
SignalStore.backup.mediaRootBackupKey.deriveMediaSecrets(attachment.requireMediaName()),
cdnCredentials,
archiveFile,
pointer,

12
app/src/main/java/org/thoughtcrime/securesms/jobs/RestoreAttachmentThumbnailJob.kt
View File

@@ -9,8 +9,8 @@ import org.signal.libsignal.protocol.InvalidMessageException
import org.thoughtcrime.securesms.attachments.AttachmentId
import org.thoughtcrime.securesms.attachments.InvalidAttachmentException
import org.thoughtcrime.securesms.backup.v2.BackupRepository
import org.thoughtcrime.securesms.backup.v2.BackupRepository.getThumbnailMediaName
import org.thoughtcrime.securesms.backup.v2.database.createArchiveThumbnailPointer
import org.thoughtcrime.securesms.backup.v2.createArchiveThumbnailPointer
import org.thoughtcrime.securesms.backup.v2.requireThumbnailMediaName
import org.thoughtcrime.securesms.database.AttachmentTable
import org.thoughtcrime.securesms.database.SignalDatabase
import org.thoughtcrime.securesms.dependencies.AppDependencies
@@ -112,8 +112,8 @@ class RestoreAttachmentThumbnailJob private constructor(
return
}
if (attachment.archiveMediaName == null) {
Log.w(TAG, "$attachmentId was never archived! Cannot proceed.")
if (attachment.remoteDigest == null) {
Log.w(TAG, "$attachmentId has no digest! Cannot proceed.")
return
}
@@ -132,7 +132,7 @@ class RestoreAttachmentThumbnailJob private constructor(
Log.i(TAG, "Downloading thumbnail for $attachmentId")
val downloadResult = AppDependencies.signalServiceMessageReceiver
.retrieveArchivedAttachment(
SignalStore.backup.mediaRootBackupKey.deriveMediaSecrets(attachment.getThumbnailMediaName()),
SignalStore.backup.mediaRootBackupKey.deriveMediaSecrets(attachment.requireThumbnailMediaName()),
cdnCredentials,
thumbnailTransferFile,
pointer,
@@ -142,7 +142,7 @@ class RestoreAttachmentThumbnailJob private constructor(
progressListener
)
SignalDatabase.attachments.finalizeAttachmentThumbnailAfterDownload(attachmentId, attachment.archiveMediaId!!, downloadResult.dataStream, thumbnailTransferFile)
SignalDatabase.attachments.finalizeAttachmentThumbnailAfterDownload(attachmentId, attachment.remoteDigest!!, downloadResult.dataStream, thumbnailTransferFile)
if (!SignalDatabase.messages.isStory(messageId)) {
AppDependencies.messageNotifier.updateNotification(context)

113
app/src/main/java/org/thoughtcrime/securesms/jobs/SyncArchivedMediaJob.kt
View File

@@ -1,113 +0,0 @@
/*
* Copyright 2024 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.thoughtcrime.securesms.jobs
import org.signal.core.util.logging.Log
import org.signal.core.util.withinTransaction
import org.thoughtcrime.securesms.backup.v2.ArchivedMediaObject
import org.thoughtcrime.securesms.backup.v2.BackupRepository
import org.thoughtcrime.securesms.database.SignalDatabase
import org.thoughtcrime.securesms.jobmanager.Job
import org.thoughtcrime.securesms.jobmanager.JsonJobData
import org.thoughtcrime.securesms.keyvalue.SignalStore
import org.whispersystems.signalservice.api.NetworkResult
import org.whispersystems.signalservice.api.archive.ArchiveGetMediaItemsResponse
import org.whispersystems.signalservice.api.push.exceptions.NonSuccessfulResponseCodeException
import java.io.IOException
import java.lang.Exception
/**
* Job responsible for keeping remote archive media objects in sync. That is
* we make sure our CDN number aligns on all media ids, as well as deleting any
* extra media ids that we don't know about.
*/
class SyncArchivedMediaJob private constructor(
parameters: Parameters,
private var jobCursor: String?
) : BaseJob(parameters) {
companion object {
private val TAG = Log.tag(BackupRestoreMediaJob::class.java)
private const val KEY_CURSOR = "cursor"
const val KEY = "SyncArchivedMediaJob"
}
constructor(cursor: String? = null) : this(
Parameters.Builder()
.setQueue("SyncArchivedMedia")
.setMaxAttempts(Parameters.UNLIMITED)
.setMaxInstancesForQueue(2)
.build(),
cursor
)
override fun serialize(): ByteArray? {
return JsonJobData.Builder()
.putString(KEY_CURSOR, jobCursor)
.serialize()
}
override fun getFactoryKey(): String = KEY
override fun onFailure() = Unit
override fun onRun() {
val batchSize = 100
val attachmentsToDelete = HashSet<ArchivedMediaObject>()
var cursor: String? = jobCursor
do {
val archivedItemPage = BackupRepository.listRemoteMediaObjects(batchSize, cursor).successOrThrow()
attachmentsToDelete += syncPage(archivedItemPage)
cursor = archivedItemPage.cursor
if (attachmentsToDelete.size >= batchSize) {
when (val result = BackupRepository.deleteAbandonedMediaObjects(attachmentsToDelete)) {
is NetworkResult.Success -> Log.i(TAG, "Deleted ${attachmentsToDelete.size} attachments off CDN")
else -> Log.w(TAG, "Failed to delete attachments from CDN", result.getCause())
}
attachmentsToDelete.clear()
}
if (attachmentsToDelete.isEmpty()) {
jobCursor = archivedItemPage.cursor
}
} while (cursor != null)
if (attachmentsToDelete.isNotEmpty()) {
BackupRepository.deleteAbandonedMediaObjects(attachmentsToDelete)
Log.i(TAG, "Deleted ${attachmentsToDelete.size} attachments off CDN")
}
SignalStore.backup.lastMediaSyncTime = System.currentTimeMillis()
}
/**
* Update CDNs of archived media items. Returns set of objects that don't match
* to a local attachment DB row.
*/
private fun syncPage(archivedItemPage: ArchiveGetMediaItemsResponse): Set<ArchivedMediaObject> {
val abandonedObjects = HashSet<ArchivedMediaObject>()
SignalDatabase.rawDatabase.withinTransaction {
archivedItemPage.storedMediaObjects.forEach { storedMediaObject ->
val rows = SignalDatabase.attachments.updateArchiveCdnByMediaId(archiveMediaId = storedMediaObject.mediaId, archiveCdn = storedMediaObject.cdn)
if (rows == 0) {
abandonedObjects.add(ArchivedMediaObject(storedMediaObject.mediaId, storedMediaObject.cdn))
}
}
}
return abandonedObjects
}
override fun onShouldRetry(e: Exception): Boolean {
return e is IOException && e !is NonSuccessfulResponseCodeException
}
class Factory : Job.Factory<SyncArchivedMediaJob> {
override fun create(parameters: Parameters, serializedData: ByteArray?): SyncArchivedMediaJob {
val data = JsonJobData.deserialize(serializedData)
return SyncArchivedMediaJob(parameters, if (data.hasString(KEY_CURSOR)) data.getString(KEY_CURSOR) else null)
}
}
}

2
app/src/main/java/org/thoughtcrime/securesms/registrationv3/ui/restore/RemoteRestoreViewModel.kt
View File

@@ -27,7 +27,6 @@ import org.thoughtcrime.securesms.dependencies.AppDependencies
import org.thoughtcrime.securesms.jobmanager.JobTracker
import org.thoughtcrime.securesms.jobs.BackupRestoreJob
import org.thoughtcrime.securesms.jobs.BackupRestoreMediaJob
import org.thoughtcrime.securesms.jobs.SyncArchivedMediaJob
import org.thoughtcrime.securesms.keyvalue.Completed
import org.thoughtcrime.securesms.keyvalue.SignalStore
import org.thoughtcrime.securesms.keyvalue.Skipped
@@ -90,7 +89,6 @@ class RemoteRestoreViewModel(isOnlyRestoreOption: Boolean) : ViewModel() {
AppDependencies
.jobManager
.startChain(BackupRestoreJob())
.then(SyncArchivedMediaJob())
.then(BackupRestoreMediaJob())
.enqueue(listener)

5
app/src/main/java/org/thoughtcrime/securesms/service/MessageBackupListener.kt
View File

@@ -13,7 +13,6 @@ import org.thoughtcrime.securesms.util.RemoteConfig
import org.thoughtcrime.securesms.util.toMillis
import java.time.LocalDateTime
import java.util.Random
import kotlin.time.Duration.Companion.days
import kotlin.time.Duration.Companion.minutes
class MessageBackupListener : PersistentAlarmManagerListener() {
@@ -27,15 +26,13 @@ class MessageBackupListener : PersistentAlarmManagerListener() {
override fun onAlarm(context: Context, scheduledTime: Long): Long {
if (SignalStore.backup.areBackupsEnabled) {
val timeSinceLastSync = System.currentTimeMillis() - SignalStore.backup.lastMediaSyncTime
BackupMessagesJob.enqueue(pruneAbandonedRemoteMedia = timeSinceLastSync >= BACKUP_MEDIA_SYNC_INTERVAL || timeSinceLastSync < 0)
BackupMessagesJob.enqueue()
}
return setNextBackupTimeToIntervalFromNow()
}
companion object {
private val BACKUP_JITTER_WINDOW_SECONDS = 10.minutes.inWholeSeconds.toInt()
private val BACKUP_MEDIA_SYNC_INTERVAL = 7.days.inWholeMilliseconds
@JvmStatic
fun schedule(context: Context?) {

11
app/src/main/java/org/thoughtcrime/securesms/video/exo/PartDataSource.java
View File

@@ -14,11 +14,15 @@ import androidx.media3.datasource.TransferListener;
import org.signal.core.util.logging.Log;
import org.signal.libsignal.protocol.InvalidMessageException;
import org.thoughtcrime.securesms.attachments.DatabaseAttachment;
import org.thoughtcrime.securesms.backup.v2.BackupRepository;
import org.thoughtcrime.securesms.backup.v2.DatabaseAttachmentArchiveUtil;
import org.thoughtcrime.securesms.database.AttachmentTable;
import org.thoughtcrime.securesms.database.SignalDatabase;
import org.thoughtcrime.securesms.keyvalue.SignalStore;
import org.thoughtcrime.securesms.mms.PartUriParser;
import org.signal.core.util.Base64;
import org.whispersystems.signalservice.api.backup.MediaId;
import org.whispersystems.signalservice.api.backup.MediaName;
import org.whispersystems.signalservice.api.backup.MediaRootBackupKey;
import org.whispersystems.signalservice.api.crypto.AttachmentCipherInputStream;
import org.whispersystems.signalservice.api.crypto.AttachmentCipherStreamUtil;
@@ -68,10 +72,13 @@ class PartDataSource implements DataSource {
if (inProgress && !hasData && hasIncrementalDigest && attachmentKey != null) {
final byte[] decode = Base64.decode(attachmentKey);
if (attachment.transferState == AttachmentTable.TRANSFER_RESTORE_IN_PROGRESS && attachment.archiveMediaId != null) {
if (attachment.transferState == AttachmentTable.TRANSFER_RESTORE_IN_PROGRESS && attachment.archiveTransferState == AttachmentTable.ArchiveTransferState.FINISHED) {
final File archiveFile = attachmentDatabase.getOrCreateArchiveTransferFile(attachment.attachmentId);
try {
MediaRootBackupKey.MediaKeyMaterial mediaKeyMaterial = SignalStore.backup().getMediaRootBackupKey().deriveMediaSecretsFromMediaId(attachment.archiveMediaId);
String mediaName = DatabaseAttachmentArchiveUtil.requireMediaNameAsString(attachment);
String mediaId = MediaName.toMediaIdString(mediaName, SignalStore.backup().getMediaRootBackupKey());
MediaRootBackupKey.MediaKeyMaterial mediaKeyMaterial = SignalStore.backup().getMediaRootBackupKey().deriveMediaSecretsFromMediaId(mediaId);
long originalCipherLength = AttachmentCipherStreamUtil.getCiphertextLength(PaddingInputStream.getPaddedSize(attachment.size));
this.inputStream = AttachmentCipherInputStream.createStreamingForArchivedAttachment(mediaKeyMaterial, archiveFile, originalCipherLength, attachment.size, attachment.remoteDigest, decode, attachment.getIncrementalDigest(), attachment.incrementalMacChunkSize);

1
app/src/main/protowire/JobData.proto
View File

@@ -142,6 +142,7 @@ message UploadAttachmentToArchiveJobData {
message BackupMediaSnapshotSyncJobData {
uint64 syncTime = 1;
string serverCursor = 2;
}
message DeviceNameChangeJobData {

16
app/src/test/java/org/thoughtcrime/securesms/backup/v2/ArchivedMediaObjectIteratorTest.kt
View File

@@ -4,8 +4,13 @@ import assertk.assertThat
import assertk.assertions.hasSize
import io.mockk.every
import io.mockk.mockk
import io.mockk.mockkObject
import org.junit.Before
import org.junit.Test
import org.thoughtcrime.securesms.MockCursor
import org.thoughtcrime.securesms.keyvalue.BackupValues
import org.thoughtcrime.securesms.keyvalue.SignalStore
import org.whispersystems.signalservice.api.backup.MediaRootBackupKey
class ArchivedMediaObjectIteratorTest {
private val cursor = mockk<MockCursor>(relaxed = true) {
@@ -17,6 +22,15 @@ class ArchivedMediaObjectIteratorTest {
every { isAfterLast } answers { callOriginal() }
}
@Before
fun setup() {
val mockBackupValues = mockk<BackupValues>()
every { mockBackupValues.mediaRootBackupKey } returns MediaRootBackupKey(ByteArray(32))
mockkObject(SignalStore)
every { SignalStore.backup } returns mockBackupValues
}
@Test
fun `Given a cursor with 0 items, when I convert to a list, then I expect a size of 0`() {
runTest(0)
@@ -29,7 +43,7 @@ class ArchivedMediaObjectIteratorTest {
private fun runTest(size: Int) {
every { cursor.count } returns size
val iterator = ArchivedMediaObjectIterator(cursor)
val iterator = ArchiveMediaItemIterator(cursor)
val list = iterator.asSequence().toList()

3
app/src/test/java/org/thoughtcrime/securesms/sms/UploadDependencyGraphTest.kt
View File

@@ -227,7 +227,6 @@ class UploadDependencyGraphTest {
mmsId = AttachmentTable.PREUPLOAD_MESSAGE_ID,
hasData = false,
hasThumbnail = false,
hasArchiveThumbnail = false,
contentType = attachment.contentType,
transferProgress = AttachmentTable.TRANSFER_PROGRESS_PENDING,
size = attachment.size,
@@ -254,8 +253,6 @@ class UploadDependencyGraphTest {
displayOrder = 0,
uploadTimestamp = attachment.uploadTimestamp,
dataHash = null,
archiveMediaId = null,
archiveMediaName = null,
archiveCdn = 0,
thumbnailRestoreState = AttachmentTable.ThumbnailRestoreState.NONE,
archiveTransferState = AttachmentTable.ArchiveTransferState.NONE,

3
app/src/testShared/org/thoughtcrime/securesms/database/FakeMessageRecords.kt
View File

@@ -72,7 +72,6 @@ object FakeMessageRecords {
mmsId = mmsId,
hasData = hasData,
hasThumbnail = hasThumbnail,
hasArchiveThumbnail = hasArchiveThumbnail,
contentType = contentType,
transferProgress = transferProgress,
size = size,
@@ -100,8 +99,6 @@ object FakeMessageRecords {
uploadTimestamp = uploadTimestamp,
dataHash = dataHash,
archiveCdn = archiveCdn,
archiveMediaName = archiveMediaId,
archiveMediaId = archiveMediaName,
thumbnailRestoreState = thumbnailRestoreState,
archiveTransferState = archiveTransferState,
uuid = null

2
core-util/src/main/java/org/signal/core/util/SqlUtil.kt
View File

@@ -14,7 +14,7 @@ object SqlUtil {
private val TAG = Log.tag(SqlUtil::class.java)
/** The maximum number of arguments (i.e. question marks) allowed in a SQL statement. */
private const val MAX_QUERY_ARGS = 999
const val MAX_QUERY_ARGS = 999
@JvmField
val COUNT = arrayOf("COUNT(*)")

5
libsignal-service/src/main/java/org/whispersystems/signalservice/api/NetworkResult.kt
View File

@@ -21,6 +21,7 @@ import java.util.concurrent.TimeoutException
import kotlin.reflect.KClass
import kotlin.reflect.cast
import kotlin.time.Duration
import kotlin.time.Duration.Companion.seconds
typealias StatusCodeErrorAction = (StatusCodeError<*>) -> Unit
@@ -215,6 +216,10 @@ sealed class NetworkResult<T>(
fun header(key: String): String? {
return headers[key.lowercase()]
}
fun retryAfter(): Duration? {
return header("retry-after")?.toLongOrNull()?.seconds
}
}
/** Indicates that the application somehow failed in a way unrelated to network activity. Usually a runtime crash. */

2
libsignal-service/src/main/java/org/whispersystems/signalservice/api/archive/ArchiveApi.kt
View File

@@ -333,7 +333,7 @@ class ArchiveApi(
* POST /v1/archives/media/delete
*
* - 400: Bad args or made on an authenticated channel
* - 401: Bad presentation, invalid public key signature, no matching backupId on teh server, or the credential was of the wrong type (messages/media)
* - 401: Bad presentation, invalid public key signature, no matching backupId on the server, or the credential was of the wrong type (messages/media)
* - 403: Forbidden
* - 429: Rate-limited
*/

12
libsignal-service/src/main/java/org/whispersystems/signalservice/api/backup/MediaName.kt
View File

@@ -17,6 +17,18 @@ value class MediaName(val name: String) {
fun fromDigest(digest: ByteArray) = MediaName(Hex.toStringCondensed(digest))
fun fromDigestForThumbnail(digest: ByteArray) = MediaName("${Hex.toStringCondensed(digest)}_thumbnail")
fun forThumbnailFromMediaName(mediaName: String) = MediaName("${mediaName}_thumbnail")
/**
* For java, since it struggles with value classes.
*/
@JvmStatic
fun toMediaIdString(mediaName: String, mediaRootBackupKey: MediaRootBackupKey): String {
return MediaName(mediaName).toMediaId(mediaRootBackupKey).encode()
}
}
fun toMediaId(mediaRootBackupKey: MediaRootBackupKey): MediaId {
return mediaRootBackupKey.deriveMediaId(this)
}
fun toByteArray(): ByteArray {