Signal/Android
1
0
Fork 0
mirror of https://github.com/signalapp/Signal-Android.git synced 2026-04-21 17:29:32 +01:00
Code Issues Packages Projects Releases Wiki Activity

Validate plaintext hashes for archived attachments.

Browse Source
This commit is contained in:
Greyson Parrelli
2025-06-20 15:26:23 -04:00
committed by Cody Henthorne
parent 38c8f852bf
commit 607b83d65b
21 changed files with 470 additions and 436 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
app/src/main/java/org/thoughtcrime/securesms/backup/v2/DatabaseAttachmentArchiveUtil.kt
View File

@@ -60,6 +60,10 @@ object DatabaseAttachmentArchiveUtil {
}
private fun hadIntegrityCheckPerformed(attachment: DatabaseAttachment): Boolean {
if (attachment.archiveTransferState == AttachmentTable.ArchiveTransferState.FINISHED) {
return true
}
return when (attachment.transferState) {
AttachmentTable.TRANSFER_PROGRESS_DONE,
AttachmentTable.TRANSFER_NEEDS_RESTORE,
@@ -92,8 +96,8 @@ fun DatabaseAttachment.createArchiveAttachmentPointer(useArchiveCdn: Boolean): S
throw InvalidAttachmentException("empty encrypted key")
}
if (remoteDigest == null) {
throw InvalidAttachmentException("no digest")
if (remoteDigest == null && dataHash == null) {
throw InvalidAttachmentException("no integrity check available")
}
return try {

4
app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/backup/InternalBackupStatsTab.kt
View File

@@ -55,10 +55,10 @@ fun InternalBackupStatsTab(stats: InternalBackupPlaygroundViewModel.StatsState,
Spacer(modifier = Modifier.size(16.dp))
Text(text = "Unique/archived data files: ${stats.attachmentStats.attachmentFileCount}/${stats.attachmentStats.finishedAttachmentFileCount}")
Text(text = "Unique/archived verified digest count: ${stats.attachmentStats.attachmentDigestCount}/${stats.attachmentStats.finishedAttachmentDigestCount}")
Text(text = "Unique/archived verified digest count: ${stats.attachmentStats.attachmentPlaintextHashAndKeyCount}/${stats.attachmentStats.finishedAttachmentPlaintextHashAndKeyCount}")
Text(text = "Unique/expected thumbnail files: ${stats.attachmentStats.thumbnailFileCount}/${stats.attachmentStats.estimatedThumbnailCount}")
Text(text = "Local Total: ${stats.attachmentStats.attachmentFileCount + stats.attachmentStats.thumbnailFileCount}")
Text(text = "Expected remote total: ${stats.attachmentStats.estimatedThumbnailCount + stats.attachmentStats.finishedAttachmentDigestCount}")
Text(text = "Expected remote total: ${stats.attachmentStats.estimatedThumbnailCount + stats.attachmentStats.finishedAttachmentPlaintextHashAndKeyCount}")
Spacer(modifier = Modifier.size(16.dp))

221
app/src/main/java/org/thoughtcrime/securesms/database/AttachmentTable.kt
View File

@@ -34,7 +34,6 @@ import org.json.JSONArray
import org.json.JSONException
import org.signal.core.util.Base64
import org.signal.core.util.SqlUtil
import org.signal.core.util.StreamUtil
import org.signal.core.util.ThreadUtil
import org.signal.core.util.copyTo
import org.signal.core.util.count
@@ -60,7 +59,6 @@ import org.signal.core.util.requireNonNullString
import org.signal.core.util.requireObject
import org.signal.core.util.requireString
import org.signal.core.util.select
import org.signal.core.util.stream.NullOutputStream
import org.signal.core.util.toInt
import org.signal.core.util.update
import org.signal.core.util.updateAll
@@ -98,7 +96,6 @@ import org.thoughtcrime.securesms.util.StorageUtil
import org.thoughtcrime.securesms.util.Util
import org.thoughtcrime.securesms.video.EncryptedMediaDataSource
import org.whispersystems.signalservice.api.attachment.AttachmentUploadResult
import org.whispersystems.signalservice.api.crypto.AttachmentCipherOutputStream
import org.whispersystems.signalservice.api.crypto.AttachmentCipherStreamUtil
import org.whispersystems.signalservice.api.util.UuidUtil
import org.whispersystems.signalservice.internal.crypto.PaddingInputStream
@@ -397,14 +394,14 @@ class AttachmentTable(
}
/**
* Returns a cursor (with just the digest+archive_cdn) for all attachments that are eligible for archive upload.
* In practice, this means that the attachments have a digest and have not hit a permanent archive upload failure.
* Returns a cursor (with just the plaintextHash+remoteKey+archive_cdn) for all attachments that are eligible for archive upload.
* In practice, this means that the attachments have a plaintextHash and have not hit a permanent archive upload failure.
*/
fun getAttachmentsEligibleForArchiveUpload(): Cursor {
return readableDatabase
.select(REMOTE_DIGEST, ARCHIVE_CDN)
.select(DATA_HASH_END, REMOTE_KEY, ARCHIVE_CDN)
.from(TABLE_NAME)
.where("$REMOTE_DIGEST IS NOT NULL AND $ARCHIVE_TRANSFER_STATE != ${ArchiveTransferState.PERMANENT_FAILURE.value}")
.where("$DATA_HASH_END NOT NULL AND $REMOTE_KEY NOT NULL AND $ARCHIVE_TRANSFER_STATE != ${ArchiveTransferState.PERMANENT_FAILURE.value}")
.run()
}
@@ -530,7 +527,7 @@ class AttachmentTable(
fun getRestorableOptimizedAttachments(): List<RestorableAttachment> {
return readableDatabase
.select(ID, MESSAGE_ID, DATA_SIZE, REMOTE_DIGEST, REMOTE_KEY)
.select(ID, MESSAGE_ID, DATA_SIZE, DATA_HASH_END, REMOTE_KEY)
.from(TABLE_NAME)
.where("$TRANSFER_STATE = ? AND $DATA_HASH_END NOT NULL AND $REMOTE_KEY NOT NULL", TRANSFER_RESTORE_OFFLOADED)
.orderBy("$ID DESC")
@@ -689,7 +686,7 @@ class AttachmentTable(
ARCHIVE_TRANSFER_STATE to ArchiveTransferState.NONE.value,
ARCHIVE_CDN to null
)
.where("$DATA_HASH_END = ? AND $REMOTE_KEY = ?", plaintextHash, remoteKey)
.where("$DATA_HASH_END = ? AND $REMOTE_KEY = ?", Base64.encodeWithPadding(plaintextHash), Base64.encodeWithPadding(remoteKey))
.run()
}
@@ -768,11 +765,12 @@ class AttachmentTable(
"""
SELECT SUM($DATA_SIZE)
FROM (
SELECT DISTINCT $REMOTE_DIGEST, $DATA_SIZE
SELECT DISTINCT $DATA_HASH_END, $REMOTE_KEY, $DATA_SIZE
FROM $TABLE_NAME
WHERE
$DATA_FILE NOT NULL AND
$REMOTE_DIGEST NOT NULL AND
$DATA_HASH_END NOT NULL AND
$REMOTE_KEY NOT NULL AND
$ARCHIVE_TRANSFER_STATE NOT IN (${ArchiveTransferState.FINISHED.value}, ${ArchiveTransferState.PERMANENT_FAILURE.value})
)
""".trimIndent()
@@ -1257,7 +1255,7 @@ class AttachmentTable(
}
@Throws(IOException::class)
fun finalizeAttachmentThumbnailAfterDownload(attachmentId: AttachmentId, digest: ByteArray, inputStream: InputStream, transferFile: File) {
fun finalizeAttachmentThumbnailAfterDownload(attachmentId: AttachmentId, plaintextHash: String?, remoteKey: String?, inputStream: InputStream, transferFile: File) {
Log.i(TAG, "[finalizeAttachmentThumbnailAfterDownload] Finalizing downloaded data for $attachmentId.")
val fileWriteResult: DataFileWriteResult = writeToDataFile(newDataFile(context), inputStream, TransformProperties.empty())
@@ -1268,10 +1266,14 @@ class AttachmentTable(
THUMBNAIL_RESTORE_STATE to ThumbnailRestoreState.FINISHED.value
)
db.update(TABLE_NAME)
.values(values)
.where("$REMOTE_DIGEST = ?", digest)
.run()
if (plaintextHash != null && remoteKey != null) {
db.update(TABLE_NAME)
.values(values)
.where("$DATA_HASH_END = ? AND $REMOTE_KEY = ?", plaintextHash, remoteKey)
.run()
} else {
Log.w(TAG, "[finalizeAttachmentThumbnailAfterDownload] No plaintext hash or remote key provided for $attachmentId. Cannot update other possible thumbnails.")
}
}
notifyConversationListListeners()
@@ -1287,7 +1289,8 @@ class AttachmentTable(
*/
fun finalizeAttachmentThumbnailAfterUpload(
attachmentId: AttachmentId,
attachmentDigest: ByteArray,
attachmentPlaintextHash: String?,
attachmentRemoteKey: String?,
data: ByteArray
) {
Log.i(TAG, "[finalizeAttachmentThumbnailAfterUpload] Finalizing archive data for $attachmentId thumbnail.")
@@ -1300,10 +1303,14 @@ class AttachmentTable(
THUMBNAIL_RESTORE_STATE to ThumbnailRestoreState.FINISHED.value
)
db.update(TABLE_NAME)
.values(values)
.where("$ID = ? OR $REMOTE_DIGEST = ?", attachmentId, attachmentDigest)
.run()
if (attachmentPlaintextHash != null && attachmentRemoteKey != null) {
db.update(TABLE_NAME)
.values(values)
.where("$DATA_HASH_END = ? AND $REMOTE_KEY = ?", attachmentPlaintextHash, attachmentRemoteKey)
.run()
} else {
Log.w(TAG, "[finalizeAttachmentThumbnailAfterUpload] No plaintext hash or remote key provided for $attachmentId. Cannot update other possible thumbnails.")
}
}
}
@@ -1494,7 +1501,7 @@ class AttachmentTable(
}
/**
* As part of the digest backfill process, this updates the (key, IV, digest) tuple for all attachments that share a data file (and are done downloading).
* As part of the digest backfill process, this updates the (key, digest) tuple for all attachments that share a data file (and are done downloading).
*/
fun updateRemoteKeyAndDigestByDataFile(dataFile: String, key: ByteArray, digest: ByteArray) {
writableDatabase
@@ -1551,74 +1558,6 @@ class AttachmentTable(
return insertedAttachments
}
fun debugCopyAttachmentForArchiveRestore(
mmsId: Long,
attachment: DatabaseAttachment,
forThumbnail: Boolean
) {
val copy =
"""
INSERT INTO $TABLE_NAME
(
$MESSAGE_ID,
$CONTENT_TYPE,
$TRANSFER_STATE,
$CDN_NUMBER,
$REMOTE_LOCATION,
$REMOTE_DIGEST,
$REMOTE_INCREMENTAL_DIGEST,
$REMOTE_INCREMENTAL_DIGEST_CHUNK_SIZE,
$REMOTE_KEY,
$FILE_NAME,
$DATA_SIZE,
$VOICE_NOTE,
$BORDERLESS,
$VIDEO_GIF,
$WIDTH,
$HEIGHT,
$CAPTION,
$UPLOAD_TIMESTAMP,
$BLUR_HASH,
$DATA_SIZE,
$DATA_RANDOM,
$DATA_HASH_START,
$DATA_HASH_END,
$ARCHIVE_CDN,
$THUMBNAIL_RESTORE_STATE
)
SELECT
$mmsId,
$CONTENT_TYPE,
$TRANSFER_NEEDS_RESTORE,
$CDN_NUMBER,
$REMOTE_LOCATION,
$REMOTE_DIGEST,
$REMOTE_INCREMENTAL_DIGEST,
$REMOTE_INCREMENTAL_DIGEST_CHUNK_SIZE,
$REMOTE_KEY,
$FILE_NAME,
$DATA_SIZE,
$VOICE_NOTE,
$BORDERLESS,
$VIDEO_GIF,
$WIDTH,
$HEIGHT,
$CAPTION,
${System.currentTimeMillis()},
$BLUR_HASH,
$DATA_SIZE,
$DATA_RANDOM,
$DATA_HASH_START,
$DATA_HASH_END,
${attachment.archiveCdn},
${if (forThumbnail) ThumbnailRestoreState.NEEDS_RESTORE.value else ThumbnailRestoreState.NONE.value}
FROM $TABLE_NAME
WHERE $ID = ${attachment.attachmentId.id}
"""
writableDatabase.execSQL(copy)
}
/**
* Updates the data stored for an existing attachment. This happens after transformations, like transcoding.
*/
@@ -1921,56 +1860,47 @@ class AttachmentTable(
}
/**
* Sets the archive data for the specific attachment, as well as for any attachments that use the same underlying file.
* Sets the archive data for the specific attachment, as well as for any attachments that have the same mediaName (plaintextHash + remoteKey).
*/
fun setArchiveCdn(attachmentId: AttachmentId, archiveCdn: Int) {
writableDatabase.withinTransaction { db ->
val dataFile = db
.select(DATA_FILE)
val plaintextHashAndRemoteKey = db
.select(DATA_HASH_END, REMOTE_KEY)
.from(TABLE_NAME)
.where("$ID = ?", attachmentId.id)
.run()
.readToSingleObject { it.requireString(DATA_FILE) }
.readToSingleObject {
it.requireNonNullString(DATA_HASH_END) to it.requireNonNullString(REMOTE_KEY)
}
if (dataFile == null) {
if (plaintextHashAndRemoteKey == null) {
Log.w(TAG, "No data file found for attachment $attachmentId. Can't set archive data.")
return@withinTransaction
}
val (plaintextHash, remoteKey) = plaintextHashAndRemoteKey
db.update(TABLE_NAME)
.values(
ARCHIVE_CDN to archiveCdn,
ARCHIVE_TRANSFER_STATE to ArchiveTransferState.FINISHED.value
)
.where("$DATA_FILE = ?", dataFile)
.where("$DATA_HASH_END = ? AND $REMOTE_KEY = ?", plaintextHash, remoteKey)
.run()
}
}
/**
* Updates all attachments that share the same digest with the given archive CDN.
* Updates all attachments that share the same mediaName (plaintextHash + remoteKey) with the given archive CDN.
*/
fun setArchiveCdnByPlaintextHashAndRemoteKey(plaintextHash: ByteArray, remoteKey: ByteArray, archiveCdn: Int) {
writableDatabase
.update(TABLE_NAME)
.values(ARCHIVE_CDN to archiveCdn)
.where("$DATA_HASH_END= ? AND $REMOTE_KEY = ?", plaintextHash, remoteKey)
.where("$DATA_HASH_END = ? AND $REMOTE_KEY = ?", Base64.encodeWithPadding(plaintextHash), Base64.encodeWithPadding(remoteKey))
.run()
}
fun clearArchiveData(attachmentIds: List<AttachmentId>) {
SqlUtil.buildCollectionQuery(ID, attachmentIds.map { it.id })
.forEach { query ->
writableDatabase
.update(TABLE_NAME)
.values(
ARCHIVE_CDN to null
)
.where(query.where, query.whereArgs)
.run()
}
}
fun clearAllArchiveData() {
writableDatabase
.updateAll(TABLE_NAME)
@@ -1981,18 +1911,6 @@ class AttachmentTable(
.run()
}
private fun calculateDigest(fileInfo: DataFileWriteResult, key: ByteArray, iv: ByteArray = Util.getSecretBytes(16)): ByteArray {
return calculateDigest(file = fileInfo.file, random = fileInfo.random, length = fileInfo.length, key = key, iv = iv)
}
private fun calculateDigest(file: File, random: ByteArray, length: Long, key: ByteArray, iv: ByteArray): ByteArray {
val stream = PaddingInputStream(getDataStream(file, random, 0), length)
val cipherOutputStream = AttachmentCipherOutputStream(key, iv, NullOutputStream)
StreamUtil.copy(stream, cipherOutputStream)
return cipherOutputStream.transmittedDigest
}
/**
* Deletes the data file if there's no strong references to other attachments.
* If deleted, it will also clear all weak references (i.e. quotes) of the attachment.
@@ -2476,16 +2394,21 @@ class AttachmentTable(
fun getEstimatedArchiveMediaSize(): Long {
val estimatedThumbnailCount = readableDatabase
.select("COUNT(DISTINCT $REMOTE_DIGEST)")
.from(TABLE_NAME)
.where(
.select("COUNT(*)")
.from(
"""
(
SELECT DISTINCT $DATA_HASH_END, $REMOTE_KEY
FROM $TABLE_NAME
WHERE
$DATA_FILE NOT NULL AND
$DATA_HASH_END NOT NULL AND
$REMOTE_KEY NOT NULL AND
$TRANSFER_STATE = $TRANSFER_PROGRESS_DONE AND
$ARCHIVE_TRANSFER_STATE != ${ArchiveTransferState.PERMANENT_FAILURE.value} AND
($CONTENT_TYPE LIKE 'image/%' OR $CONTENT_TYPE LIKE 'video/%')
)
"""
$DATA_FILE NOT NULL AND
$REMOTE_DIGEST NOT NULL AND
$TRANSFER_STATE = $TRANSFER_PROGRESS_DONE AND
$ARCHIVE_TRANSFER_STATE != ${ArchiveTransferState.PERMANENT_FAILURE.value} AND
($CONTENT_TYPE LIKE 'image/%' OR $CONTENT_TYPE LIKE 'video/%')
"""
)
.run()
.readToSingleLong(0L)
@@ -2495,11 +2418,12 @@ class AttachmentTable(
"""
SELECT $DATA_SIZE
FROM (
SELECT DISTINCT $REMOTE_DIGEST, $DATA_SIZE
SELECT DISTINCT $DATA_HASH_END, $REMOTE_KEY, $DATA_SIZE
FROM $TABLE_NAME
WHERE
$DATA_FILE NOT NULL AND
$REMOTE_DIGEST NOT NULL AND
$DATA_HASH_END NOT NULL AND
$REMOTE_KEY NOT NULL AND
$TRANSFER_STATE = $TRANSFER_PROGRESS_DONE AND
$ARCHIVE_TRANSFER_STATE != ${ArchiveTransferState.PERMANENT_FAILURE.value}
)
@@ -2657,23 +2581,23 @@ class AttachmentTable(
)
val transferStateCounts = transferStates
.map { (state, name) -> name to readableDatabase.count().from(TABLE_NAME).where("$TRANSFER_STATE = $state AND $REMOTE_DIGEST NOT NULL").run().readToSingleLong(-1L) }
.map { (state, name) -> name to readableDatabase.count().from(TABLE_NAME).where("$TRANSFER_STATE = $state AND $DATA_HASH_END NOT NULL AND $REMOTE_KEY NOT NULL").run().readToSingleLong(-1L) }
.toMap()
val validForArchiveTransferStateCounts = transferStates
.map { (state, name) -> name to readableDatabase.count().from(TABLE_NAME).where("$TRANSFER_STATE = $state AND $REMOTE_DIGEST NOT NULL AND $DATA_FILE NOT NULL").run().readToSingleLong(-1L) }
.map { (state, name) -> name to readableDatabase.count().from(TABLE_NAME).where("$TRANSFER_STATE = $state AND $DATA_HASH_END NOT NULL AND $REMOTE_KEY NOT NULL AND $DATA_FILE NOT NULL").run().readToSingleLong(-1L) }
.toMap()
val archiveStateCounts = ArchiveTransferState
.entries
.associate { it to readableDatabase.count().from(TABLE_NAME).where("$ARCHIVE_TRANSFER_STATE = ${it.value} AND $REMOTE_DIGEST NOT NULL").run().readToSingleLong(-1L) }
.associate { it to readableDatabase.count().from(TABLE_NAME).where("$ARCHIVE_TRANSFER_STATE = ${it.value} AND $DATA_HASH_END NOT NULL AND $REMOTE_KEY NOT NULL").run().readToSingleLong(-1L) }
val attachmentFileCount = readableDatabase.query("SELECT COUNT(DISTINCT $DATA_FILE) FROM $TABLE_NAME WHERE $DATA_FILE NOT NULL AND $REMOTE_DIGEST NOT NULL").readToSingleLong(-1L)
val finishedAttachmentFileCount = readableDatabase.query("SELECT COUNT(DISTINCT $DATA_FILE) FROM $TABLE_NAME WHERE $DATA_FILE NOT NULL AND $REMOTE_DIGEST NOT NULL AND $ARCHIVE_TRANSFER_STATE = ${ArchiveTransferState.FINISHED.value}").readToSingleLong(-1L)
val attachmentDigestCount = readableDatabase.query("SELECT COUNT(DISTINCT $REMOTE_DIGEST) FROM $TABLE_NAME WHERE $REMOTE_DIGEST NOT NULL AND $TRANSFER_STATE in ($TRANSFER_PROGRESS_DONE, $TRANSFER_RESTORE_OFFLOADED, $TRANSFER_RESTORE_IN_PROGRESS, $TRANSFER_NEEDS_RESTORE)").readToSingleLong(-1L)
val finishedAttachmentDigestCount = readableDatabase.query("SELECT COUNT(DISTINCT $REMOTE_DIGEST) FROM $TABLE_NAME WHERE $REMOTE_DIGEST NOT NULL AND $ARCHIVE_TRANSFER_STATE = ${ArchiveTransferState.FINISHED.value}").readToSingleLong(-1L)
val attachmentFileCount = readableDatabase.query("SELECT COUNT(DISTINCT $DATA_FILE) FROM $TABLE_NAME WHERE $DATA_FILE NOT NULL AND $DATA_HASH_END NOT NULL AND $REMOTE_KEY NOT NULL").readToSingleLong(-1L)
val finishedAttachmentFileCount = readableDatabase.query("SELECT COUNT(DISTINCT $DATA_FILE) FROM $TABLE_NAME WHERE $DATA_FILE NOT NULL AND $DATA_HASH_END NOT NULL $REMOTE_KEY NOT NULL AND $ARCHIVE_TRANSFER_STATE = ${ArchiveTransferState.FINISHED.value}").readToSingleLong(-1L)
val attachmentPlaintextHashAndKeyCount = readableDatabase.query("SELECT COUNT(*) FROM (SELECT DISTINCT $DATA_HASH_END, $REMOTE_KEY FROM $TABLE_NAME WHERE $DATA_HASH_END NOT NULL AND $REMOTE_KEY NOT NULL AND $TRANSFER_STATE in ($TRANSFER_PROGRESS_DONE, $TRANSFER_RESTORE_OFFLOADED, $TRANSFER_RESTORE_IN_PROGRESS, $TRANSFER_NEEDS_RESTORE))").readToSingleLong(-1L)
val finishedAttachmentDigestCount = readableDatabase.query("SELECT COUNT(*) FROM (SELECT DISTINCT $DATA_HASH_END, $REMOTE_KEY) FROM $TABLE_NAME WHERE $DATA_HASH_END NOT NULL AND $REMOTE_KEY NOT NULL AND $ARCHIVE_TRANSFER_STATE = ${ArchiveTransferState.FINISHED.value})").readToSingleLong(-1L)
val thumbnailFileCount = readableDatabase.query("SELECT COUNT(DISTINCT $THUMBNAIL_FILE) FROM $TABLE_NAME WHERE $THUMBNAIL_FILE IS NOT NULL").readToSingleLong(-1L)
val estimatedThumbnailCount = readableDatabase.query("SELECT COUNT(DISTINCT $REMOTE_DIGEST) FROM $TABLE_NAME WHERE $ARCHIVE_TRANSFER_STATE = ${ArchiveTransferState.FINISHED.value} AND $REMOTE_DIGEST NOT NULL AND ($CONTENT_TYPE LIKE 'image/%' OR $CONTENT_TYPE LIKE 'video/%')").readToSingleLong(-1L)
val estimatedThumbnailCount = readableDatabase.query("SELECT COUNT(*) FROM (SELECT DISTINCT $DATA_HASH_END, $REMOTE_KEY) FROM $TABLE_NAME WHERE $ARCHIVE_TRANSFER_STATE = ${ArchiveTransferState.FINISHED.value} AND $DATA_HASH_END NOT NULL AND $REMOTE_KEY NOT NULL AND ($CONTENT_TYPE LIKE 'image/%' OR $CONTENT_TYPE LIKE 'video/%'))").readToSingleLong(-1L)
val pendingUploadBytes = getPendingArchiveUploadBytes()
val uploadedAttachmentBytes = readableDatabase
@@ -2681,11 +2605,12 @@ class AttachmentTable(
"""
SELECT $DATA_SIZE
FROM (
SELECT DISTINCT $REMOTE_DIGEST, $DATA_SIZE
SELECT DISTINCT $DATA_HASH_END, $REMOTE_KEY, $DATA_SIZE
FROM $TABLE_NAME
WHERE
$DATA_FILE NOT NULL AND
$REMOTE_DIGEST NOT NULL AND
$DATA_HASH_END NOT NULL AND
$REMOTE_KEY NOT NULL AND
$ARCHIVE_TRANSFER_STATE = ${ArchiveTransferState.FINISHED.value}
)
""".trimIndent()
@@ -2702,8 +2627,8 @@ class AttachmentTable(
archiveStateCounts = archiveStateCounts,
attachmentFileCount = attachmentFileCount,
finishedAttachmentFileCount = finishedAttachmentFileCount,
attachmentDigestCount = attachmentDigestCount,
finishedAttachmentDigestCount = finishedAttachmentDigestCount,
attachmentPlaintextHashAndKeyCount = attachmentPlaintextHashAndKeyCount,
finishedAttachmentPlaintextHashAndKeyCount = finishedAttachmentDigestCount,
thumbnailFileCount = thumbnailFileCount,
estimatedThumbnailCount = estimatedThumbnailCount,
pendingUploadBytes = pendingUploadBytes,
@@ -2953,8 +2878,8 @@ class AttachmentTable(
val archiveStateCounts: Map<ArchiveTransferState, Long> = emptyMap(),
val attachmentFileCount: Long = 0L,
val finishedAttachmentFileCount: Long = 0L,
val attachmentDigestCount: Long = 0L,
val finishedAttachmentDigestCount: Long,
val attachmentPlaintextHashAndKeyCount: Long = 0L,
val finishedAttachmentPlaintextHashAndKeyCount: Long,
val thumbnailFileCount: Long = 0L,
val pendingUploadBytes: Long = 0L,
val uploadedAttachmentBytes: Long = 0L,

2
app/src/main/java/org/thoughtcrime/securesms/database/BackupMediaSnapshotTable.kt
View File

@@ -244,7 +244,7 @@ class BackupMediaSnapshotTable(context: Context, database: SignalDatabase) : Dat
return readableDatabase.rawQuery(
"""
WITH input_pairs($MEDIA_ID, $CDN) AS (VALUES $inputValues)
SELECT a.$PLAINTEXT_HASH, a.$REMOTE_KEY b.$CDN
SELECT a.$PLAINTEXT_HASH, a.$REMOTE_KEY, b.$CDN
FROM $TABLE_NAME a
JOIN input_pairs b ON a.$MEDIA_ID = b.$MEDIA_ID
WHERE a.$CDN != b.$CDN AND a.$IS_THUMBNAIL = 0 AND $SNAPSHOT_VERSION = $MAX_VERSION

3
app/src/main/java/org/thoughtcrime/securesms/jobs/ArchiveThumbnailUploadJob.kt
View File

@@ -148,7 +148,8 @@ class ArchiveThumbnailUploadJob private constructor(
// save attachment thumbnail
SignalDatabase.attachments.finalizeAttachmentThumbnailAfterUpload(
attachmentId = attachmentId,
attachmentDigest = attachment.remoteDigest,
attachmentPlaintextHash = attachment.dataHash,
attachmentRemoteKey = attachment.remoteKey,
data = thumbnailResult.data
)

2
app/src/main/java/org/thoughtcrime/securesms/jobs/AttachmentDownloadJob.kt
View File

@@ -94,7 +94,7 @@ class AttachmentDownloadJob private constructor(
AttachmentTable.TRANSFER_PROGRESS_PENDING,
AttachmentTable.TRANSFER_PROGRESS_FAILED -> {
if (SignalStore.backup.backsUpMedia && databaseAttachment.remoteLocation == null) {
if (SignalStore.backup.backsUpMedia && (databaseAttachment.remoteLocation == null || databaseAttachment.remoteDigest == null)) {
if (databaseAttachment.archiveTransferState == AttachmentTable.ArchiveTransferState.FINISHED) {
Log.i(TAG, "Trying to restore attachment from archive cdn")
RestoreAttachmentJob.restoreAttachment(databaseAttachment)

4
app/src/main/java/org/thoughtcrime/securesms/jobs/BackupMessagesJob.kt
View File

@@ -131,8 +131,8 @@ class BackupMessagesJob private constructor(
val stopwatch = Stopwatch("BackupMessagesJob")
SignalDatabase.attachments.createRemoteKeyForAttachmentsThatNeedArchiveUpload().takeIf { it > 0 }?.let { count -> Log.w(TAG, "Needed to create $count key/iv/digests.") }
stopwatch.split("key-iv-digest")
SignalDatabase.attachments.createRemoteKeyForAttachmentsThatNeedArchiveUpload().takeIf { it > 0 }?.let { count -> Log.w(TAG, "Needed to create $count remote keys.") }
stopwatch.split("keygen")
if (isCanceled) {
return Result.failure()

7
app/src/main/java/org/thoughtcrime/securesms/jobs/RestoreAttachmentJob.kt
View File

@@ -10,6 +10,7 @@ import android.content.Intent
import androidx.core.app.NotificationCompat
import androidx.core.app.NotificationManagerCompat
import org.greenrobot.eventbus.EventBus
import org.signal.core.util.Base64.decodeBase64OrThrow
import org.signal.core.util.PendingIntentFlags
import org.signal.core.util.logging.Log
import org.signal.libsignal.protocol.InvalidMacException
@@ -182,9 +183,10 @@ class RestoreAttachmentJob private constructor(
if (attachment.transferState != AttachmentTable.TRANSFER_NEEDS_RESTORE &&
attachment.transferState != AttachmentTable.TRANSFER_RESTORE_IN_PROGRESS &&
(attachment.transferState != AttachmentTable.TRANSFER_RESTORE_OFFLOADED)
attachment.transferState != AttachmentTable.TRANSFER_PROGRESS_FAILED &&
attachment.transferState != AttachmentTable.TRANSFER_RESTORE_OFFLOADED
) {
Log.w(TAG, "Attachment does not need to be restored.")
Log.w(TAG, "Attachment does not need to be restored. Current state: ${attachment.transferState}")
return
}
@@ -263,6 +265,7 @@ class RestoreAttachmentJob private constructor(
messageReceiver
.retrieveArchivedAttachment(
SignalStore.backup.mediaRootBackupKey.deriveMediaSecrets(attachment.requireMediaName()),
attachment.dataHash!!.decodeBase64OrThrow(),
cdnCredentials,
archiveFile,
pointer,

6
app/src/main/java/org/thoughtcrime/securesms/jobs/RestoreAttachmentThumbnailJob.kt
View File

@@ -113,8 +113,8 @@ class RestoreAttachmentThumbnailJob private constructor(
return
}
if (attachment.remoteDigest == null) {
Log.w(TAG, "$attachmentId has no digest! Cannot proceed.")
if (attachment.dataHash == null) {
Log.w(TAG, "$attachmentId has no plaintext hash! Cannot proceed.")
return
}
@@ -142,7 +142,7 @@ class RestoreAttachmentThumbnailJob private constructor(
progressListener
)
SignalDatabase.attachments.finalizeAttachmentThumbnailAfterDownload(attachmentId, attachment.remoteDigest, decryptingStream, thumbnailTransferFile)
SignalDatabase.attachments.finalizeAttachmentThumbnailAfterDownload(attachmentId, attachment.dataHash, attachment.remoteKey, decryptingStream, thumbnailTransferFile)
if (!SignalDatabase.messages.isStory(messageId)) {
AppDependencies.messageNotifier.updateNotification(context)

77
app/src/main/java/org/thoughtcrime/securesms/mms/AttachmentStreamLocalUriFetcher.java
View File

@@ -1,77 +0,0 @@
package org.thoughtcrime.securesms.mms;
import androidx.annotation.NonNull;
import com.bumptech.glide.Priority;
import com.bumptech.glide.load.DataSource;
import com.bumptech.glide.load.data.DataFetcher;
import org.signal.core.util.logging.Log;
import org.signal.libsignal.protocol.InvalidMessageException;
import org.whispersystems.signalservice.api.crypto.AttachmentCipherInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Optional;
class AttachmentStreamLocalUriFetcher implements DataFetcher<InputStream> {
private static final String TAG = Log.tag(AttachmentStreamLocalUriFetcher.class);
private final File attachment;
private final byte[] key;
private final Optional<byte[]> digest;
private final long plaintextLength;
private InputStream is;
AttachmentStreamLocalUriFetcher(File attachment, long plaintextLength, byte[] key, Optional<byte[]> digest) {
this.attachment = attachment;
this.plaintextLength = plaintextLength;
this.digest = digest;
this.key = key;
}
@Override
public void loadData(@NonNull Priority priority, @NonNull DataCallback<? super InputStream> callback) {
try {
if (!digest.isPresent()) throw new InvalidMessageException("No attachment digest!");
is = AttachmentCipherInputStream.createForAttachment(attachment,
plaintextLength,
key,
digest.get(),
null,
0);
callback.onDataReady(is);
} catch (IOException | InvalidMessageException e) {
callback.onLoadFailed(e);
}
}
@Override
public void cleanup() {
try {
if (is != null) is.close();
is = null;
} catch (IOException ioe) {
Log.w(TAG, "ioe");
}
}
@Override
public void cancel() {}
@Override
public @NonNull Class<InputStream> getDataClass() {
return InputStream.class;
}
@Override
public @NonNull DataSource getDataSource() {
return DataSource.LOCAL;
}
}

89
app/src/main/java/org/thoughtcrime/securesms/mms/AttachmentStreamUriLoader.java
View File

@@ -1,89 +0,0 @@
package org.thoughtcrime.securesms.mms;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import com.bumptech.glide.load.Key;
import com.bumptech.glide.load.Options;
import com.bumptech.glide.load.model.ModelLoader;
import com.bumptech.glide.load.model.ModelLoaderFactory;
import com.bumptech.glide.load.model.MultiModelLoaderFactory;
import org.thoughtcrime.securesms.mms.AttachmentStreamUriLoader.AttachmentModel;
import java.io.File;
import java.io.InputStream;
import java.security.MessageDigest;
import java.util.Optional;
public class AttachmentStreamUriLoader implements ModelLoader<AttachmentModel, InputStream> {
@Override
public @Nullable LoadData<InputStream> buildLoadData(@NonNull AttachmentModel attachmentModel, int width, int height, @NonNull Options options) {
return new LoadData<>(attachmentModel, new AttachmentStreamLocalUriFetcher(attachmentModel.attachment, attachmentModel.plaintextLength, attachmentModel.key, attachmentModel.digest));
}
@Override
public boolean handles(@NonNull AttachmentModel attachmentModel) {
return true;
}
static class Factory implements ModelLoaderFactory<AttachmentModel, InputStream> {
@Override
public @NonNull ModelLoader<AttachmentModel, InputStream> build(@NonNull MultiModelLoaderFactory multiFactory) {
return new AttachmentStreamUriLoader();
}
@Override
public void teardown() {
// Do nothing.
}
}
public static class AttachmentModel implements Key {
public @NonNull File attachment;
public @NonNull byte[] key;
public @NonNull Optional<byte[]> digest;
public @NonNull Optional<byte[]> incrementalDigest;
public int incrementalMacChunkSize;
public long plaintextLength;
public AttachmentModel(@NonNull File attachment,
@NonNull byte[] key,
long plaintextLength,
@NonNull Optional<byte[]> digest,
@NonNull Optional<byte[]> incrementalDigest,
int incrementalMacChunkSize)
{
this.attachment = attachment;
this.key = key;
this.digest = digest;
this.incrementalDigest = incrementalDigest;
this.incrementalMacChunkSize = incrementalMacChunkSize;
this.plaintextLength = plaintextLength;
}
@Override
public void updateDiskCacheKey(@NonNull MessageDigest messageDigest) {
messageDigest.update(attachment.toString().getBytes());
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
AttachmentModel that = (AttachmentModel)o;
return attachment.equals(that.attachment);
}
@Override
public int hashCode() {
return attachment.hashCode();
}
}
}

2
app/src/main/java/org/thoughtcrime/securesms/mms/SignalGlideComponents.java
View File

@@ -41,7 +41,6 @@ import org.thoughtcrime.securesms.glide.cache.EncryptedCacheDecoder;
import org.thoughtcrime.securesms.glide.cache.EncryptedCacheEncoder;
import org.thoughtcrime.securesms.glide.cache.EncryptedGifDrawableResourceEncoder;
import org.thoughtcrime.securesms.glide.cache.WebpSanDecoder;
import org.thoughtcrime.securesms.mms.AttachmentStreamUriLoader.AttachmentModel;
import org.thoughtcrime.securesms.mms.DecryptableStreamUriLoader.DecryptableUri;
import org.thoughtcrime.securesms.stickers.StickerRemoteUri;
import org.thoughtcrime.securesms.stickers.StickerRemoteUriLoader;
@@ -96,7 +95,6 @@ public class SignalGlideComponents implements RegisterGlideComponents {
registry.append(ConversationShortcutPhoto.class, Bitmap.class, new ConversationShortcutPhoto.Loader.Factory(context));
registry.append(ContactPhoto.class, InputStream.class, new ContactPhotoLoader.Factory(context));
registry.append(DecryptableUri.class, InputStream.class, new DecryptableStreamUriLoader.Factory(context));
registry.append(AttachmentModel.class, InputStream.class, new AttachmentStreamUriLoader.Factory());
registry.append(ChunkedImageUrl.class, InputStream.class, new ChunkedImageUrlLoader.Factory());
registry.append(StickerRemoteUri.class, InputStream.class, new StickerRemoteUriLoader.Factory());
registry.append(BlurHash.class, BlurHash.class, new BlurHashModelLoader.Factory());

6
app/src/main/java/org/thoughtcrime/securesms/video/exo/PartDataSource.java
View File

@@ -86,7 +86,11 @@ class PartDataSource implements DataSource {
throw new InvalidMessageException("Missing digest!");
}
this.inputStream = AttachmentCipherInputStream.createForArchivedMedia(mediaKeyMaterial, archiveFile, originalCipherLength, attachment.size, decodedKey, attachment.remoteDigest, attachment.getIncrementalDigest(), attachment.incrementalMacChunkSize);
if (attachment.dataHash == null || attachment.dataHash.isEmpty()) {
throw new InvalidMessageException("Missing plaintextHash!");
}
this.inputStream = AttachmentCipherInputStream.createForArchivedMedia(mediaKeyMaterial, archiveFile, originalCipherLength, attachment.size, decodedKey, Base64.decodeOrThrow(attachment.dataHash), attachment.getIncrementalDigest(), attachment.incrementalMacChunkSize);
} catch (InvalidMessageException e) {
throw new IOException("Error decrypting attachment stream!", e);
}