From fd32ec9598d00393373ae39985bd6368570ecec5 Mon Sep 17 00:00:00 2001 From: Greyson Parrelli Date: Mon, 12 Jan 2026 23:56:24 -0500 Subject: [PATCH] Improve internal logging for reconciliation errors. --- .../securesms/database/AttachmentTable.kt | 94 +++++++++++-------- .../ArchiveAttachmentReconciliationJob.kt | 21 +++-- 2 files changed, 71 insertions(+), 44 deletions(-) diff --git a/app/src/main/java/org/thoughtcrime/securesms/database/AttachmentTable.kt b/app/src/main/java/org/thoughtcrime/securesms/database/AttachmentTable.kt index 23ccc6c4f3..d04e799677 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/database/AttachmentTable.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/database/AttachmentTable.kt @@ -100,6 +100,7 @@ import org.thoughtcrime.securesms.database.MessageTable.SyncMessageId import org.thoughtcrime.securesms.database.SignalDatabase.Companion.messages import org.thoughtcrime.securesms.database.SignalDatabase.Companion.threads import org.thoughtcrime.securesms.database.model.MessageId +import org.thoughtcrime.securesms.database.model.MessageRecord import org.thoughtcrime.securesms.database.model.databaseprotos.AudioWaveFormData import org.thoughtcrime.securesms.dependencies.AppDependencies import org.thoughtcrime.securesms.jobs.AttachmentDownloadJob @@ -3425,6 +3426,51 @@ class AttachmentTable( else -> null } } + + fun debugGetAttachmentDataForMediaIds(mediaIds: Collection): List { + if (mediaIds.isEmpty()) return emptyList() + val mediaIdByteStrings = mediaIds.map { it.value.toByteString() }.toSet() + + val found: MutableList = mutableListOf() + + readableDatabase + .select(*PROJECTION) + .from(TABLE_NAME) + .where("$REMOTE_KEY NOT NULL AND $DATA_HASH_END NOT NULL") + .groupBy("$DATA_HASH_END, $REMOTE_KEY") + .run() + .forEach { cursor -> + val remoteKey = Base64.decode(cursor.requireNonNullString(REMOTE_KEY)) + val plaintextHash = Base64.decode(cursor.requireNonNullString(DATA_HASH_END)) + val messageId = cursor.requireLong(MESSAGE_ID) + + val mediaId = MediaName + .fromPlaintextHashAndRemoteKey(plaintextHash, remoteKey) + .toMediaId(SignalStore.backup.mediaRootBackupKey) + .value + .toByteString() + + val mediaIdThumbnail = MediaName + .fromPlaintextHashAndRemoteKeyForThumbnail(plaintextHash, remoteKey) + .toMediaId(SignalStore.backup.mediaRootBackupKey) + .value + .toByteString() + + if (mediaId in mediaIdByteStrings) { + val attachment = getAttachment(cursor) + val messageRecord = messages.getMessageRecordOrNull(messageId) + found.add(ArchiveAttachmentMatch(attachment = attachment, isThumbnail = false, isWallpaper = messageId == WALLPAPER_MESSAGE_ID, messageRecord = messageRecord)) + } + + if (mediaIdThumbnail in mediaIdByteStrings) { + val attachment = getAttachment(cursor) + val messageRecord = messages.getMessageRecordOrNull(messageId) + found.add(ArchiveAttachmentMatch(attachment = attachment, isThumbnail = true, isWallpaper = messageId == WALLPAPER_MESSAGE_ID, messageRecord = messageRecord)) + } + } + + return found + } fun getMediaObjectsThatCantBeFound(objects: Set): Set { if (objects.isEmpty()) { @@ -3452,43 +3498,6 @@ class AttachmentTable( return objectsByMediaId.values.toSet() } - /** - * Important: This is an expensive query that involves iterating over every row in the table. Only call this for debug stuff! - */ - fun debugGetAttachmentsForMediaIds(mediaIds: Set, limit: Int): List> { - val byteStringMediaIds: Set = mediaIds.map { it.value.toByteString() }.toSet() - val found = mutableListOf>() - - run { - readableDatabase - .select(*PROJECTION) - .from(TABLE_NAME) - .where("$REMOTE_KEY NOT NULL AND $DATA_HASH_END NOT NULL") - .groupBy("$DATA_HASH_END, $REMOTE_KEY") - .run() - .forEach { cursor -> - val remoteKey = Base64.decode(cursor.requireNonNullString(REMOTE_KEY)) - val plaintextHash = Base64.decode(cursor.requireNonNullString(DATA_HASH_END)) - val mediaId = MediaName.fromPlaintextHashAndRemoteKey(plaintextHash, remoteKey).toMediaId(SignalStore.backup.mediaRootBackupKey).value.toByteString() - val mediaIdThumbnail = MediaName.fromPlaintextHashAndRemoteKeyForThumbnail(plaintextHash, remoteKey).toMediaId(SignalStore.backup.mediaRootBackupKey).value.toByteString() - val isQuote = cursor.requireBoolean(QUOTE) - val messageId = cursor.requireLong(MESSAGE_ID) - - if (mediaId in byteStringMediaIds) { - found.add(getAttachment(cursor) to false) - } - - if (mediaIdThumbnail in byteStringMediaIds && !isQuote && messageId != WALLPAPER_MESSAGE_ID) { - found.add(getAttachment(cursor) to true) - } - - if (found.size >= limit) return@run - } - } - - return found - } - fun debugGetAttachmentStats(): DebugAttachmentStats { val totalAttachmentRows = readableDatabase.count().from(TABLE_NAME).run().readToSingleLong(0) @@ -4085,4 +4094,15 @@ class AttachmentTable( val contentType: String?, val isThumbnail: Boolean ) + + data class ArchiveAttachmentMatch( + val attachment: DatabaseAttachment, + val isThumbnail: Boolean, + val isWallpaper: Boolean, + val messageRecord: MessageRecord? + ) { + override fun toString(): String { + return "attachmentId=${attachment.attachmentId}, messageId=${attachment.mmsId}, isThumbnail=${isThumbnail}, contentType=${attachment.contentType}, quote=${attachment.quote}, wallpaper=${isWallpaper}, transferState=${attachment.transferState}, archiveTransferState=${attachment.archiveTransferState}, hasData=${attachment.hasData}, dateSent=${messageRecord?.dateSent}, messageType=${messageRecord?.type}, messageFrom=${messageRecord?.fromRecipient?.id}, messageTo=${messageRecord?.toRecipient?.id}, expiresIn=${messageRecord?.expiresIn}, expireStarted=${messageRecord?.expireStarted}" + } + } } diff --git a/app/src/main/java/org/thoughtcrime/securesms/jobs/ArchiveAttachmentReconciliationJob.kt b/app/src/main/java/org/thoughtcrime/securesms/jobs/ArchiveAttachmentReconciliationJob.kt index 8fc20ec62e..67f24fc44b 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/jobs/ArchiveAttachmentReconciliationJob.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/jobs/ArchiveAttachmentReconciliationJob.kt @@ -250,16 +250,15 @@ class ArchiveAttachmentReconciliationJob private constructor( if (internalUser && mediaIdsThatNeedUpload.isNotEmpty()) { Log.w(TAG, "Starting internal-only lookup of matching attachments. May take a while!", true) - val matchingAttachments = SignalDatabase.attachments.debugGetAttachmentsForMediaIds(mediaIdsThatNeedUpload, limit = 10_000) + val matchingAttachments = SignalDatabase.attachments.debugGetAttachmentDataForMediaIds(mediaIdsThatNeedUpload) Log.w(TAG, "Found ${matchingAttachments.size} out of the ${mediaIdsThatNeedUpload.size} attachments we looked up (capped lookups to 10k).", true) - matchingAttachments.forEach { pair -> - val (attachment, isThumbnail) = pair - if (isThumbnail) { - val thumbnailTransferState = SignalDatabase.attachments.getArchiveThumbnailTransferState(attachment.attachmentId) - Log.w(TAG, "[Thumbnail] Needed Upload: attachmentId=${attachment.attachmentId}, messageId=${attachment.mmsId}, contentType=${attachment.contentType}, quote=${attachment.quote}, transferState=${attachment.transferState}, archiveTransferState=${attachment.archiveTransferState}, archiveThumbnailTransferState=$thumbnailTransferState, hasData=${attachment.hasData}", true) + matchingAttachments.forEach { match -> + if (match.isThumbnail) { + val thumbnailTransferState = SignalDatabase.attachments.getArchiveThumbnailTransferState(match.attachment.attachmentId) + Log.w(TAG, "[Thumbnail] Needed Upload: $match, archiveThumbnailTransferState: $thumbnailTransferState", true) } else { - Log.w(TAG, "[Fullsize] Needed Upload: attachmentId=${attachment.attachmentId}, messageId=${attachment.mmsId}, contentType=${attachment.contentType}, quote=${attachment.quote}, transferState=${attachment.transferState}, archiveTransferState=${attachment.archiveTransferState}, hasData=${attachment.hasData}", true) + Log.w(TAG, "[Fullsize] Needed Upload: $match", true) } } stopwatch.split("internal-lookup") @@ -379,6 +378,14 @@ class ArchiveAttachmentReconciliationJob private constructor( } stopwatch.split("fix-state") + if (RemoteConfig.internalUser && foundLocally.isNotEmpty()) { + Log.w(TAG, "Starting internal-only lookup of attachments that we thought we could delete remotely, but still had record of locally.", true) + val matches = SignalDatabase.attachments.debugGetAttachmentDataForMediaIds(foundLocally.map { MediaId(it.mediaId) }) + for (match in matches) { + Log.w(TAG, "[PreventedDelete] $match") + } + } + if (validatedDeletes.isEmpty()) { return null }