Add internal backup stats tab.

This commit is contained in:
Cody Henthorne
2025-06-04 11:54:13 -04:00
parent be4af1d560
commit 26b6019b28
5 changed files with 283 additions and 22 deletions

View File

@@ -1082,7 +1082,7 @@ object BackupRepository {
/**
* Returns an object with details about the remote backup state.
*/
fun getRemoteBackupState(): NetworkResult<BackupMetadata> {
fun debugGetRemoteBackupState(): NetworkResult<DebugBackupMetadata> {
return initBackupAndFetchAuth()
.then { credential ->
SignalNetwork.archive.getBackupInfo(SignalStore.account.requireAci(), credential.mediaBackupAccess)
@@ -1091,11 +1091,11 @@ object BackupRepository {
.then { pair ->
val (mediaBackupInfo, credential) = pair
SignalNetwork.archive.debugGetUploadedMediaItemMetadata(SignalStore.account.requireAci(), credential.mediaBackupAccess)
.also { Log.i(TAG, "MediaItemMetadataResult: $it") }
.map { mediaObjects ->
BackupMetadata(
DebugBackupMetadata(
usedSpace = mediaBackupInfo.usedSpace ?: 0,
mediaCount = mediaObjects.size.toLong()
mediaCount = mediaObjects.size.toLong(),
mediaSize = mediaObjects.sumOf { it.objectLength }
)
}
}
@@ -1156,7 +1156,7 @@ object BackupRepository {
/**
* Returns an object with details about the remote backup state.
*/
private fun debugGetArchivedMediaState(): NetworkResult<List<ArchiveGetMediaItemsResponse.StoredMediaObject>> {
fun debugGetArchivedMediaState(): NetworkResult<List<ArchiveGetMediaItemsResponse.StoredMediaObject>> {
return initBackupAndFetchAuth()
.then { credential ->
SignalNetwork.archive.debugGetUploadedMediaItemMetadata(SignalStore.account.requireAci(), credential.mediaBackupAccess)
@@ -1648,9 +1648,10 @@ class ImportState(val mediaRootBackupKey: MediaRootBackupKey) {
}
}
class BackupMetadata(
class DebugBackupMetadata(
val usedSpace: Long,
val mediaCount: Long
val mediaCount: Long,
val mediaSize: Long
)
sealed class ImportResult {

View File

@@ -35,6 +35,7 @@ import androidx.compose.material3.Text
import androidx.compose.material3.TextButton
import androidx.compose.material3.TopAppBar
import androidx.compose.runtime.Composable
import androidx.compose.runtime.LaunchedEffect
import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableIntStateOf
import androidx.compose.runtime.mutableStateOf
@@ -52,6 +53,7 @@ import androidx.compose.ui.text.input.KeyboardType
import androidx.compose.ui.unit.dp
import androidx.compose.ui.window.DialogProperties
import androidx.fragment.app.viewModels
import androidx.lifecycle.compose.collectAsStateWithLifecycle
import androidx.lifecycle.lifecycleScope
import androidx.navigation.fragment.findNavController
import com.google.android.material.dialog.MaterialAlertDialogBuilder
@@ -133,6 +135,11 @@ class InternalBackupPlaygroundFragment : ComposeFragment() {
override fun FragmentContent() {
val context = LocalContext.current
val state by viewModel.state
val statsState by viewModel.statsState.collectAsStateWithLifecycle()
LaunchedEffect(Unit) {
viewModel.loadStats()
}
Tabs(
onBack = { findNavController().popBackStack() },
@@ -225,6 +232,14 @@ class InternalBackupPlaygroundFragment : ComposeFragment() {
.show()
}
)
},
statsContent = {
InternalBackupStatsTab(
statsState,
object : StatsCallbacks {
override fun loadRemoteState() = viewModel.loadRemoteStats()
}
)
}
)
}
@@ -234,9 +249,10 @@ class InternalBackupPlaygroundFragment : ComposeFragment() {
@Composable
fun Tabs(
onBack: () -> Unit,
mainContent: @Composable () -> Unit
mainContent: @Composable () -> Unit,
statsContent: @Composable () -> Unit
) {
val tabs = listOf("Main")
val tabs = listOf("Main", "Stats")
var tabIndex by remember { mutableIntStateOf(0) }
val snackbarHostState: SnackbarHostState = remember { SnackbarHostState() }
@@ -274,6 +290,7 @@ fun Tabs(
Surface(modifier = Modifier.padding(it)) {
when (tabIndex) {
0 -> mainContent()
1 -> statsContent()
}
}
}

View File

@@ -11,6 +11,7 @@ import androidx.compose.runtime.State
import androidx.compose.runtime.mutableStateOf
import androidx.documentfile.provider.DocumentFile
import androidx.lifecycle.ViewModel
import androidx.lifecycle.viewModelScope
import io.reactivex.rxjava3.android.schedulers.AndroidSchedulers
import io.reactivex.rxjava3.core.Single
import io.reactivex.rxjava3.disposables.CompositeDisposable
@@ -18,6 +19,9 @@ import io.reactivex.rxjava3.kotlin.plusAssign
import io.reactivex.rxjava3.kotlin.subscribeBy
import io.reactivex.rxjava3.schedulers.Schedulers
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.update
import kotlinx.coroutines.launch
import kotlinx.coroutines.withContext
import org.signal.core.util.Hex
import org.signal.core.util.bytes
@@ -32,8 +36,8 @@ import org.thoughtcrime.securesms.attachments.AttachmentId
import org.thoughtcrime.securesms.attachments.DatabaseAttachment
import org.thoughtcrime.securesms.backup.ArchiveUploadProgress
import org.thoughtcrime.securesms.backup.v2.ArchiveValidator
import org.thoughtcrime.securesms.backup.v2.BackupMetadata
import org.thoughtcrime.securesms.backup.v2.BackupRepository
import org.thoughtcrime.securesms.backup.v2.DebugBackupMetadata
import org.thoughtcrime.securesms.backup.v2.MessageBackupTier
import org.thoughtcrime.securesms.backup.v2.local.ArchiveFileSystem
import org.thoughtcrime.securesms.backup.v2.local.ArchiveResult
@@ -42,6 +46,8 @@ import org.thoughtcrime.securesms.backup.v2.local.LocalArchiver.FailureCause
import org.thoughtcrime.securesms.backup.v2.local.SnapshotFileSystem
import org.thoughtcrime.securesms.backup.v2.stream.EncryptedBackupReader.Companion.MAC_SIZE
import org.thoughtcrime.securesms.database.AttachmentTable
import org.thoughtcrime.securesms.database.AttachmentTable.DebugAttachmentStats
import org.thoughtcrime.securesms.database.SignalDatabase
import org.thoughtcrime.securesms.dependencies.AppDependencies
import org.thoughtcrime.securesms.jobs.BackupMessagesJob
import org.thoughtcrime.securesms.jobs.BackupRestoreJob
@@ -84,6 +90,8 @@ class InternalBackupPlaygroundViewModel : ViewModel() {
)
val state: State<ScreenState> = _state
val statsState: MutableStateFlow<StatsState> = MutableStateFlow(StatsState())
enum class DialogState {
None,
ImportCredentials
@@ -256,7 +264,7 @@ class InternalBackupPlaygroundViewModel : ViewModel() {
disposables += Single
.fromCallable {
BackupRepository.restoreBackupTier(SignalStore.account.requireAci())
BackupRepository.getRemoteBackupState()
BackupRepository.debugGetRemoteBackupState()
}
.subscribeOn(Schedulers.io())
.subscribe { result ->
@@ -348,6 +356,29 @@ class InternalBackupPlaygroundViewModel : ViewModel() {
}
}
fun loadStats() {
viewModelScope.launch(Dispatchers.IO) {
launch {
var stats = SignalDatabase.attachments.debugGetAttachmentStats()
statsState.update { it.copy(attachmentStats = stats) }
}
}
}
fun loadRemoteStats() {
viewModelScope.launch(Dispatchers.IO) {
launch {
statsState.update { it.copy(loadingRemoteStats = true) }
val (remoteState: DebugBackupMetadata?, errorMsg: String?) = when (val result = BackupRepository.debugGetRemoteBackupState()) {
is NetworkResult.Success -> result.result to null
else -> null to result.toString()
}
statsState.update { it.copy(remoteState = remoteState, remoteFailureMsg = errorMsg, loadingRemoteStats = false) }
}
}
}
suspend fun deleteRemoteBackupData(): Boolean = withContext(Dispatchers.IO) {
when (val result = BackupRepository.debugDeleteAllArchivedMedia()) {
is NetworkResult.Success -> Log.i(TAG, "Remote data deleted")
@@ -386,7 +417,7 @@ class InternalBackupPlaygroundViewModel : ViewModel() {
sealed class RemoteBackupState {
data object Unknown : RemoteBackupState()
data object NotFound : RemoteBackupState()
data class Available(val response: BackupMetadata) : RemoteBackupState()
data class Available(val response: DebugBackupMetadata) : RemoteBackupState()
}
data class MediaState(
@@ -451,4 +482,13 @@ class InternalBackupPlaygroundViewModel : ViewModel() {
val messageBackupKey: MessageBackupKey,
val aci: ACI
)
data class StatsState(
val attachmentStats: DebugAttachmentStats? = null,
val loadingRemoteStats: Boolean = false,
val remoteState: DebugBackupMetadata? = null,
val remoteFailureMsg: String? = null
) {
val valid = attachmentStats != null
}
}

View File

@@ -0,0 +1,116 @@
/*
* Copyright 2025 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.thoughtcrime.securesms.components.settings.app.internal.backup
import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.Spacer
import androidx.compose.foundation.layout.size
import androidx.compose.foundation.rememberScrollState
import androidx.compose.foundation.verticalScroll
import androidx.compose.material3.Button
import androidx.compose.material3.CircularProgressIndicator
import androidx.compose.material3.Text
import androidx.compose.runtime.Composable
import androidx.compose.ui.Modifier
import androidx.compose.ui.unit.dp
import org.signal.core.ui.compose.Dividers
import org.signal.core.ui.compose.Texts
import org.signal.core.util.bytes
@Composable
fun InternalBackupStatsTab(stats: InternalBackupPlaygroundViewModel.StatsState, callbacks: StatsCallbacks) {
val scrollState = rememberScrollState()
Column(modifier = Modifier.verticalScroll(scrollState)) {
Texts.SectionHeader(text = "Local Attachment State")
if (stats.attachmentStats != null) {
Text(text = "Attachment Count: ${stats.attachmentStats.attachmentCount}")
Text(text = "Transit Download State:")
stats.attachmentStats.transferStateCounts.forEach { (state, count) ->
if (count > 0) {
Text(text = "$state: $count")
}
}
Text(text = "Valid for archive Transit Download State:")
stats.attachmentStats.validForArchiveTransferStateCounts.forEach { (state, count) ->
if (count > 0) {
Text(text = "$state: $count")
}
}
Spacer(modifier = Modifier.size(4.dp))
Text(text = "Archive State:")
stats.attachmentStats.archiveStateCounts.forEach { (state, count) ->
if (count > 0) {
Text(text = "$state: $count")
}
}
Spacer(modifier = Modifier.size(16.dp))
Text(text = "Unique/archived data files: ${stats.attachmentStats.attachmentFileCount}/${stats.attachmentStats.finishedAttachmentFileCount}")
Text(text = "Unique/archived verified digest count: ${stats.attachmentStats.attachmentDigestCount}/${stats.attachmentStats.finishedAttachmentDigestCount}")
Text(text = "Unique/expected thumbnail files: ${stats.attachmentStats.thumbnailFileCount}/${stats.attachmentStats.estimatedThumbnailCount}")
Text(text = "Local Total: ${stats.attachmentStats.attachmentFileCount + stats.attachmentStats.thumbnailFileCount}")
Text(text = "Expected remote total: ${stats.attachmentStats.estimatedThumbnailCount + stats.attachmentStats.finishedAttachmentDigestCount}")
Spacer(modifier = Modifier.size(16.dp))
Text(text = "Pending upload: ${stats.attachmentStats.pendingUploadBytes} (~${stats.attachmentStats.pendingUploadBytes.bytes.toUnitString()})")
Text(text = "Est uploaded attachments: ${stats.attachmentStats.uploadedAttachmentBytes} (~${stats.attachmentStats.uploadedAttachmentBytes.bytes.toUnitString()})")
Text(text = "Est uploaded thumbnails: ${stats.attachmentStats.thumbnailBytes} (~${stats.attachmentStats.thumbnailBytes.bytes.toUnitString()})")
val total = stats.attachmentStats.thumbnailBytes + stats.attachmentStats.uploadedAttachmentBytes
Text(text = "Est total: $total (~${total.bytes.toUnitString()})")
} else {
CircularProgressIndicator()
}
Dividers.Default()
Texts.SectionHeader(text = "Remote State")
if (!stats.loadingRemoteStats && stats.remoteState == null && stats.remoteFailureMsg == null) {
Button(onClick = callbacks::loadRemoteState) {
Text(text = "Load remote stats (expensive and long)")
}
} else {
if (stats.loadingRemoteStats) {
CircularProgressIndicator()
} else if (stats.remoteState != null) {
Text(text = "Media item count: ${stats.remoteState.mediaCount}")
Text(text = "Media items sum size: ${stats.remoteState.mediaSize} (~${stats.remoteState.mediaSize.bytes.toUnitString()})")
Text(text = "Server estimated used size: ${stats.remoteState.usedSpace} (~${stats.remoteState.usedSpace.bytes.toUnitString()})")
} else if (stats.remoteFailureMsg != null) {
Text(text = stats.remoteFailureMsg)
}
Dividers.Default()
Texts.SectionHeader(text = "Expected vs Actual")
if (stats.attachmentStats != null && stats.remoteState != null) {
val finished = stats.attachmentStats.finishedAttachmentFileCount
val thumbnails = stats.attachmentStats.thumbnailFileCount
Text(text = "Expected Count/Actual Remote Count: ${finished + thumbnails} / ${stats.remoteState.mediaCount}")
} else {
CircularProgressIndicator()
}
}
}
}
interface StatsCallbacks {
fun loadRemoteState()
companion object {
val EMPTY = object : StatsCallbacks {
override fun loadRemoteState() = Unit
}
}
}

View File

@@ -79,6 +79,17 @@ import org.thoughtcrime.securesms.crypto.AttachmentSecret
import org.thoughtcrime.securesms.crypto.ClassicDecryptingPartInputStream
import org.thoughtcrime.securesms.crypto.ModernDecryptingPartInputStream
import org.thoughtcrime.securesms.crypto.ModernEncryptingPartOutputStream
import org.thoughtcrime.securesms.database.AttachmentTable.ArchiveTransferState.COPY_PENDING
import org.thoughtcrime.securesms.database.AttachmentTable.ArchiveTransferState.FINISHED
import org.thoughtcrime.securesms.database.AttachmentTable.ArchiveTransferState.NONE
import org.thoughtcrime.securesms.database.AttachmentTable.ArchiveTransferState.PERMANENT_FAILURE
import org.thoughtcrime.securesms.database.AttachmentTable.ArchiveTransferState.UPLOAD_IN_PROGRESS
import org.thoughtcrime.securesms.database.AttachmentTable.ArchiveTransferState.entries
import org.thoughtcrime.securesms.database.AttachmentTable.Companion.DATA_FILE
import org.thoughtcrime.securesms.database.AttachmentTable.Companion.DATA_HASH_END
import org.thoughtcrime.securesms.database.AttachmentTable.Companion.PREUPLOAD_MESSAGE_ID
import org.thoughtcrime.securesms.database.AttachmentTable.Companion.TRANSFER_PROGRESS_DONE
import org.thoughtcrime.securesms.database.AttachmentTable.ThumbnailRestoreState.entries
import org.thoughtcrime.securesms.database.MessageTable.SyncMessageId
import org.thoughtcrime.securesms.database.SignalDatabase.Companion.messages
import org.thoughtcrime.securesms.database.SignalDatabase.Companion.threads
@@ -95,11 +106,13 @@ import org.thoughtcrime.securesms.stickers.StickerLocator
import org.thoughtcrime.securesms.util.FileUtils
import org.thoughtcrime.securesms.util.JsonUtils.SaneJSONObject
import org.thoughtcrime.securesms.util.MediaUtil
import org.thoughtcrime.securesms.util.RemoteConfig
import org.thoughtcrime.securesms.util.StorageUtil
import org.thoughtcrime.securesms.util.Util
import org.thoughtcrime.securesms.video.EncryptedMediaDataSource
import org.whispersystems.signalservice.api.attachment.AttachmentUploadResult
import org.whispersystems.signalservice.api.crypto.AttachmentCipherOutputStream
import org.whispersystems.signalservice.api.crypto.AttachmentCipherStreamUtil
import org.whispersystems.signalservice.api.util.UuidUtil
import org.whispersystems.signalservice.internal.crypto.PaddingInputStream
import org.whispersystems.signalservice.internal.util.JsonUtil
@@ -2697,16 +2710,74 @@ class AttachmentTable(
}
}
fun debugGetLatestAttachments(): List<DatabaseAttachment> {
return readableDatabase
.select(*PROJECTION)
.from(TABLE_NAME)
.where("$REMOTE_LOCATION IS NOT NULL AND $REMOTE_KEY IS NOT NULL")
.orderBy("$ID DESC")
.limit(30)
.run()
.readToList { it.readAttachments() }
.flatten()
fun debugGetAttachmentStats(): DebugAttachmentStats {
val count = readableDatabase.count().from(TABLE_NAME).run().readToSingleLong(0)
val transferStates = mapOf(
TRANSFER_PROGRESS_DONE to "TRANSFER_PROGRESS_DONE",
TRANSFER_PROGRESS_STARTED to "TRANSFER_PROGRESS_STARTED",
TRANSFER_PROGRESS_PENDING to "TRANSFER_PROGRESS_PENDING",
TRANSFER_PROGRESS_FAILED to "TRANSFER_PROGRESS_FAILED",
TRANSFER_PROGRESS_PERMANENT_FAILURE to "TRANSFER_PROGRESS_PERMANENT_FAILURE",
TRANSFER_NEEDS_RESTORE to "TRANSFER_NEEDS_RESTORE",
TRANSFER_RESTORE_IN_PROGRESS to "TRANSFER_RESTORE_IN_PROGRESS",
TRANSFER_RESTORE_OFFLOADED to "TRANSFER_RESTORE_OFFLOADED"
)
val transferStateCounts = transferStates
.map { (state, name) -> name to readableDatabase.count().from(TABLE_NAME).where("$TRANSFER_STATE = $state AND $REMOTE_DIGEST NOT NULL").run().readToSingleLong(-1L) }
.toMap()
val validForArchiveTransferStateCounts = transferStates
.map { (state, name) -> name to readableDatabase.count().from(TABLE_NAME).where("$TRANSFER_STATE = $state AND $REMOTE_DIGEST NOT NULL AND $DATA_FILE NOT NULL").run().readToSingleLong(-1L) }
.toMap()
val archiveStateCounts = ArchiveTransferState
.entries
.associate { it to readableDatabase.count().from(TABLE_NAME).where("$ARCHIVE_TRANSFER_STATE = ${it.value} AND $REMOTE_DIGEST NOT NULL").run().readToSingleLong(-1L) }
val attachmentFileCount = readableDatabase.query("SELECT COUNT(DISTINCT $DATA_FILE) FROM $TABLE_NAME WHERE $DATA_FILE NOT NULL AND $REMOTE_DIGEST NOT NULL").readToSingleLong(-1L)
val finishedAttachmentFileCount = readableDatabase.query("SELECT COUNT(DISTINCT $DATA_FILE) FROM $TABLE_NAME WHERE $DATA_FILE NOT NULL AND $REMOTE_DIGEST NOT NULL AND $ARCHIVE_TRANSFER_STATE = ${ArchiveTransferState.FINISHED.value}").readToSingleLong(-1L)
val attachmentDigestCount = readableDatabase.query("SELECT COUNT(DISTINCT $REMOTE_DIGEST) FROM $TABLE_NAME WHERE $REMOTE_DIGEST NOT NULL AND $TRANSFER_STATE in ($TRANSFER_PROGRESS_DONE, $TRANSFER_RESTORE_OFFLOADED, $TRANSFER_RESTORE_IN_PROGRESS, $TRANSFER_NEEDS_RESTORE)").readToSingleLong(-1L)
val finishedAttachmentDigestCount = readableDatabase.query("SELECT COUNT(DISTINCT $REMOTE_DIGEST) FROM $TABLE_NAME WHERE $REMOTE_DIGEST NOT NULL AND $ARCHIVE_TRANSFER_STATE = ${ArchiveTransferState.FINISHED.value}").readToSingleLong(-1L)
val thumbnailFileCount = readableDatabase.query("SELECT COUNT(DISTINCT $THUMBNAIL_FILE) FROM $TABLE_NAME WHERE $THUMBNAIL_FILE IS NOT NULL").readToSingleLong(-1L)
val estimatedThumbnailCount = readableDatabase.query("SELECT COUNT(DISTINCT $REMOTE_DIGEST) FROM $TABLE_NAME WHERE $ARCHIVE_TRANSFER_STATE = ${ArchiveTransferState.FINISHED.value} AND $REMOTE_DIGEST NOT NULL AND ($CONTENT_TYPE LIKE 'image/%' OR $CONTENT_TYPE LIKE 'video/%')").readToSingleLong(-1L)
val pendingUploadBytes = getPendingArchiveUploadBytes()
val uploadedAttachmentBytes = readableDatabase
.rawQuery(
"""
SELECT $DATA_SIZE
FROM (
SELECT DISTINCT $REMOTE_DIGEST, $DATA_SIZE
FROM $TABLE_NAME
WHERE
$DATA_FILE NOT NULL AND
$REMOTE_DIGEST NOT NULL AND
$ARCHIVE_TRANSFER_STATE = ${ArchiveTransferState.FINISHED.value}
)
""".trimIndent()
)
.readToList { it.requireLong(DATA_SIZE) }
.sumOf { AttachmentCipherStreamUtil.getCiphertextLength(AttachmentCipherStreamUtil.getCiphertextLength(PaddingInputStream.getPaddedSize(it))) }
val uploadedThumbnailBytes = estimatedThumbnailCount * RemoteConfig.backupMaxThumbnailFileSize.inWholeBytes
return DebugAttachmentStats(
attachmentCount = count,
transferStateCounts = transferStateCounts,
validForArchiveTransferStateCounts = validForArchiveTransferStateCounts,
archiveStateCounts = archiveStateCounts,
attachmentFileCount = attachmentFileCount,
finishedAttachmentFileCount = finishedAttachmentFileCount,
attachmentDigestCount = attachmentDigestCount,
finishedAttachmentDigestCount = finishedAttachmentDigestCount,
thumbnailFileCount = thumbnailFileCount,
estimatedThumbnailCount = estimatedThumbnailCount,
pendingUploadBytes = pendingUploadBytes,
uploadedAttachmentBytes = uploadedAttachmentBytes,
thumbnailBytes = uploadedThumbnailBytes
)
}
class DataFileWriteResult(
@@ -2944,4 +3015,20 @@ class AttachmentTable(
return attachmentId.hashCode()
}
}
data class DebugAttachmentStats(
val attachmentCount: Long = 0L,
val transferStateCounts: Map<String, Long> = emptyMap(),
val archiveStateCounts: Map<ArchiveTransferState, Long> = emptyMap(),
val attachmentFileCount: Long = 0L,
val finishedAttachmentFileCount: Long = 0L,
val attachmentDigestCount: Long = 0L,
val finishedAttachmentDigestCount: Long,
val thumbnailFileCount: Long = 0L,
val pendingUploadBytes: Long = 0L,
val uploadedAttachmentBytes: Long = 0L,
val thumbnailBytes: Long = 0L,
val validForArchiveTransferStateCounts: Map<String, Long>,
val estimatedThumbnailCount: Long
)
}