diff --git a/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/storage/InternalStorageServicePlaygroundFragment.kt b/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/storage/InternalStorageServicePlaygroundFragment.kt index 742eea4e3c..de518c8ad6 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/storage/InternalStorageServicePlaygroundFragment.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/storage/InternalStorageServicePlaygroundFragment.kt @@ -11,6 +11,7 @@ import androidx.compose.foundation.layout.Row import androidx.compose.foundation.layout.Spacer import androidx.compose.foundation.layout.fillMaxHeight import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.height import androidx.compose.foundation.layout.padding import androidx.compose.foundation.layout.width import androidx.compose.foundation.lazy.LazyColumn @@ -48,6 +49,7 @@ import org.signal.core.ui.compose.SignalPreview import org.signal.core.util.Hex import org.thoughtcrime.securesms.R import org.thoughtcrime.securesms.components.settings.app.internal.storage.InternalStorageServicePlaygroundViewModel.OneOffEvent +import org.thoughtcrime.securesms.components.settings.app.internal.storage.InternalStorageServicePlaygroundViewModel.StorageInsights import org.thoughtcrime.securesms.compose.ComposeFragment import org.thoughtcrime.securesms.dependencies.AppDependencies import org.thoughtcrime.securesms.jobs.StorageForcePushJob @@ -70,6 +72,7 @@ class InternalStorageServicePlaygroundFragment : ComposeFragment() { override fun FragmentContent() { val manifest by viewModel.manifest val storageRecords by viewModel.storageRecords + val storageInsights by viewModel.storageInsights val oneOffEvent by viewModel.oneOffEvents var forceSsreToggled by remember { mutableStateOf(SignalStore.internal.forceSsre2Capability) } @@ -77,6 +80,7 @@ class InternalStorageServicePlaygroundFragment : ComposeFragment() { onBackPressed = { findNavController().popBackStack() }, manifest = manifest, storageRecords = storageRecords, + storageInsights = storageInsights, oneOffEvent = oneOffEvent, forceSsreCapability = forceSsreToggled, onForceSsreToggled = { checked -> @@ -93,6 +97,7 @@ class InternalStorageServicePlaygroundFragment : ComposeFragment() { fun Screen( manifest: SignalStorageManifest, storageRecords: List, + storageInsights: StorageInsights, forceSsreCapability: Boolean, oneOffEvent: OneOffEvent, onForceSsreToggled: (Boolean) -> Unit = {}, @@ -143,6 +148,7 @@ fun Screen( 1 -> ViewScreen( manifest = manifest, storageRecords = storageRecords, + storageInsights = storageInsights, oneOffEvent = oneOffEvent ) } @@ -191,6 +197,7 @@ fun ToolScreen( fun ViewScreen( manifest: SignalStorageManifest, storageRecords: List, + storageInsights: StorageInsights, oneOffEvent: OneOffEvent ) { val context = LocalContext.current @@ -217,6 +224,10 @@ fun ViewScreen( ManifestRow(manifest) Dividers.Default() } + item(key = "insights") { + InsightsRow(storageInsights) + Dividers.Default() + } storageRecords.forEach { record -> item(key = Hex.toStringCondensed(record.id.raw)) { StorageRecordRow(record) @@ -236,10 +247,46 @@ private fun ManifestRow(manifest: SignalStorageManifest) { } } +@Composable +private fun InsightsRow(insights: StorageInsights) { + Column { + ManifestItemRow("Total Manifest Size", insights.totalManifestSize.toUnitString()) + ManifestItemRow("Total Record Size", insights.totalRecordSize.toUnitString()) + + Spacer(Modifier.height(16.dp)) + + ManifestItemRow("Total Account Record Size", insights.totalAccountRecordSize.toUnitString()) + ManifestItemRow("Total Contact Record Size", insights.totalContactSize.toUnitString()) + ManifestItemRow("Total GroupV1 Record Size", insights.totalGroupV1Size.toUnitString()) + ManifestItemRow("Total GroupV2 Record Size", insights.totalGroupV2Size.toUnitString()) + ManifestItemRow("Total Call Link Record Size", insights.totalCallLinkSize.toUnitString()) + ManifestItemRow("Total Distribution List Record Size", insights.totalDistributionListSize.toUnitString()) + ManifestItemRow("Total Chat Folder Record Size", insights.totalChatFolderSize.toUnitString()) + ManifestItemRow("Total Unknown Record Size", insights.totalUnknownSize.toUnitString()) + + Spacer(Modifier.height(16.dp)) + + if (listOf( + insights.totalContactSize, + insights.totalGroupV1Size, + insights.totalGroupV2Size, + insights.totalAccountRecordSize, + insights.totalCallLinkSize, + insights.totalDistributionListSize, + insights.totalChatFolderSize + ).sumOf { it.bytes } != insights.totalRecordSize.bytes + ) { + Text("Mismatch! Sum of record sizes does not match our total record size!") + } else { + Text("Everything adds up \uD83D\uDC4D") + } + } +} + @Composable private fun ManifestItemRow(title: String, value: String) { Row(modifier = Modifier.fillMaxWidth()) { - Text(title + ":", fontWeight = FontWeight.Bold) + Text("$title:", fontWeight = FontWeight.Bold) Spacer(Modifier.width(6.dp)) Text(value) } @@ -329,6 +376,7 @@ fun ScreenPreview() { forceSsreCapability = true, manifest = SignalStorageManifest.EMPTY, storageRecords = emptyList(), + storageInsights = StorageInsights(), oneOffEvent = OneOffEvent.None ) } @@ -361,6 +409,7 @@ fun ViewScreenPreview() { storageIds = storageRecords.map { it.id } ), storageRecords = storageRecords, + storageInsights = StorageInsights(), oneOffEvent = OneOffEvent.None ) } diff --git a/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/storage/InternalStorageServicePlaygroundViewModel.kt b/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/storage/InternalStorageServicePlaygroundViewModel.kt index cf30c3856c..cc34711513 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/storage/InternalStorageServicePlaygroundViewModel.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/storage/InternalStorageServicePlaygroundViewModel.kt @@ -13,6 +13,8 @@ import androidx.lifecycle.viewModelScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.launch import kotlinx.coroutines.withContext +import org.signal.core.util.ByteSize +import org.signal.core.util.bytes import org.signal.core.util.logging.Log import org.thoughtcrime.securesms.dependencies.AppDependencies import org.thoughtcrime.securesms.keyvalue.SignalStore @@ -34,6 +36,10 @@ class InternalStorageServicePlaygroundViewModel : ViewModel() { val storageRecords: State> get() = _storageItems + private val _storageInsights: MutableState = mutableStateOf(StorageInsights()) + val storageInsights: State + get() = _storageInsights + private val _oneOffEvents: MutableState = mutableStateOf(OneOffEvent.None) val oneOffEvents: State get() = _oneOffEvents @@ -69,11 +75,44 @@ class InternalStorageServicePlaygroundViewModel : ViewModel() { } _storageItems.value = records + + // TODO get total manifest size -- we need the raw proto, which we don't have + val insights = StorageInsights( + totalManifestSize = manifest.protoByteSize, + totalRecordSize = records.sumOf { it.sizeInBytes() }.bytes, + totalContactSize = records.filter { it.proto.contact != null }.sumOf { it.sizeInBytes() }.bytes, + totalGroupV1Size = records.filter { it.proto.groupV1 != null }.sumOf { it.sizeInBytes() }.bytes, + totalGroupV2Size = records.filter { it.proto.groupV2 != null }.sumOf { it.sizeInBytes() }.bytes, + totalAccountRecordSize = records.filter { it.proto.account != null }.sumOf { it.sizeInBytes() }.bytes, + totalCallLinkSize = records.filter { it.proto.callLink != null }.sumOf { it.sizeInBytes() }.bytes, + totalDistributionListSize = records.filter { it.proto.storyDistributionList != null }.sumOf { it.sizeInBytes() }.bytes, + totalChatFolderSize = records.filter { it.proto.chatFolder != null }.sumOf { it.sizeInBytes() }.bytes, + totalUnknownSize = records.filter { it.isUnknown }.sumOf { it.sizeInBytes() }.bytes + ) + + _storageInsights.value = insights } } } + private fun SignalStorageRecord.sizeInBytes(): Int { + return this.proto.encode().size + } + enum class OneOffEvent { None, ManifestDecryptionError, StorageRecordDecryptionError, ManifestNotFoundError } + + data class StorageInsights( + val totalManifestSize: ByteSize = 0.bytes, + val totalRecordSize: ByteSize = 0.bytes, + val totalContactSize: ByteSize = 0.bytes, + val totalGroupV1Size: ByteSize = 0.bytes, + val totalGroupV2Size: ByteSize = 0.bytes, + val totalAccountRecordSize: ByteSize = 0.bytes, + val totalCallLinkSize: ByteSize = 0.bytes, + val totalDistributionListSize: ByteSize = 0.bytes, + val totalChatFolderSize: ByteSize = 0.bytes, + val totalUnknownSize: ByteSize = 0.bytes + ) } diff --git a/app/src/main/java/org/thoughtcrime/securesms/net/StorageServiceSizeLoggingInterceptor.kt b/app/src/main/java/org/thoughtcrime/securesms/net/StorageServiceSizeLoggingInterceptor.kt new file mode 100644 index 0000000000..31f9582f93 --- /dev/null +++ b/app/src/main/java/org/thoughtcrime/securesms/net/StorageServiceSizeLoggingInterceptor.kt @@ -0,0 +1,110 @@ +/* + * Copyright 2025 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ + +package org.thoughtcrime.securesms.net + +import okhttp3.Interceptor +import okhttp3.Request +import okhttp3.Response +import org.signal.core.util.bytes +import org.signal.core.util.concurrent.SignalExecutors +import org.signal.core.util.logging.Log +import org.thoughtcrime.securesms.database.LocalMetricsDatabase +import org.thoughtcrime.securesms.database.model.LocalMetricsEvent +import org.thoughtcrime.securesms.database.model.LocalMetricsSplit +import org.thoughtcrime.securesms.dependencies.AppDependencies +import java.util.UUID +import java.util.concurrent.Executor +import java.util.concurrent.TimeUnit + +/** + * We're investigating a bug around the size of storage service request and response sizes. + * This interceptor logs the size of requests and responses to the local metrics database. + */ +class StorageServiceSizeLoggingInterceptor : Interceptor { + + companion object { + private val TAG = Log.tag(StorageServiceSizeLoggingInterceptor::class) + + private const val KEY_REQUEST_SIZE = "storage-request-size" + private const val KEY_RESPONSE_SIZE = "storage-response-size" + private val PATH_REGEX = ".*/v1/storage.*".toRegex() + } + + private val executor: Executor = SignalExecutors.UNBOUNDED + + override fun intercept(chain: Interceptor.Chain): Response { + val request = chain.request() + val path = request.url.encodedPath + + if (!PATH_REGEX.matches(path)) { + return chain.proceed(request) + } + + val requestSize = request.body?.contentLength() ?: -1L + + val response = chain.proceed(request) + val responseBody = response.body ?: return response + + val source = responseBody.source() + source.request(Long.MAX_VALUE) // Buffer the entire body. + val responseSize = source.buffer.size + + Log.d(TAG, "[${request.method} $path] Request(size = ${requestSize.bytes.toUnitString()}), Response(code = ${response.code}, size = ${responseSize.bytes.toUnitString()})") + executor.execute { + if (!response.isSuccessful) { + return@execute + } + + if (requestSize > 0) { + val event = LocalMetricsEvent( + createdAt = System.currentTimeMillis(), + eventId = "$KEY_REQUEST_SIZE-${UUID.randomUUID()}}", + eventName = "[${KEY_REQUEST_SIZE}] ${request.buildEventName()}", + splits = mutableListOf( + LocalMetricsSplit( + name = "size", + duration = requestSize, + timeunit = TimeUnit.NANOSECONDS + ) + ), + timeUnit = TimeUnit.NANOSECONDS, + extraLabel = null + ) + LocalMetricsDatabase.getInstance(AppDependencies.application).insert(System.currentTimeMillis(), event) + } + + if (responseSize > 0) { + val event = LocalMetricsEvent( + createdAt = System.currentTimeMillis(), + eventId = "$KEY_RESPONSE_SIZE-${UUID.randomUUID()}}", + eventName = "[${KEY_RESPONSE_SIZE}] ${request.buildEventName()}", + splits = mutableListOf( + LocalMetricsSplit( + name = "size", + duration = responseSize, + timeunit = TimeUnit.NANOSECONDS + ) + ), + timeUnit = TimeUnit.NANOSECONDS, + extraLabel = null + ) + LocalMetricsDatabase.getInstance(AppDependencies.application).insert(System.currentTimeMillis(), event) + } + } + + return response + } + + private fun Request.buildEventName(): String { + var path = this.url.encodedPath + val method = this.method + if (path.matches("/v1/storage/manifest/version/\\d+".toRegex())) { + path = "/v1/storage/manifest/version/_version" + } + + return "$method $path" + } +} diff --git a/app/src/main/java/org/thoughtcrime/securesms/push/SignalServiceNetworkAccess.kt b/app/src/main/java/org/thoughtcrime/securesms/push/SignalServiceNetworkAccess.kt index f6a8849122..029abbdfe9 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/push/SignalServiceNetworkAccess.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/push/SignalServiceNetworkAccess.kt @@ -22,6 +22,7 @@ import org.thoughtcrime.securesms.net.RemoteDeprecationDetectorInterceptor import org.thoughtcrime.securesms.net.SequentialDns import org.thoughtcrime.securesms.net.StandardUserAgentInterceptor import org.thoughtcrime.securesms.net.StaticDns +import org.thoughtcrime.securesms.net.StorageServiceSizeLoggingInterceptor import org.whispersystems.signalservice.api.push.TrustStore import org.whispersystems.signalservice.internal.configuration.HttpProxy import org.whispersystems.signalservice.internal.configuration.SignalCdnUrl @@ -168,6 +169,7 @@ class SignalServiceNetworkAccess(context: Context) { private val interceptors: List = listOf( StandardUserAgentInterceptor(), + StorageServiceSizeLoggingInterceptor(), RemoteDeprecationDetectorInterceptor(this::getConfiguration), DeprecatedClientPreventionInterceptor(), DeviceTransferBlockingInterceptor.getInstance() diff --git a/libsignal-service/src/main/java/org/whispersystems/signalservice/api/storage/SignalStorageManifest.kt b/libsignal-service/src/main/java/org/whispersystems/signalservice/api/storage/SignalStorageManifest.kt index f5a8d777b5..b5d2faca8b 100644 --- a/libsignal-service/src/main/java/org/whispersystems/signalservice/api/storage/SignalStorageManifest.kt +++ b/libsignal-service/src/main/java/org/whispersystems/signalservice/api/storage/SignalStorageManifest.kt @@ -3,6 +3,8 @@ package org.whispersystems.signalservice.api.storage import okio.ByteString import okio.ByteString.Companion.EMPTY import okio.ByteString.Companion.toByteString +import org.signal.core.util.ByteSize +import org.signal.core.util.bytes import org.signal.core.util.isNotEmpty import org.signal.core.util.toOptional import org.whispersystems.signalservice.internal.storage.protos.ManifestRecord @@ -13,9 +15,18 @@ data class SignalStorageManifest( @JvmField val version: Long, val sourceDeviceId: Int, val recordIkm: RecordIkm?, - @JvmField val storageIds: List + @JvmField val storageIds: List, + val protoByteSize: ByteSize ) { + constructor(version: Long, sourceDeviceId: Int, recordIkm: RecordIkm?, storageIds: List) : this( + version = version, + sourceDeviceId = sourceDeviceId, + recordIkm = recordIkm, + storageIds = storageIds, + protoByteSize = toProto(version, storageIds, sourceDeviceId, recordIkm).encode().size.bytes + ) + companion object { val EMPTY: SignalStorageManifest = SignalStorageManifest(0, 1, null, emptyList()) @@ -30,7 +41,25 @@ data class SignalStorageManifest( version = manifest.version, sourceDeviceId = manifestRecord.sourceDevice, recordIkm = manifestRecord.recordIkm.takeIf { it.isNotEmpty() }?.toByteArray()?.let { RecordIkm(it) }, - storageIds = ids + storageIds = ids, + protoByteSize = serialized.size.bytes + ) + } + + private fun toProto(version: Long, storageIds: List, sourceDeviceId: Int, recordIkm: RecordIkm?): StorageManifest { + val ids: List = storageIds.map { id -> + ManifestRecord.Identifier.fromPossiblyUnknownType(id.type, id.raw) + } + + val manifestRecord = ManifestRecord( + identifiers = ids, + sourceDevice = sourceDeviceId, + recordIkm = recordIkm?.value?.toByteString() ?: ByteString.EMPTY + ) + + return StorageManifest( + version = version, + value_ = manifestRecord.encodeByteString() ) } } diff --git a/libsignal-service/src/main/java/org/whispersystems/signalservice/api/storage/StorageServiceRepository.kt b/libsignal-service/src/main/java/org/whispersystems/signalservice/api/storage/StorageServiceRepository.kt index 385630194c..0b6bd1a8de 100644 --- a/libsignal-service/src/main/java/org/whispersystems/signalservice/api/storage/StorageServiceRepository.kt +++ b/libsignal-service/src/main/java/org/whispersystems/signalservice/api/storage/StorageServiceRepository.kt @@ -9,6 +9,7 @@ import com.squareup.wire.FieldEncoding import okio.ByteString import okio.ByteString.Companion.toByteString import okio.IOException +import org.signal.core.util.bytes import org.signal.core.util.isNotEmpty import org.signal.libsignal.protocol.InvalidKeyException import org.whispersystems.signalservice.api.NetworkResult @@ -272,7 +273,8 @@ class StorageServiceRepository(private val storageServiceApi: StorageServiceApi) version = manifestRecord.version, sourceDeviceId = manifestRecord.sourceDevice, recordIkm = manifestRecord.recordIkm.takeIf { it.isNotEmpty() }?.toByteArray()?.let { RecordIkm(it) }, - storageIds = ids + storageIds = ids, + protoByteSize = this.encode().size.bytes ) }