Add additional local metrics around storage service writes/reads.

app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/storage/InternalStorageServicePlaygroundFragment.kt

@@ -11,6 +11,7 @@ import androidx.compose.foundation.layout.Row
 import androidx.compose.foundation.layout.Spacer
 import androidx.compose.foundation.layout.fillMaxHeight
 import androidx.compose.foundation.layout.fillMaxWidth
+import androidx.compose.foundation.layout.height
 import androidx.compose.foundation.layout.padding
 import androidx.compose.foundation.layout.width
 import androidx.compose.foundation.lazy.LazyColumn
@@ -48,6 +49,7 @@ import org.signal.core.ui.compose.SignalPreview
 import org.signal.core.util.Hex
 import org.thoughtcrime.securesms.R
 import org.thoughtcrime.securesms.components.settings.app.internal.storage.InternalStorageServicePlaygroundViewModel.OneOffEvent
+import org.thoughtcrime.securesms.components.settings.app.internal.storage.InternalStorageServicePlaygroundViewModel.StorageInsights
 import org.thoughtcrime.securesms.compose.ComposeFragment
 import org.thoughtcrime.securesms.dependencies.AppDependencies
 import org.thoughtcrime.securesms.jobs.StorageForcePushJob
@@ -70,6 +72,7 @@ class InternalStorageServicePlaygroundFragment : ComposeFragment() {
   override fun FragmentContent() {
     val manifest by viewModel.manifest
     val storageRecords by viewModel.storageRecords
+    val storageInsights by viewModel.storageInsights
     val oneOffEvent by viewModel.oneOffEvents
     var forceSsreToggled by remember { mutableStateOf(SignalStore.internal.forceSsre2Capability) }
 
@@ -77,6 +80,7 @@ class InternalStorageServicePlaygroundFragment : ComposeFragment() {
       onBackPressed = { findNavController().popBackStack() },
       manifest = manifest,
       storageRecords = storageRecords,
+      storageInsights = storageInsights,
       oneOffEvent = oneOffEvent,
       forceSsreCapability = forceSsreToggled,
       onForceSsreToggled = { checked ->
@@ -93,6 +97,7 @@ class InternalStorageServicePlaygroundFragment : ComposeFragment() {
 fun Screen(
   manifest: SignalStorageManifest,
   storageRecords: List<SignalStorageRecord>,
+  storageInsights: StorageInsights,
   forceSsreCapability: Boolean,
   oneOffEvent: OneOffEvent,
   onForceSsreToggled: (Boolean) -> Unit = {},
@@ -143,6 +148,7 @@ fun Screen(
         1 -> ViewScreen(
           manifest = manifest,
           storageRecords = storageRecords,
+          storageInsights = storageInsights,
           oneOffEvent = oneOffEvent
         )
       }
@@ -191,6 +197,7 @@ fun ToolScreen(
 fun ViewScreen(
   manifest: SignalStorageManifest,
   storageRecords: List<SignalStorageRecord>,
+  storageInsights: StorageInsights,
   oneOffEvent: OneOffEvent
 ) {
   val context = LocalContext.current
@@ -217,6 +224,10 @@ fun ViewScreen(
       ManifestRow(manifest)
       Dividers.Default()
     }
+    item(key = "insights") {
+      InsightsRow(storageInsights)
+      Dividers.Default()
+    }
     storageRecords.forEach { record ->
       item(key = Hex.toStringCondensed(record.id.raw)) {
         StorageRecordRow(record)
@@ -236,10 +247,46 @@ private fun ManifestRow(manifest: SignalStorageManifest) {
   }
 }
 
+@Composable
+private fun InsightsRow(insights: StorageInsights) {
+  Column {
+    ManifestItemRow("Total Manifest Size", insights.totalManifestSize.toUnitString())
+    ManifestItemRow("Total Record Size", insights.totalRecordSize.toUnitString())
+
+    Spacer(Modifier.height(16.dp))
+
+    ManifestItemRow("Total Account Record Size", insights.totalAccountRecordSize.toUnitString())
+    ManifestItemRow("Total Contact Record Size", insights.totalContactSize.toUnitString())
+    ManifestItemRow("Total GroupV1 Record Size", insights.totalGroupV1Size.toUnitString())
+    ManifestItemRow("Total GroupV2 Record Size", insights.totalGroupV2Size.toUnitString())
+    ManifestItemRow("Total Call Link Record Size", insights.totalCallLinkSize.toUnitString())
+    ManifestItemRow("Total Distribution List Record Size", insights.totalDistributionListSize.toUnitString())
+    ManifestItemRow("Total Chat Folder Record Size", insights.totalChatFolderSize.toUnitString())
+    ManifestItemRow("Total Unknown Record Size", insights.totalUnknownSize.toUnitString())
+
+    Spacer(Modifier.height(16.dp))
+
+    if (listOf(
+        insights.totalContactSize,
+        insights.totalGroupV1Size,
+        insights.totalGroupV2Size,
+        insights.totalAccountRecordSize,
+        insights.totalCallLinkSize,
+        insights.totalDistributionListSize,
+        insights.totalChatFolderSize
+      ).sumOf { it.bytes } != insights.totalRecordSize.bytes
+    ) {
+      Text("Mismatch! Sum of record sizes does not match our total record size!")
+    } else {
+      Text("Everything adds up \uD83D\uDC4D")
+    }
+  }
+}
+
 @Composable
 private fun ManifestItemRow(title: String, value: String) {
   Row(modifier = Modifier.fillMaxWidth()) {
-    Text(title + ":", fontWeight = FontWeight.Bold)
+    Text("$title:", fontWeight = FontWeight.Bold)
     Spacer(Modifier.width(6.dp))
     Text(value)
   }
@@ -329,6 +376,7 @@ fun ScreenPreview() {
       forceSsreCapability = true,
       manifest = SignalStorageManifest.EMPTY,
       storageRecords = emptyList(),
+      storageInsights = StorageInsights(),
       oneOffEvent = OneOffEvent.None
     )
   }
@@ -361,6 +409,7 @@ fun ViewScreenPreview() {
         storageIds = storageRecords.map { it.id }
       ),
       storageRecords = storageRecords,
+      storageInsights = StorageInsights(),
       oneOffEvent = OneOffEvent.None
     )
   }

app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/storage/InternalStorageServicePlaygroundViewModel.kt

@@ -13,6 +13,8 @@ import androidx.lifecycle.viewModelScope
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.launch
 import kotlinx.coroutines.withContext
+import org.signal.core.util.ByteSize
+import org.signal.core.util.bytes
 import org.signal.core.util.logging.Log
 import org.thoughtcrime.securesms.dependencies.AppDependencies
 import org.thoughtcrime.securesms.keyvalue.SignalStore
@@ -34,6 +36,10 @@ class InternalStorageServicePlaygroundViewModel : ViewModel() {
   val storageRecords: State<List<SignalStorageRecord>>
     get() = _storageItems
 
+  private val _storageInsights: MutableState<StorageInsights> = mutableStateOf(StorageInsights())
+  val storageInsights: State<StorageInsights>
+    get() = _storageInsights
+
   private val _oneOffEvents: MutableState<OneOffEvent> = mutableStateOf(OneOffEvent.None)
   val oneOffEvents: State<OneOffEvent>
     get() = _oneOffEvents
@@ -69,11 +75,44 @@ class InternalStorageServicePlaygroundViewModel : ViewModel() {
         }
 
         _storageItems.value = records
+
+        // TODO get total manifest size -- we need the raw proto, which we don't have
+        val insights = StorageInsights(
+          totalManifestSize = manifest.protoByteSize,
+          totalRecordSize = records.sumOf { it.sizeInBytes() }.bytes,
+          totalContactSize = records.filter { it.proto.contact != null }.sumOf { it.sizeInBytes() }.bytes,
+          totalGroupV1Size = records.filter { it.proto.groupV1 != null }.sumOf { it.sizeInBytes() }.bytes,
+          totalGroupV2Size = records.filter { it.proto.groupV2 != null }.sumOf { it.sizeInBytes() }.bytes,
+          totalAccountRecordSize = records.filter { it.proto.account != null }.sumOf { it.sizeInBytes() }.bytes,
+          totalCallLinkSize = records.filter { it.proto.callLink != null }.sumOf { it.sizeInBytes() }.bytes,
+          totalDistributionListSize = records.filter { it.proto.storyDistributionList != null }.sumOf { it.sizeInBytes() }.bytes,
+          totalChatFolderSize = records.filter { it.proto.chatFolder != null }.sumOf { it.sizeInBytes() }.bytes,
+          totalUnknownSize = records.filter { it.isUnknown }.sumOf { it.sizeInBytes() }.bytes
+        )
+
+        _storageInsights.value = insights
       }
     }
   }
 
+  private fun SignalStorageRecord.sizeInBytes(): Int {
+    return this.proto.encode().size
+  }
+
   enum class OneOffEvent {
     None, ManifestDecryptionError, StorageRecordDecryptionError, ManifestNotFoundError
   }
+
+  data class StorageInsights(
+    val totalManifestSize: ByteSize = 0.bytes,
+    val totalRecordSize: ByteSize = 0.bytes,
+    val totalContactSize: ByteSize = 0.bytes,
+    val totalGroupV1Size: ByteSize = 0.bytes,
+    val totalGroupV2Size: ByteSize = 0.bytes,
+    val totalAccountRecordSize: ByteSize = 0.bytes,
+    val totalCallLinkSize: ByteSize = 0.bytes,
+    val totalDistributionListSize: ByteSize = 0.bytes,
+    val totalChatFolderSize: ByteSize = 0.bytes,
+    val totalUnknownSize: ByteSize = 0.bytes
+  )
 }

app/src/main/java/org/thoughtcrime/securesms/net/StorageServiceSizeLoggingInterceptor.kt

@@ -0,0 +1,110 @@
+/*
+ * Copyright 2025 Signal Messenger, LLC
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+package org.thoughtcrime.securesms.net
+
+import okhttp3.Interceptor
+import okhttp3.Request
+import okhttp3.Response
+import org.signal.core.util.bytes
+import org.signal.core.util.concurrent.SignalExecutors
+import org.signal.core.util.logging.Log
+import org.thoughtcrime.securesms.database.LocalMetricsDatabase
+import org.thoughtcrime.securesms.database.model.LocalMetricsEvent
+import org.thoughtcrime.securesms.database.model.LocalMetricsSplit
+import org.thoughtcrime.securesms.dependencies.AppDependencies
+import java.util.UUID
+import java.util.concurrent.Executor
+import java.util.concurrent.TimeUnit
+
+/**
+ * We're investigating a bug around the size of storage service request and response sizes.
+ * This interceptor logs the size of requests and responses to the local metrics database.
+ */
+class StorageServiceSizeLoggingInterceptor : Interceptor {
+
+  companion object {
+    private val TAG = Log.tag(StorageServiceSizeLoggingInterceptor::class)
+
+    private const val KEY_REQUEST_SIZE = "storage-request-size"
+    private const val KEY_RESPONSE_SIZE = "storage-response-size"
+    private val PATH_REGEX = ".*/v1/storage.*".toRegex()
+  }
+
+  private val executor: Executor = SignalExecutors.UNBOUNDED
+
+  override fun intercept(chain: Interceptor.Chain): Response {
+    val request = chain.request()
+    val path = request.url.encodedPath
+
+    if (!PATH_REGEX.matches(path)) {
+      return chain.proceed(request)
+    }
+
+    val requestSize = request.body?.contentLength() ?: -1L
+
+    val response = chain.proceed(request)
+    val responseBody = response.body ?: return response
+
+    val source = responseBody.source()
+    source.request(Long.MAX_VALUE) // Buffer the entire body.
+    val responseSize = source.buffer.size
+
+    Log.d(TAG, "[${request.method} $path] Request(size = ${requestSize.bytes.toUnitString()}), Response(code = ${response.code}, size = ${responseSize.bytes.toUnitString()})")
+    executor.execute {
+      if (!response.isSuccessful) {
+        return@execute
+      }
+
+      if (requestSize > 0) {
+        val event = LocalMetricsEvent(
+          createdAt = System.currentTimeMillis(),
+          eventId = "$KEY_REQUEST_SIZE-${UUID.randomUUID()}}",
+          eventName = "[${KEY_REQUEST_SIZE}] ${request.buildEventName()}",
+          splits = mutableListOf(
+            LocalMetricsSplit(
+              name = "size",
+              duration = requestSize,
+              timeunit = TimeUnit.NANOSECONDS
+            )
+          ),
+          timeUnit = TimeUnit.NANOSECONDS,
+          extraLabel = null
+        )
+        LocalMetricsDatabase.getInstance(AppDependencies.application).insert(System.currentTimeMillis(), event)
+      }
+
+      if (responseSize > 0) {
+        val event = LocalMetricsEvent(
+          createdAt = System.currentTimeMillis(),
+          eventId = "$KEY_RESPONSE_SIZE-${UUID.randomUUID()}}",
+          eventName = "[${KEY_RESPONSE_SIZE}] ${request.buildEventName()}",
+          splits = mutableListOf(
+            LocalMetricsSplit(
+              name = "size",
+              duration = responseSize,
+              timeunit = TimeUnit.NANOSECONDS
+            )
+          ),
+          timeUnit = TimeUnit.NANOSECONDS,
+          extraLabel = null
+        )
+        LocalMetricsDatabase.getInstance(AppDependencies.application).insert(System.currentTimeMillis(), event)
+      }
+    }
+
+    return response
+  }
+
+  private fun Request.buildEventName(): String {
+    var path = this.url.encodedPath
+    val method = this.method
+    if (path.matches("/v1/storage/manifest/version/\\d+".toRegex())) {
+      path = "/v1/storage/manifest/version/_version"
+    }
+
+    return "$method $path"
+  }
+}

app/src/main/java/org/thoughtcrime/securesms/push/SignalServiceNetworkAccess.kt

@@ -22,6 +22,7 @@ import org.thoughtcrime.securesms.net.RemoteDeprecationDetectorInterceptor
 import org.thoughtcrime.securesms.net.SequentialDns
 import org.thoughtcrime.securesms.net.StandardUserAgentInterceptor
 import org.thoughtcrime.securesms.net.StaticDns
+import org.thoughtcrime.securesms.net.StorageServiceSizeLoggingInterceptor
 import org.whispersystems.signalservice.api.push.TrustStore
 import org.whispersystems.signalservice.internal.configuration.HttpProxy
 import org.whispersystems.signalservice.internal.configuration.SignalCdnUrl
@@ -168,6 +169,7 @@ class SignalServiceNetworkAccess(context: Context) {
 
   private val interceptors: List<Interceptor> = listOf(
     StandardUserAgentInterceptor(),
+    StorageServiceSizeLoggingInterceptor(),
     RemoteDeprecationDetectorInterceptor(this::getConfiguration),
     DeprecatedClientPreventionInterceptor(),
     DeviceTransferBlockingInterceptor.getInstance()

libsignal-service/src/main/java/org/whispersystems/signalservice/api/storage/SignalStorageManifest.kt

@@ -3,6 +3,8 @@ package org.whispersystems.signalservice.api.storage
 import okio.ByteString
 import okio.ByteString.Companion.EMPTY
 import okio.ByteString.Companion.toByteString
+import org.signal.core.util.ByteSize
+import org.signal.core.util.bytes
 import org.signal.core.util.isNotEmpty
 import org.signal.core.util.toOptional
 import org.whispersystems.signalservice.internal.storage.protos.ManifestRecord
@@ -13,9 +15,18 @@ data class SignalStorageManifest(
   @JvmField val version: Long,
   val sourceDeviceId: Int,
   val recordIkm: RecordIkm?,
-  @JvmField val storageIds: List<StorageId>
+  @JvmField val storageIds: List<StorageId>,
+  val protoByteSize: ByteSize
 ) {
 
+  constructor(version: Long, sourceDeviceId: Int, recordIkm: RecordIkm?, storageIds: List<StorageId>) : this(
+    version = version,
+    sourceDeviceId = sourceDeviceId,
+    recordIkm = recordIkm,
+    storageIds = storageIds,
+    protoByteSize = toProto(version, storageIds, sourceDeviceId, recordIkm).encode().size.bytes
+  )
+
   companion object {
     val EMPTY: SignalStorageManifest = SignalStorageManifest(0, 1, null, emptyList())
 
@@ -30,7 +41,25 @@ data class SignalStorageManifest(
         version = manifest.version,
         sourceDeviceId = manifestRecord.sourceDevice,
         recordIkm = manifestRecord.recordIkm.takeIf { it.isNotEmpty() }?.toByteArray()?.let { RecordIkm(it) },
-        storageIds = ids
+        storageIds = ids,
+        protoByteSize = serialized.size.bytes
+      )
+    }
+
+    private fun toProto(version: Long, storageIds: List<StorageId>, sourceDeviceId: Int, recordIkm: RecordIkm?): StorageManifest {
+      val ids: List<ManifestRecord.Identifier> = storageIds.map { id ->
+        ManifestRecord.Identifier.fromPossiblyUnknownType(id.type, id.raw)
+      }
+
+      val manifestRecord = ManifestRecord(
+        identifiers = ids,
+        sourceDevice = sourceDeviceId,
+        recordIkm = recordIkm?.value?.toByteString() ?: ByteString.EMPTY
+      )
+
+      return StorageManifest(
+        version = version,
+        value_ = manifestRecord.encodeByteString()
       )
     }
   }

libsignal-service/src/main/java/org/whispersystems/signalservice/api/storage/StorageServiceRepository.kt

@@ -9,6 +9,7 @@ import com.squareup.wire.FieldEncoding
 import okio.ByteString
 import okio.ByteString.Companion.toByteString
 import okio.IOException
+import org.signal.core.util.bytes
 import org.signal.core.util.isNotEmpty
 import org.signal.libsignal.protocol.InvalidKeyException
 import org.whispersystems.signalservice.api.NetworkResult
@@ -272,7 +273,8 @@ class StorageServiceRepository(private val storageServiceApi: StorageServiceApi)
       version = manifestRecord.version,
       sourceDeviceId = manifestRecord.sourceDevice,
       recordIkm = manifestRecord.recordIkm.takeIf { it.isNotEmpty() }?.toByteArray()?.let { RecordIkm(it) },
-      storageIds = ids
+      storageIds = ids,
+      protoByteSize = this.encode().size.bytes
     )
   }