Add storage sync hot loop detection and rate limiting.

This commit is contained in:
Cody Henthorne
2026-08-19 19:05:46 -04:00
parent e3fd0897e8
commit 9c01310df4
13 changed files with 1064 additions and 18 deletions
@@ -26,6 +26,9 @@ import org.thoughtcrime.securesms.notifications.NotificationIds
import org.thoughtcrime.securesms.util.RemoteConfig
import java.io.ByteArrayOutputStream
import java.io.PrintStream
import kotlin.time.Duration
import kotlin.time.Duration.Companion.days
import kotlin.time.Duration.Companion.minutes
/**
* Records noteworthy runtime issues to the [LogDatabase] issue table on a low-priority background thread.
@@ -41,6 +44,7 @@ object IssueReporter {
const val ISSUE_SLOW_DATABASE_WRITE = "Slow Database Write"
const val ISSUE_SLOW_DATABASE_READ = "Slow Database Read"
const val ISSUE_SLOW_DATABASE_LOCK = "Slow Database Lock"
const val ISSUE_STORAGE_SYNC_LOOP = "Storage Sync Loop"
const val SLOW_WRITE_LOW_PRIORITY_MS = 1_000L
const val SLOW_WRITE_MEDIUM_PRIORITY_MS = 5_000L
@@ -51,6 +55,12 @@ object IssueReporter {
private const val NON_INTERNAL_DEBOUNCE_MS = 5_000L
/** Applies per issue name, so a persistently-firing issue can't bury every other notification. */
private val DEFAULT_NOTIFY_COOLDOWN = 30.minutes
/** Ages entries out of the persisted map. Must exceed the longest cooldown a caller passes. */
private val MAX_NOTIFY_TIME_AGE = 7.days
private val IGNORED_DB_STACK_TRACE_CLASSES = listOf(
"BackupRepository",
"BackupMessagesJob",
@@ -72,9 +82,7 @@ object IssueReporter {
/**
* Records a generic issue. Safe to call from any thread.
*/
@JvmStatic
@JvmOverloads
fun report(name: String, description: String, throwable: Throwable? = null, priority: IssuePriority = IssuePriority.LOW, duration: Long? = null) {
fun report(name: String, description: String, throwable: Throwable? = null, priority: IssuePriority = IssuePriority.LOW, duration: Long? = null, notifyCooldown: Duration = DEFAULT_NOTIFY_COOLDOWN) {
val now = System.currentTimeMillis()
if (!RemoteConfig.internalUser) {
@@ -86,7 +94,9 @@ object IssueReporter {
requests.add(IssueRequest(now, BuildConfig.VERSION_NAME, name, description, throwable, priority, duration))
maybeNotify(name, priority)
if (RemoteConfig.internalUser) {
maybeNotify(name, priority, notifyCooldown)
}
}
@JvmStatic
@@ -138,12 +148,17 @@ object IssueReporter {
report(ISSUE_SLOW_DATABASE_READ, query?.trim() ?: "", throwable, priority = priority, duration = durationMs)
}
private fun maybeNotify(name: String, priority: IssuePriority) {
if (!RemoteConfig.internalUser) {
/** Synchronized because the cooldown is a read-modify-write of a persisted value, and this is safe to call anywhere. */
@Synchronized
private fun maybeNotify(name: String, priority: IssuePriority, cooldown: Duration) {
if (priority.value < SignalStore.internal.issueNotificationPriority.value) {
return
}
if (priority.value < SignalStore.internal.issueNotificationPriority.value) {
val now = System.currentTimeMillis()
val notifyTimes = SignalStore.internal.issueNotifyTimes
if (now - (notifyTimes.lastNotifyTimeByName[name] ?: 0) < cooldown.inWholeMilliseconds) {
return
}
@@ -160,6 +175,15 @@ object IssueReporter {
.build()
NotificationManagerCompat.from(context).notify(NotificationIds.INTERNAL_ERROR, notification)
val updatedTimes = notifyTimes
.lastNotifyTimeByName
.filterValues { now < it || now - it < MAX_NOTIFY_TIME_AGE.inWholeMilliseconds }
.plus(name to now)
SignalStore.internal.issueNotifyTimes = notifyTimes.copy(
lastNotifyTimeByName = updatedTimes
)
}
private fun isExpectedSlowDatabaseOperation(): Boolean {
@@ -11,10 +11,12 @@ import org.signal.libsignal.protocol.InvalidKeyException
import org.signal.network.service.StorageServiceService
import org.signal.network.service.StorageServiceService.ManifestIfDifferentVersionResult
import org.thoughtcrime.securesms.database.ChatFolderTables.ChatFolderTable
import org.thoughtcrime.securesms.database.IssueReporter
import org.thoughtcrime.securesms.database.NotificationProfileTables
import org.thoughtcrime.securesms.database.RecipientTable
import org.thoughtcrime.securesms.database.SignalDatabase
import org.thoughtcrime.securesms.database.StickerTables
import org.thoughtcrime.securesms.database.model.IssuePriority
import org.thoughtcrime.securesms.dependencies.AppDependencies
import org.thoughtcrime.securesms.jobmanager.Job
import org.thoughtcrime.securesms.jobmanager.impl.NetworkConstraint
@@ -31,6 +33,7 @@ import org.thoughtcrime.securesms.storage.NotificationProfileRecordProcessor
import org.thoughtcrime.securesms.storage.StickerPackRecordProcessor
import org.thoughtcrime.securesms.storage.StorageSyncHelper
import org.thoughtcrime.securesms.storage.StorageSyncHelper.WriteOperationResult
import org.thoughtcrime.securesms.storage.StorageSyncLoopDetector
import org.thoughtcrime.securesms.storage.StorageSyncModels
import org.thoughtcrime.securesms.storage.StorageSyncValidations
import org.thoughtcrime.securesms.storage.StoryDistributionListRecordProcessor
@@ -64,6 +67,7 @@ import org.whispersystems.signalservice.internal.storage.protos.ManifestRecord
import java.io.IOException
import java.util.concurrent.TimeUnit
import java.util.stream.Collectors
import kotlin.time.Duration.Companion.days
import kotlin.time.Duration.Companion.milliseconds
/**
@@ -257,7 +261,8 @@ class StorageSyncJob private constructor(parameters: Parameters, private var loc
val repository = StorageServiceService(SignalNetwork.storageService)
val localManifest = SignalStore.storageService.manifest
val remoteManifest = if (localManifestOutOfDate || localManifest.version < 1 || runAttempt >= 3) {
val fetchRemoteManifest = localManifestOutOfDate || localManifest.version < 1 || runAttempt >= 3
val remoteManifest = if (fetchRemoteManifest) {
Log.i(TAG, "Local manifest is invalid. Fetching remote manifest. (localManifestOutOfDate: $localManifestOutOfDate, localManifest.version: ${localManifest.version}, runAttempt: $runAttempt)")
when (val result = repository.getStorageManifestIfDifferentVersion(storageServiceKey, localManifest.version)) {
is ManifestIfDifferentVersionResult.DifferentVersion -> result.manifest
@@ -444,7 +449,26 @@ class StorageSyncJob private constructor(parameters: Parameters, private var loc
}
stopwatch.split("local-data-transaction")
if (!remoteWriteOperation.isEmpty) {
val loopCheck = if (remoteWriteOperation.isEmpty) {
StorageSyncLoopDetector.Decision.Allowed
} else {
StorageSyncLoopDetector.onWriteAttempt(remoteWriteOperation, fetchRemoteManifest, isRetry = runAttempt > 0)
}
if (remoteWriteOperation.isEmpty) {
Log.i(TAG, "No remote writes needed. Still at version: " + remoteManifest.versionString)
StorageSyncLoopDetector.onConverged()
} else if (loopCheck is StorageSyncLoopDetector.Decision.Denied) {
Log.w(TAG, "Skipping remote write, another device is likely undoing it. Cause: ${loopCheck.cause}, level: ${loopCheck.level}. WriteOperationResult :: $remoteWriteOperation")
IssueReporter.report(
name = IssueReporter.ISSUE_STORAGE_SYNC_LOOP,
description = "Throttling storage service writes. Cause: ${loopCheck.cause}, level: ${loopCheck.level}. $remoteWriteOperation",
throwable = Throwable(),
priority = IssuePriority.HIGH,
notifyCooldown = 3.days
)
} else {
Log.i(TAG, "We have something to write remotely.")
Log.i(TAG, "WriteOperationResult :: $remoteWriteOperation")
@@ -452,9 +476,16 @@ class StorageSyncJob private constructor(parameters: Parameters, private var loc
when (val result = repository.writeStorageRecords(storageServiceKey, remoteWriteOperation.manifest, remoteWriteOperation.inserts, remoteWriteOperation.deletes)) {
StorageServiceService.WriteStorageRecordsResult.Success -> Unit
is StorageServiceService.WriteStorageRecordsResult.StatusCodeError -> throw result.exception
is StorageServiceService.WriteStorageRecordsResult.NetworkError -> throw result.exception
is StorageServiceService.WriteStorageRecordsResult.StatusCodeError -> {
StorageSyncLoopDetector.onWriteFailed()
throw result.exception
}
is StorageServiceService.WriteStorageRecordsResult.NetworkError -> {
StorageSyncLoopDetector.onWriteFailed()
throw result.exception
}
StorageServiceService.WriteStorageRecordsResult.ConflictError -> {
StorageSyncLoopDetector.onWriteFailed()
Log.w(TAG, "Hit a conflict when trying to resolve the conflict! Retrying.")
localManifestOutOfDate = true
throw RetryLaterException()
@@ -468,8 +499,6 @@ class StorageSyncJob private constructor(parameters: Parameters, private var loc
stopwatch.split("remote-write")
needsMultiDeviceSync = true
} else {
Log.i(TAG, "No remote writes needed. Still at version: " + remoteManifest.versionString)
}
if (needsForcePush && SignalStore.account.isPrimaryDevice) {
@@ -4,6 +4,7 @@ import org.signal.archive.proto.BackupDebugInfo
import org.signal.ringrtc.CallManager.DataMode
import org.thoughtcrime.securesms.BuildConfig
import org.thoughtcrime.securesms.database.model.IssuePriority
import org.thoughtcrime.securesms.keyvalue.protos.IssueNotifyTimes
import org.thoughtcrime.securesms.util.Environment.Calling.defaultSfuUrl
import org.thoughtcrime.securesms.util.RemoteConfig
@@ -43,6 +44,7 @@ class InternalValues internal constructor(store: KeyValueStore) : SignalStoreVal
const val USE_NEW_MEDIA_ACTIVITY: String = "internal.use_new_media_activity"
const val ANR_DETECTION_CRASH: String = "internal.anr_detection_crash"
const val ISSUE_NOTIFICATION_PRIORITY: String = "internal.issue_notification_priority"
const val ISSUE_NOTIFY_TIMES: String = "internal.issue_notify_times"
}
public override fun onFirstEverAppLaunch() = Unit
@@ -225,6 +227,9 @@ class InternalValues internal constructor(store: KeyValueStore) : SignalStoreVal
get() = IssuePriority.fromValue(getInteger(ISSUE_NOTIFICATION_PRIORITY, IssuePriority.HIGH.value))
set(value) = putInteger(ISSUE_NOTIFICATION_PRIORITY, value.value)
/** Persisted so an issue's notification cooldown isn't reset by process death. */
var issueNotifyTimes: IssueNotifyTimes by protoValue(ISSUE_NOTIFY_TIMES, IssueNotifyTimes(), IssueNotifyTimes.ADAPTER)
var showArchiveStateHint by booleanValue(SHOW_ARCHIVE_STATE_HINT, false).defaultForExternalUsers()
/** Whether or not we should include a debuglog in the backup debug info when generating a backup. */
@@ -3,6 +3,7 @@ package org.thoughtcrime.securesms.keyvalue;
import androidx.annotation.AnyThread;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import androidx.annotation.VisibleForTesting;
import androidx.annotation.WorkerThread;
import org.signal.core.util.ThreadUtil;
@@ -14,7 +15,7 @@ import java.util.Collection;
import java.util.HashSet;
import java.util.Set;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executor;
/**
* An replacement for {@link android.content.SharedPreferences} that stores key-value pairs in our
@@ -30,13 +31,18 @@ public final class KeyValueStore implements KeyValueReader {
private static final String TAG = Log.tag(KeyValueStore.class);
private final ExecutorService executor;
private final Executor executor;
private final KeyValuePersistentStorage storage;
private KeyValueDataSet dataSet;
public KeyValueStore(@NonNull KeyValuePersistentStorage storage) {
this.executor = SignalExecutors.newCachedSingleThreadExecutor("signal-KeyValueStore", ThreadUtil.PRIORITY_BACKGROUND_THREAD);
this(storage, SignalExecutors.newCachedSingleThreadExecutor("signal-KeyValueStore", ThreadUtil.PRIORITY_BACKGROUND_THREAD));
}
@VisibleForTesting
public KeyValueStore(@NonNull KeyValuePersistentStorage storage, @NonNull Executor executor) {
this.executor = executor;
this.storage = storage;
}
@@ -2,6 +2,7 @@ package org.thoughtcrime.securesms.keyvalue
import org.signal.core.models.storageservice.StorageKey
import org.signal.core.util.logging.Log
import org.thoughtcrime.securesms.keyvalue.protos.StorageSyncLoopState
import org.whispersystems.signalservice.api.storage.SignalStorageManifest
class StorageServiceValues internal constructor(store: KeyValueStore) : SignalStoreValues(store) {
@@ -11,6 +12,7 @@ class StorageServiceValues internal constructor(store: KeyValueStore) : SignalSt
private const val LAST_SYNC_TIME = "storage.last_sync_time"
private const val NEEDS_ACCOUNT_RESTORE = "storage.needs_account_restore"
private const val MANIFEST = "storage.manifest"
private const val SYNC_LOOP_STATE = "storage.sync_loop_state"
}
public override fun onFirstEverAppLaunch() = Unit
@@ -26,6 +28,9 @@ class StorageServiceValues internal constructor(store: KeyValueStore) : SignalSt
var needsAccountRestore: Boolean by booleanValue(NEEDS_ACCOUNT_RESTORE, false)
/** Owned by [org.thoughtcrime.securesms.storage.StorageSyncLoopDetector] */
var syncLoopState: StorageSyncLoopState by protoValue(SYNC_LOOP_STATE, StorageSyncLoopState(), StorageSyncLoopState.ADAPTER)
var manifest: SignalStorageManifest
get() {
val data = getBlob(MANIFEST, null)
@@ -0,0 +1,171 @@
/*
* Copyright 2026 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.thoughtcrime.securesms.storage
import org.thoughtcrime.securesms.keyvalue.SignalStore
import org.thoughtcrime.securesms.storage.StorageSyncHelper.WriteOperationResult
import org.thoughtcrime.securesms.util.LeakyBucket
import kotlin.time.Duration
import kotlin.time.Duration.Companion.hours
import kotlin.time.Duration.Companion.milliseconds
import kotlin.time.Duration.Companion.minutes
/**
* Throttles storage service writes when another device keeps undoing them. Left alone, the two devices ping-pong
* manifest versions indefinitely, which burns a large amount of battery and network.
*
* Declining the write is what actually breaks the loop: we only ask other devices to sync after a successful write, so
* not writing stops us from prodding the other device, which stops it from prodding us back.
*
* Two [LeakyBucket]s, both persisted so a loop can't shake them off by killing the process:
*
* - The content bucket is only charged when a write repeats one of the last [FINGERPRINT_HISTORY] payload fingerprints
* on a run that fetched the remote manifest. Novel content is always free, and so is a run that reused the local
* manifest, because a loop only ever writes in reaction to a peer's manifest. Storage ids and deletes are left out of the
* fingerprint because they rotate on every write, loop or not.
* - The rate bucket is charged for every write on a run that fetched the remote manifest, regardless of content. It is
* much larger, and exists to bound loops the content bucket can't see -- ones whose payloads aren't stable.
*/
object StorageSyncLoopDetector {
/** How many past payload fingerprints a write is compared against. */
private const val FINGERPRINT_HISTORY = 3
private val contentBucket = LeakyBucket(
capacity = 3,
dripInterval = 1.hours,
state = ContentBucketState
)
private val rateBucket = LeakyBucket(
capacity = 100,
dripInterval = 10.minutes,
state = RateBucketState
)
/**
* Charges a write against whichever buckets apply and reports whether it may proceed. Call once per write attempt:
* levels rise here, not on success, so a write that fails still costs what it would have.
*
* Job retries are exempt entirely, which keeps that cost at once per failure rather than once per attempt.
*/
@Synchronized
fun onWriteAttempt(write: WriteOperationResult, fetchedRemoteManifest: Boolean, isRetry: Boolean, now: Duration = System.currentTimeMillis().milliseconds): Decision {
if (!SignalStore.account.isMultiDevice) {
return Decision.Allowed
}
if (isRetry) {
return Decision.Allowed
}
val fingerprint = fingerprint(write)
val chargeContent = fetchedRemoteManifest && fingerprint != null && SignalStore.storageService.syncLoopState.recentFingerprints.contains(fingerprint)
if (chargeContent && !contentBucket.hasRoom(now)) {
return Decision.Denied(Cause.REPEATED_PAYLOAD, contentBucket.level(now))
}
if (fetchedRemoteManifest && !rateBucket.hasRoom(now)) {
return Decision.Denied(Cause.WRITE_RATE, rateBucket.level(now))
}
if (chargeContent) {
contentBucket.use(now)
}
if (fetchedRemoteManifest) {
rateBucket.use(now)
}
if (fingerprint != null) {
remember(fingerprint)
}
return Decision.Allowed
}
/**
* Refunds a write that likely never landed.
*/
@Synchronized
fun onWriteFailed(now: Duration = System.currentTimeMillis().milliseconds) {
contentBucket.refund(now)
rateBucket.refund(now)
}
/**
* Called when a sync had nothing to write, meaning we agree with the remote state. Only the content bucket clears:
* agreement says the payload disagreement resolved, but says nothing about the write volume the rate bucket tracks.
*/
@Synchronized
fun onConverged() {
contentBucket.clear()
}
private fun remember(fingerprint: Int) {
val state = SignalStore.storageService.syncLoopState
SignalStore.storageService.syncLoopState = state.copy(
recentFingerprints = (listOf(fingerprint) + state.recentFingerprints).distinct().take(FINGERPRINT_HISTORY)
)
}
/** Null when there's nothing content-addressable to compare, i.e. a write of nothing but deletes. */
private fun fingerprint(write: WriteOperationResult): Int? {
if (write.inserts.isEmpty()) {
return null
}
return write.inserts
.map { it.proto.encode().contentHashCode() }
.sorted()
.hashCode()
}
private object ContentBucketState : LeakyBucket.State {
override val level: Int
get() = SignalStore.storageService.syncLoopState.contentLevel
override val levelUpdatedAt: Long
get() = SignalStore.storageService.syncLoopState.contentLevelAsOf
override fun update(level: Int, levelAsOf: Long) {
SignalStore.storageService.syncLoopState = SignalStore.storageService.syncLoopState.copy(
contentLevel = level,
contentLevelAsOf = levelAsOf
)
}
}
private object RateBucketState : LeakyBucket.State {
override val level: Int
get() = SignalStore.storageService.syncLoopState.rateLevel
override val levelUpdatedAt: Long
get() = SignalStore.storageService.syncLoopState.rateLevelAsOf
override fun update(level: Int, levelAsOf: Long) {
SignalStore.storageService.syncLoopState = SignalStore.storageService.syncLoopState.copy(
rateLevel = level,
rateLevelAsOf = levelAsOf
)
}
}
enum class Cause {
REPEATED_PAYLOAD,
WRITE_RATE
}
sealed interface Decision {
/** The write should proceed as normal. */
data object Allowed : Decision
/** The write found no room in a bucket and should be skipped. */
data class Denied(val cause: Cause, val level: Int) : Decision
}
}
@@ -0,0 +1,92 @@
/*
* Copyright 2026 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.thoughtcrime.securesms.util
import kotlin.time.Duration
import kotlin.time.Duration.Companion.milliseconds
/**
* A leaky bucket, in the same spirit as [LeakyBucketLimiter]: each use raises the level, and the level drips back down
* one per [dripInterval]. Once the level reaches [capacity] there is no room until enough has dripped away.
*
* Not thread safe. [hasRoom] and [use] are separate calls so a caller can check several buckets before charging any of
* them, so callers need to serialize access themselves anyway.
*/
class LeakyBucket(
private val capacity: Int,
private val dripInterval: Duration,
private val state: State
) {
/** The level after everything that has dripped away is credited, without recording that drip. */
fun level(now: Duration = System.currentTimeMillis().milliseconds): Int {
return calculateStateForCurrentTime(now).level
}
fun hasRoom(now: Duration = System.currentTimeMillis().milliseconds): Boolean {
return level(now) < capacity
}
/** Raises the level by one, recording any drip along the way. Only call when [hasRoom]. */
fun use(now: Duration = System.currentTimeMillis().milliseconds) {
val currentState = calculateStateForCurrentTime(now)
state.update(currentState.level + 1, currentState.levelUpdatedAt)
}
/** Lowers the level by one, recording any drip along the way. Pairs with [use] for work that ended up not happening. */
fun refund(now: Duration = System.currentTimeMillis().milliseconds) {
val currentState = calculateStateForCurrentTime(now)
state.update((currentState.level - 1).coerceAtLeast(0), currentState.levelUpdatedAt)
}
fun clear() {
state.update(0, 0)
}
/**
* [Snapshot.levelUpdatedAt] advances by whole drip intervals only, so a partially elapsed one carries toward the next drip
* rather than being discarded.
*/
private fun calculateStateForCurrentTime(now: Duration): Snapshot {
val level = state.level
val levelAsOf = state.levelUpdatedAt
val elapsed = now - levelAsOf.milliseconds
// The level only rises when there was room, so a clock behind levelAsOf would report a full bucket forever with
// nothing left to advance the timestamp that drains it. Empty it rather than wedge until the clock catches up.
if (level <= 0 || elapsed < Duration.ZERO) {
return Snapshot(0, now.inWholeMilliseconds)
}
val drips = (elapsed / dripInterval).toInt()
return Snapshot((level - drips).coerceAtLeast(0), levelAsOf + (dripInterval * drips).inWholeMilliseconds)
}
private data class Snapshot(val level: Int, val levelUpdatedAt: Long)
interface State {
val level: Int
val levelUpdatedAt: Long
fun update(level: Int, levelAsOf: Long)
}
class InMemoryState : State {
override var level: Int = 0
private set
override var levelUpdatedAt: Long = 0
private set
override fun update(level: Int, levelAsOf: Long) {
this.level = level
this.levelUpdatedAt = levelAsOf
}
}
}
+13 -1
View File
@@ -103,4 +103,16 @@ message BackupDownloadNotifierState {
uint64 lastSheetDisplaySeconds = 2;
uint64 intervalSeconds = 3;
Type type = 4;
}
}
message IssueNotifyTimes {
map<string, uint64> lastNotifyTimeByName = 1;
}
message StorageSyncLoopState {
repeated sint32 recentFingerprints = 1;
uint32 contentLevel = 2;
uint64 contentLevelAsOf = 3;
uint32 rateLevel = 4;
uint64 rateLevelAsOf = 5;
}
@@ -7,9 +7,12 @@ package org.thoughtcrime.securesms.jobs
import android.app.Application
import androidx.test.core.app.ApplicationProvider
import io.mockk.CapturingSlot
import io.mockk.every
import io.mockk.mockkObject
import io.mockk.slot
import io.mockk.unmockkObject
import io.mockk.verify
import okio.ByteString.Companion.toByteString
import org.junit.After
import org.junit.Assert.assertArrayEquals
@@ -31,8 +34,10 @@ import org.signal.core.util.Util
import org.signal.core.util.logging.Log
import org.signal.core.util.update
import org.signal.core.util.withinTransaction
import org.thoughtcrime.securesms.database.IssueReporter
import org.thoughtcrime.securesms.database.RecipientTable
import org.thoughtcrime.securesms.database.SignalDatabase
import org.thoughtcrime.securesms.database.model.IssuePriority
import org.thoughtcrime.securesms.database.model.StickerPackId
import org.thoughtcrime.securesms.groups.GroupId
import org.thoughtcrime.securesms.jobmanager.Job
@@ -91,6 +96,7 @@ class StorageSyncJobTest {
@After
fun tearDown() {
unmockkObject(RemoteConfig)
unmockkObject(IssueReporter)
}
@Test
@@ -384,6 +390,79 @@ class StorageSyncJobTest {
assertEquals(0, remoteStorage.records.count { it.proto.contact != null })
}
@Test
fun `given another device keeps undoing my write, when I run again, then I stop writing`() {
stubIssueReporter()
val throttledRun = loopWritesUntilThrottled()
assertTrue(throttledRun.isSuccess)
assertEquals(0, remoteStorage.writeCount)
assertEquals(0, remoteStorage.records.count { it.proto.contact?.givenName == "Loop" })
}
@Test
fun `given another device keeps undoing my write, when I throttle it, then I report it at high priority`() {
val priority = stubIssueReporter()
loopWritesUntilThrottled()
verify { IssueReporter.report(any(), any(), any(), any(), any(), any()) }
assertEquals(IssuePriority.HIGH, priority.captured)
}
@Test
fun `given a local-only contact the other device leaves alone, when I run repeatedly, then I keep writing`() {
stubIssueReporter()
every { recipients.signalStore.account.isMultiDevice } returns true
SignalDatabase.recipients.rotateStorageId(recipients.createRecipient("Local Contact"))
check(runJob(StorageSyncJob.forLocalChange()).isSuccess)
repeat(4) {
SignalDatabase.recipients.rotateStorageId(recipients.createRecipient("Local Contact ${it + 2}"))
remoteStorage.resetCounters()
assertTrue(runJob(StorageSyncJob.forLocalChange()).isSuccess)
assertEquals(1, remoteStorage.writeCount)
}
}
/**
* Writes the same contact, with the other device deleting it again after each, until a run gets throttled. Returns
* that run, leaving [FakeStorageServiceRule.writeCount] counting only it. Driven by observation rather than a fixed
* count because the record's payload settles after its first sync, so the number of passes isn't fixed.
*/
private fun loopWritesUntilThrottled(): Job.Result {
every { recipients.signalStore.account.isMultiDevice } returns true
SignalDatabase.recipients.rotateStorageId(recipients.createRecipient("Loop Contact"))
repeat(12) { pass ->
remoteStorage.resetCounters()
val result = runJob(StorageSyncJob.forRemoteChange())
check(result.isSuccess) { "Loop write ${pass + 1} failed!" }
if (remoteStorage.writeCount == 0) {
return result
}
val withoutLoopContact = remoteStorage.records.filterNot { record -> record.proto.contact?.givenName == "Loop" }
remoteStorage.setRemoteState(withoutLoopContact, version = remoteStorage.manifest!!.version + 1)
}
throw AssertionError("Writes were never throttled!")
}
private fun stubIssueReporter(): CapturingSlot<IssuePriority> {
val priority = slot<IssuePriority>()
mockkObject(IssueReporter)
every { IssueReporter.report(any(), any(), any(), capture(priority), any(), any()) } returns Unit
return priority
}
/**
* Gets us to a steady state: remote holds our account record at version 1, then a sync pushes up everything else
* the fresh database came with (the default chat folder), leaving both sides at [BASE_MANIFEST_VERSION].
@@ -0,0 +1,436 @@
/*
* Copyright 2026 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.thoughtcrime.securesms.storage
import android.app.Application
import org.junit.Assert.assertEquals
import org.junit.Before
import org.junit.Rule
import org.junit.Test
import org.junit.runner.RunWith
import org.robolectric.RobolectricTestRunner
import org.robolectric.annotation.Config
import org.signal.core.util.Util
import org.thoughtcrime.securesms.keyvalue.SignalStore
import org.thoughtcrime.securesms.keyvalue.protos.StorageSyncLoopState
import org.thoughtcrime.securesms.storage.StorageSyncHelper.WriteOperationResult
import org.thoughtcrime.securesms.testutil.SignalStoreRule
import org.thoughtcrime.securesms.testutil.TestHelpers
import org.whispersystems.signalservice.api.storage.SignalStorageManifest
import org.whispersystems.signalservice.api.storage.SignalStorageRecord
import org.whispersystems.signalservice.api.storage.StorageId
import org.whispersystems.signalservice.internal.storage.protos.ContactRecord
import org.whispersystems.signalservice.internal.storage.protos.StorageRecord
import kotlin.time.Duration
import kotlin.time.Duration.Companion.hours
import kotlin.time.Duration.Companion.milliseconds
import kotlin.time.Duration.Companion.minutes
@RunWith(RobolectricTestRunner::class)
@Config(manifest = Config.NONE, application = Application::class)
class StorageSyncLoopDetectorTest {
companion object {
private val NOW = 1_700_000_000_000L.milliseconds
}
@get:Rule
val signalStore = SignalStoreRule()
@Before
fun setup() {
SignalStore.storageService.syncLoopState = StorageSyncLoopState()
// Throttling only applies when another device could be undoing our writes.
SignalStore.account.isMultiDevice = true
}
@Test
fun `given no linked devices, when I repeat the same write, then I expect it to always be allowed`() {
SignalStore.account.isMultiDevice = false
repeat(20) {
assertEquals(StorageSyncLoopDetector.Decision.Allowed, attempt(writeOf(1)))
}
}
@Test
fun `given no linked devices, when I repeat the same write, then I expect nothing to be charged`() {
SignalStore.account.isMultiDevice = false
repeat(20) { attempt(writeOf(1)) }
assertEquals(0, SignalStore.storageService.syncLoopState.contentLevel)
assertEquals(0, SignalStore.storageService.syncLoopState.rateLevel)
}
@Test
fun `given a write that failed, when I refund it, then I expect the charge to come back`() {
attempt(writeOf(1))
attempt(writeOf(1))
val chargedLevel = SignalStore.storageService.syncLoopState.contentLevel
StorageSyncLoopDetector.onWriteFailed(NOW)
assertEquals(chargedLevel - 1, SignalStore.storageService.syncLoopState.contentLevel)
}
@Test
fun `given writes that keep failing, when I retry forever, then I expect to never be throttled`() {
repeat(20) {
assertEquals(StorageSyncLoopDetector.Decision.Allowed, attempt(writeOf(1)))
StorageSyncLoopDetector.onWriteFailed(NOW)
}
assertEquals(0, SignalStore.storageService.syncLoopState.contentLevel)
assertEquals(0, SignalStore.storageService.syncLoopState.rateLevel)
}
@Test
fun `given nothing charged, when I refund, then I expect the level to stay at zero`() {
StorageSyncLoopDetector.onWriteFailed(NOW)
assertEquals(0, SignalStore.storageService.syncLoopState.contentLevel)
assertEquals(0, SignalStore.storageService.syncLoopState.rateLevel)
}
@Test
fun `allows a write nothing like the ones before it`() {
assertEquals(StorageSyncLoopDetector.Decision.Allowed, attempt(writeOf(1)))
}
@Test
fun `never charges the content bucket for novel payloads`() {
repeat(50) { index ->
assertEquals(StorageSyncLoopDetector.Decision.Allowed, attempt(writeOf(index), fetchedRemoteManifest = false))
}
assertEquals(0, SignalStore.storageService.syncLoopState.contentLevel)
}
@Test
fun `charges the content bucket for a repeated payload and denies once it is dry`() {
assertEquals(StorageSyncLoopDetector.Decision.Allowed, attempt(writeOf(1)))
repeat(3) {
assertEquals(StorageSyncLoopDetector.Decision.Allowed, attempt(writeOf(1)))
}
assertEquals(
StorageSyncLoopDetector.Decision.Denied(StorageSyncLoopDetector.Cause.REPEATED_PAYLOAD, 3),
attempt(writeOf(1))
)
}
@Test
fun `compares against the last three payloads, not just the previous one`() {
attempt(writeOf(1))
attempt(writeOf(2))
attempt(writeOf(3))
repeat(3) {
assertEquals(StorageSyncLoopDetector.Decision.Allowed, attempt(writeOf(1)))
}
assertEquals(StorageSyncLoopDetector.Cause.REPEATED_PAYLOAD, denialCause(attempt(writeOf(1))))
}
@Test
fun `forgets a payload once three newer ones have been written`() {
attempt(writeOf(1))
attempt(writeOf(2))
attempt(writeOf(3))
attempt(writeOf(4))
assertEquals(StorageSyncLoopDetector.Decision.Allowed, attempt(writeOf(1)))
assertEquals(0, SignalStore.storageService.syncLoopState.contentLevel)
}
@Test
fun `restores one content permit per hour`() {
exhaustContentBucket()
assertEquals(StorageSyncLoopDetector.Decision.Allowed, attempt(writeOf(1), now = NOW + 1.hours))
assertEquals(StorageSyncLoopDetector.Cause.REPEATED_PAYLOAD, denialCause(attempt(writeOf(1), now = NOW + 1.hours)))
}
@Test
fun `restores several content permits at once when enough time has passed`() {
exhaustContentBucket()
repeat(3) {
assertEquals(StorageSyncLoopDetector.Decision.Allowed, attempt(writeOf(1), now = NOW + 3.hours))
}
assertEquals(StorageSyncLoopDetector.Cause.REPEATED_PAYLOAD, denialCause(attempt(writeOf(1), now = NOW + 3.hours)))
}
@Test
fun `never restores more permits than the bucket holds`() {
exhaustContentBucket()
repeat(3) {
assertEquals(StorageSyncLoopDetector.Decision.Allowed, attempt(writeOf(1), now = NOW + 500.hours))
}
assertEquals(StorageSyncLoopDetector.Cause.REPEATED_PAYLOAD, denialCause(attempt(writeOf(1), now = NOW + 500.hours)))
}
@Test
fun `refills rather than wedging when the clock moves backwards`() {
exhaustContentBucket()
assertEquals(StorageSyncLoopDetector.Decision.Allowed, attempt(writeOf(1), now = NOW - 5.hours))
}
/** The rate bucket has no convergence reset, so a wedge here would deny manifest-fetching writes indefinitely. */
@Test
fun `refills the rate bucket when the clock moves backwards`() {
exhaustRateBucket()
assertEquals(StorageSyncLoopDetector.Decision.Allowed, attempt(writeOf(100), now = NOW - 5.hours))
}
@Test
fun `recovers normal throttling after a backwards clock write`() {
exhaustContentBucket()
val past = NOW - 5.hours
repeat(3) {
assertEquals(StorageSyncLoopDetector.Decision.Allowed, attempt(writeOf(1), now = past))
}
assertEquals(StorageSyncLoopDetector.Cause.REPEATED_PAYLOAD, denialCause(attempt(writeOf(1), now = past)))
}
/** A partially elapsed interval carries toward the next drip instead of restarting, unlike a strike counter. */
@Test
fun `keeps a partially elapsed interval toward the next drip`() {
exhaustContentBucket()
assertEquals(StorageSyncLoopDetector.Decision.Allowed, attempt(writeOf(1), now = NOW + 1.hours + 59.minutes))
assertEquals(StorageSyncLoopDetector.Decision.Allowed, attempt(writeOf(1), now = NOW + 2.hours + 1.minutes))
}
/** Twice the drip rate accumulates a level of one per hour, which a strike counter reset by any gap would not. */
@Test
fun `fills the bucket when writes outpace the drip rate`() {
repeat(6) { index ->
assertEquals(StorageSyncLoopDetector.Decision.Allowed, attempt(writeOf(1), now = NOW + (index * 30).minutes))
}
assertEquals(StorageSyncLoopDetector.Cause.REPEATED_PAYLOAD, denialCause(attempt(writeOf(1), now = NOW + 180.minutes)))
}
@Test
fun `does not credit the same elapsed time twice`() {
exhaustContentBucket()
repeat(4) {
assertEquals(StorageSyncLoopDetector.Cause.REPEATED_PAYLOAD, denialCause(attempt(writeOf(1), now = NOW + 59.minutes)))
}
}
@Test
fun `never charges the content bucket for a delete-only write`() {
repeat(30) {
assertEquals(StorageSyncLoopDetector.Decision.Allowed, attempt(writeOf(deletes = intArrayOf(1)), fetchedRemoteManifest = false))
}
assertEquals(0, SignalStore.storageService.syncLoopState.contentLevel)
}
@Test
fun `never charges the content bucket for a repeated payload when the local manifest was reused`() {
repeat(30) {
assertEquals(StorageSyncLoopDetector.Decision.Allowed, attempt(writeOf(1), fetchedRemoteManifest = false))
}
assertEquals(0, SignalStore.storageService.syncLoopState.contentLevel)
}
/** Toggling a setting back and forth reproduces a previous payload, but it is a genuine local change rather than a loop. */
@Test
fun `never throttles a setting toggled back and forth`() {
repeat(20) { index ->
assertEquals(StorageSyncLoopDetector.Decision.Allowed, attempt(writeOf(index % 2), fetchedRemoteManifest = false))
}
assertEquals(0, SignalStore.storageService.syncLoopState.contentLevel)
}
/** Local writes still populate the history, so a peer that starts undoing them is caught on the first repeat we see. */
@Test
fun `builds fingerprint history from writes that reused the local manifest`() {
repeat(4) {
attempt(writeOf(1), fetchedRemoteManifest = false)
}
repeat(3) {
assertEquals(StorageSyncLoopDetector.Decision.Allowed, attempt(writeOf(1)))
}
assertEquals(StorageSyncLoopDetector.Cause.REPEATED_PAYLOAD, denialCause(attempt(writeOf(1))))
}
@Test
fun `does not charge the rate bucket when the local manifest was reused`() {
repeat(50) { index ->
assertEquals(StorageSyncLoopDetector.Decision.Allowed, attempt(writeOf(index), fetchedRemoteManifest = false))
}
assertEquals(0, SignalStore.storageService.syncLoopState.rateLevel)
}
@Test
fun `charges the rate bucket for every write on a manifest-fetching run`() {
repeat(100) { index ->
assertEquals(StorageSyncLoopDetector.Decision.Allowed, attempt(writeOf(index)))
}
assertEquals(100, SignalStore.storageService.syncLoopState.rateLevel)
assertEquals(StorageSyncLoopDetector.Cause.WRITE_RATE, denialCause(attempt(writeOf(999))))
}
/** The rate bucket is content-blind, so a run of delete-only writes drains it just the same. */
@Test
fun `charges the rate bucket for delete-only writes too`() {
repeat(100) { index ->
assertEquals(StorageSyncLoopDetector.Decision.Allowed, attempt(writeOf(deletes = intArrayOf(index))))
}
assertEquals(StorageSyncLoopDetector.Cause.WRITE_RATE, denialCause(attempt(writeOf(deletes = intArrayOf(99)))))
}
@Test
fun `drips one rate level every ten minutes`() {
exhaustRateBucket()
assertEquals(StorageSyncLoopDetector.Decision.Allowed, attempt(writeOf(1000), now = NOW + 10.minutes))
assertEquals(StorageSyncLoopDetector.Cause.WRITE_RATE, denialCause(attempt(writeOf(1001), now = NOW + 10.minutes)))
}
@Test
fun `converging resets the content bucket`() {
exhaustContentBucket()
StorageSyncLoopDetector.onConverged()
assertEquals(0, SignalStore.storageService.syncLoopState.contentLevel)
}
@Test
fun `converging leaves the rate bucket alone`() {
exhaustRateBucket()
StorageSyncLoopDetector.onConverged()
assertEquals(100, SignalStore.storageService.syncLoopState.rateLevel)
assertEquals(StorageSyncLoopDetector.Cause.WRITE_RATE, denialCause(attempt(writeOf(999))))
}
@Test
fun `converging lets a throttled write through again`() {
exhaustContentBucket()
check(attempt(writeOf(1)) is StorageSyncLoopDetector.Decision.Denied) { "Should be throttled before converging!" }
StorageSyncLoopDetector.onConverged()
assertEquals(StorageSyncLoopDetector.Decision.Allowed, attempt(writeOf(1)))
}
@Test
fun `persists bucket state so it survives a process restart`() {
exhaustContentBucket()
assertEquals(3, SignalStore.storageService.syncLoopState.contentLevel)
assertEquals(NOW.inWholeMilliseconds, SignalStore.storageService.syncLoopState.contentLevelAsOf)
}
/** Spends every content permit, leaving [writeOf] 1 as a repeat that needs one. */
private fun exhaustContentBucket() {
repeat(4) {
check(attempt(writeOf(1)) is StorageSyncLoopDetector.Decision.Allowed) { "Content bucket drained early!" }
}
}
@Test
fun `never charges a retry`() {
repeat(50) {
assertEquals(StorageSyncLoopDetector.Decision.Allowed, attempt(writeOf(1), isRetry = true))
}
assertEquals(0, SignalStore.storageService.syncLoopState.contentLevel)
assertEquals(0, SignalStore.storageService.syncLoopState.rateLevel)
}
@Test
fun `a retry does not build up fingerprint history`() {
repeat(5) {
attempt(writeOf(1), isRetry = true)
}
assertEquals(emptyList<Int>(), SignalStore.storageService.syncLoopState.recentFingerprints)
}
@Test
fun `allows a retry of a repeated payload even when the content bucket is full`() {
repeat(4) {
attempt(writeOf(1))
}
assertEquals(StorageSyncLoopDetector.Cause.REPEATED_PAYLOAD, denialCause(attempt(writeOf(1))))
assertEquals(StorageSyncLoopDetector.Decision.Allowed, attempt(writeOf(1), isRetry = true))
}
@Test
fun `allows a retry even when the rate bucket is exhausted`() {
exhaustRateBucket()
assertEquals(StorageSyncLoopDetector.Cause.WRITE_RATE, denialCause(attempt(writeOf(500))))
assertEquals(StorageSyncLoopDetector.Decision.Allowed, attempt(writeOf(500), isRetry = true))
}
private fun exhaustRateBucket() {
repeat(100) { index ->
check(attempt(writeOf(index)) is StorageSyncLoopDetector.Decision.Allowed) { "Rate bucket drained early!" }
}
}
private fun attempt(
write: WriteOperationResult,
fetchedRemoteManifest: Boolean = true,
isRetry: Boolean = false,
now: Duration = NOW
): StorageSyncLoopDetector.Decision {
return StorageSyncLoopDetector.onWriteAttempt(write, fetchedRemoteManifest, isRetry, now)
}
private fun denial(decision: StorageSyncLoopDetector.Decision): StorageSyncLoopDetector.Decision.Denied {
return decision as? StorageSyncLoopDetector.Decision.Denied ?: throw AssertionError("Expected a denial, got $decision")
}
private fun denialCause(decision: StorageSyncLoopDetector.Decision): StorageSyncLoopDetector.Cause {
return denial(decision).cause
}
private fun writeOf(vararg payloads: Int, deletes: IntArray = IntArray(0)): WriteOperationResult {
return WriteOperationResult(
manifest = SignalStorageManifest.EMPTY,
inserts = payloads.map { contactRecord(it) },
deletes = deletes.map { TestHelpers.byteArray(it) }
)
}
/** One distinct payload per [payload], under a storage id that rotates on every call the way a real write does. */
private fun contactRecord(payload: Int): SignalStorageRecord {
return SignalStorageRecord(
id = StorageId.forContact(Util.getSecretBytes(16)),
proto = StorageRecord(contact = ContactRecord(e164 = "+1555000$payload"))
)
}
}
@@ -19,6 +19,7 @@ import org.signal.network.service.StorageServiceService
import org.thoughtcrime.securesms.keyvalue.PhoneNumberPrivacyValues.PhoneNumberDiscoverabilityMode
import org.thoughtcrime.securesms.keyvalue.PhoneNumberPrivacyValues.PhoneNumberSharingMode
import org.thoughtcrime.securesms.keyvalue.SignalStore
import org.thoughtcrime.securesms.keyvalue.protos.StorageSyncLoopState
import org.thoughtcrime.securesms.net.SignalNetwork
import org.thoughtcrime.securesms.util.RemoteConfig
import org.whispersystems.signalservice.api.storage.RecordIkm
@@ -108,12 +109,16 @@ class FakeStorageServiceRule(val storageKey: StorageKey = StorageKey(Util.getSec
*/
fun stubDefaults(store: MockSignalStoreRule) {
var localManifest = SignalStorageManifest.EMPTY
var syncLoopState = StorageSyncLoopState()
every { store.storageService.manifest } answers { localManifest }
every { store.storageService.manifest = any() } answers { localManifest = firstArg() }
every { store.storageService.storageKey } returns storageKey
every { store.storageService.storageKeyForInitialDataRestore } returns null
every { store.storageService.syncLoopState } answers { syncLoopState }
every { store.storageService.syncLoopState = any() } answers { syncLoopState = firstArg() }
every { store.svr.hasPin() } returns true
every { store.svr.hasOptedOut() } returns false
@@ -22,7 +22,7 @@ class SignalStoreRule : ExternalResource() {
override fun before() {
val application = ApplicationProvider.getApplicationContext<Application>()
SignalStore.testInject(SignalStore(application, KeyValueStore(MockKeyValuePersistentStorage.withDataSet(KeyValueDataSet()))))
SignalStore.testInject(SignalStore(application, KeyValueStore(MockKeyValuePersistentStorage.withDataSet(KeyValueDataSet()), DirectExecutor())))
}
override fun after() {
@@ -0,0 +1,182 @@
/*
* Copyright 2026 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.thoughtcrime.securesms.util
import org.junit.Assert.assertEquals
import org.junit.Assert.assertFalse
import org.junit.Assert.assertTrue
import org.junit.Test
import kotlin.time.Duration
import kotlin.time.Duration.Companion.hours
import kotlin.time.Duration.Companion.milliseconds
import kotlin.time.Duration.Companion.minutes
class LeakyBucketTest {
companion object {
private val NOW = 1_700_000_000_000L.milliseconds
private val DRIP = 15.minutes
}
private val state = LeakyBucket.InMemoryState()
private val bucket = LeakyBucket(capacity = 3, dripInterval = DRIP, state = state)
@Test
fun `starts empty`() {
assertEquals(0, bucket.level(NOW))
assertTrue(bucket.hasRoom(NOW))
}
@Test
fun `fills to capacity and then has no room`() {
repeat(3) {
assertTrue(bucket.hasRoom(NOW))
bucket.use(NOW)
}
assertEquals(3, bucket.level(NOW))
assertFalse(bucket.hasRoom(NOW))
}
@Test
fun `drips one level per interval`() {
fill()
assertEquals(2, bucket.level(NOW + DRIP))
assertTrue(bucket.hasRoom(NOW + DRIP))
}
@Test
fun `drips several levels at once when enough time has passed`() {
fill()
assertEquals(1, bucket.level(NOW + DRIP * 2))
assertEquals(0, bucket.level(NOW + DRIP * 3))
}
@Test
fun `never drips below empty`() {
fill()
assertEquals(0, bucket.level(NOW + DRIP * 500))
}
@Test
fun `does not drip for a partially elapsed interval`() {
fill()
assertEquals(3, bucket.level(NOW + DRIP - 1.milliseconds))
}
@Test
fun `carries a partially elapsed interval toward the next drip`() {
fill()
// A drip and most of a second one. Using the bucket banks the drip and keeps the remainder.
bucket.use(NOW + DRIP + 14.minutes)
assertEquals(NOW.inWholeMilliseconds + DRIP.inWholeMilliseconds, state.levelUpdatedAt)
assertEquals(3, state.level)
// One more minute crosses the second interval, rather than restarting from the last use.
assertEquals(2, bucket.level(NOW + DRIP + 15.minutes))
}
@Test
fun `reading the level does not record the drip`() {
fill()
bucket.level(NOW + DRIP)
assertEquals(3, state.level)
assertEquals(NOW.inWholeMilliseconds, state.levelUpdatedAt)
}
@Test
fun `using the bucket records the drip`() {
fill()
bucket.use(NOW + DRIP)
assertEquals(3, state.level)
assertEquals(NOW.inWholeMilliseconds + DRIP.inWholeMilliseconds, state.levelUpdatedAt)
}
@Test
fun `empties rather than wedging when the clock moves backwards`() {
fill()
assertEquals(0, bucket.level(NOW - 5.hours))
assertTrue(bucket.hasRoom(NOW - 5.hours))
}
@Test
fun `throttles again after a backwards clock use`() {
fill()
val past = NOW - 5.hours
repeat(3) {
assertTrue(bucket.hasRoom(past))
bucket.use(past)
}
assertFalse(bucket.hasRoom(past))
}
@Test
fun `clear empties the bucket`() {
fill()
bucket.clear()
assertEquals(0, bucket.level(NOW))
assertEquals(0, state.level)
}
@Test
fun `fills when uses outpace the drip rate`() {
var now = NOW
// Twice the drip rate, so the level nets up one per interval and a capacity of three fills on the fifth use.
repeat(5) {
assertTrue(bucket.hasRoom(now))
bucket.use(now)
now += DRIP / 2
}
assertFalse(bucket.hasRoom(now))
}
@Test
fun `never fills when uses match the drip rate`() {
var now = NOW
repeat(50) {
assertTrue(bucket.hasRoom(now))
bucket.use(now)
now += DRIP
}
}
@Test
fun `reads its level from the state it was given`() {
state.update(level = 2, levelAsOf = NOW.inWholeMilliseconds)
assertEquals(2, bucket.level(NOW))
assertTrue(bucket.hasRoom(NOW))
bucket.use(NOW)
assertFalse(bucket.hasRoom(NOW))
}
private fun fill(now: Duration = NOW) {
repeat(3) {
bucket.use(now)
}
}
}