Add a pessimistic locking system for operations on recently-deleted account records

This commit is contained in:
Jon Chambers
2021-07-14 16:58:51 -04:00
committed by Jon Chambers
parent b757d4b334
commit 32a95f96ff
16 changed files with 487 additions and 53 deletions

View File

@@ -148,7 +148,7 @@ class AccountsManagerConcurrentModificationIntegrationTest {
accounts,
accountsDynamoDb,
RedisClusterHelper.buildMockRedisCluster(commands),
mock(DeletedAccounts.class),
mock(DeletedAccountsManager.class),
mock(DirectoryQueue.class),
mock(KeysDynamoDb.class),
mock(MessagesManager.class),

View File

@@ -0,0 +1,142 @@
/*
* Copyright 2013-2021 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.storage;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.RegisterExtension;
import org.junit.jupiter.api.function.Executable;
import software.amazon.awssdk.services.dynamodb.model.AttributeDefinition;
import software.amazon.awssdk.services.dynamodb.model.GlobalSecondaryIndex;
import software.amazon.awssdk.services.dynamodb.model.KeySchemaElement;
import software.amazon.awssdk.services.dynamodb.model.KeyType;
import software.amazon.awssdk.services.dynamodb.model.Projection;
import software.amazon.awssdk.services.dynamodb.model.ProjectionType;
import software.amazon.awssdk.services.dynamodb.model.ProvisionedThroughput;
import software.amazon.awssdk.services.dynamodb.model.ScalarAttributeType;
import java.lang.Thread.State;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertEquals;
class DeletedAccountsManagerTest {
private static final String NEEDS_RECONCILIATION_INDEX_NAME = "needs_reconciliation_test";
@RegisterExtension
static final DynamoDbExtension DELETED_ACCOUNTS_DYNAMODB_EXTENSION = DynamoDbExtension.builder()
.tableName("deleted_accounts_test")
.hashKey(DeletedAccounts.KEY_ACCOUNT_E164)
.attributeDefinition(AttributeDefinition.builder()
.attributeName(DeletedAccounts.KEY_ACCOUNT_E164)
.attributeType(ScalarAttributeType.S).build())
.attributeDefinition(AttributeDefinition.builder()
.attributeName(DeletedAccounts.ATTR_NEEDS_CDS_RECONCILIATION)
.attributeType(ScalarAttributeType.N)
.build())
.globalSecondaryIndex(GlobalSecondaryIndex.builder()
.indexName(NEEDS_RECONCILIATION_INDEX_NAME)
.keySchema(KeySchemaElement.builder().attributeName(DeletedAccounts.KEY_ACCOUNT_E164).keyType(KeyType.HASH).build(),
KeySchemaElement.builder().attributeName(DeletedAccounts.ATTR_NEEDS_CDS_RECONCILIATION).keyType(KeyType.RANGE).build())
.projection(Projection.builder().projectionType(ProjectionType.INCLUDE).nonKeyAttributes(DeletedAccounts.ATTR_ACCOUNT_UUID).build())
.provisionedThroughput(ProvisionedThroughput.builder().readCapacityUnits(10L).writeCapacityUnits(10L).build())
.build())
.build();
@RegisterExtension
static DynamoDbExtension DELETED_ACCOUNTS_LOCK_DYNAMODB_EXTENSION = DynamoDbExtension.builder()
.tableName("deleted_accounts_lock_test")
.hashKey(DeletedAccounts.KEY_ACCOUNT_E164)
.attributeDefinition(AttributeDefinition.builder()
.attributeName(DeletedAccounts.KEY_ACCOUNT_E164)
.attributeType(ScalarAttributeType.S).build())
.build();
private DeletedAccountsManager deletedAccountsManager;
@BeforeEach
void setUp() {
final DeletedAccounts deletedAccounts = new DeletedAccounts(DELETED_ACCOUNTS_DYNAMODB_EXTENSION.getDynamoDbClient(),
DELETED_ACCOUNTS_DYNAMODB_EXTENSION.getTableName(),
NEEDS_RECONCILIATION_INDEX_NAME);
deletedAccountsManager = new DeletedAccountsManager(deletedAccounts,
DELETED_ACCOUNTS_LOCK_DYNAMODB_EXTENSION.getLegacyDynamoClient(),
DELETED_ACCOUNTS_LOCK_DYNAMODB_EXTENSION.getTableName());
}
@Test
void testReconciliationLockContention() throws ChunkProcessingFailedException, InterruptedException {
final UUID[] uuids = new UUID[3];
final String[] e164s = new String[uuids.length];
for (int i = 0; i < uuids.length; i++) {
uuids[i] = UUID.randomUUID();
e164s[i] = String.format("+1800555%04d", i);
}
final Map<String, UUID> expectedReconciledAccounts = new HashMap<>();
for (int i = 0; i < uuids.length; i++) {
deletedAccountsManager.put(uuids[i], e164s[i]);
expectedReconciledAccounts.put(e164s[i], uuids[i]);
}
final UUID replacedUUID = UUID.randomUUID();
final Map<String, UUID> reconciledAccounts = new HashMap<>();
final Thread putThread = new Thread(() -> {
try {
deletedAccountsManager.put(replacedUUID, e164s[0]);
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
},
getClass().getSimpleName() + "-put");
final Thread reconcileThread = new Thread(() -> {
try {
deletedAccountsManager.lockAndReconcileAccounts(uuids.length, deletedAccounts -> {
// We hold the lock for the first account, so a thread trying to operate on that first count should block
// waiting for the lock.
putThread.start();
// Make sure the other thread really does actually block at some point
while (putThread.getState() != State.TIMED_WAITING) {
Thread.yield();
}
deletedAccounts.forEach(pair -> reconciledAccounts.put(pair.second(), pair.first()));
return reconciledAccounts.keySet();
});
} catch (ChunkProcessingFailedException e) {
throw new AssertionError(e);
}
}, getClass().getSimpleName() + "-reconcile");
reconcileThread.start();
assertDoesNotThrow((Executable) reconcileThread::join);
assertDoesNotThrow((Executable) putThread::join);
assertEquals(expectedReconciledAccounts, reconciledAccounts);
// The "put" thread should have completed after the reconciliation thread wrapped up. We can verify that's true by
// reconciling again; the updated account (and only that account) should appear in the "needs reconciliation" list.
deletedAccountsManager.lockAndReconcileAccounts(uuids.length, deletedAccounts -> {
assertEquals(1, deletedAccounts.size());
assertEquals(replacedUUID, deletedAccounts.get(0).first());
assertEquals(e164s[0], deletedAccounts.get(0).second());
return List.of(deletedAccounts.get(0).second());
});
}
}

View File

@@ -7,8 +7,12 @@ package org.whispersystems.textsecuregcm.storage;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.UUID;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.RegisterExtension;
import org.whispersystems.textsecuregcm.util.Pair;
@@ -45,13 +49,17 @@ class DeletedAccountsTest {
.build())
.build();
@Test
void test() {
private DeletedAccounts deletedAccounts;
final DeletedAccounts deletedAccounts = new DeletedAccounts(dynamoDbExtension.getDynamoDbClient(),
@BeforeEach
void setUp() {
deletedAccounts = new DeletedAccounts(dynamoDbExtension.getDynamoDbClient(),
dynamoDbExtension.getTableName(),
NEEDS_RECONCILIATION_INDEX_NAME);
}
@Test
void testPutList() {
UUID firstUuid = UUID.randomUUID();
UUID secondUuid = UUID.randomUUID();
UUID thirdUuid = UUID.randomUUID();
@@ -81,4 +89,42 @@ class DeletedAccountsTest {
assertTrue(deletedAccounts.listAccountsToReconcile(1).isEmpty());
}
@Test
void testGetAccountsNeedingReconciliation() {
final UUID firstUuid = UUID.randomUUID();
final UUID secondUuid = UUID.randomUUID();
final String firstNumber = "+14152221234";
final String secondNumber = "+14152225678";
final String thirdNumber = "+14159998765";
assertEquals(Collections.emptySet(),
deletedAccounts.getAccountsNeedingReconciliation(List.of(firstNumber, secondNumber, thirdNumber)));
deletedAccounts.put(firstUuid, firstNumber);
deletedAccounts.put(secondUuid, secondNumber);
assertEquals(Set.of(firstNumber, secondNumber),
deletedAccounts.getAccountsNeedingReconciliation(List.of(firstNumber, secondNumber, thirdNumber)));
}
@Test
void testGetAccountsNeedingReconciliationLargeBatch() {
final int itemCount = (DeletedAccounts.GET_BATCH_SIZE * 3) + 1;
final Set<String> expectedAccountsNeedingReconciliation = new HashSet<>(itemCount);
for (int i = 0; i < itemCount; i++) {
final String e164 = String.format("+18000555%04d", i);
deletedAccounts.put(UUID.randomUUID(), e164);
expectedAccountsNeedingReconciliation.add(e164);
}
final Set<String> accountsNeedingReconciliation =
deletedAccounts.getAccountsNeedingReconciliation(expectedAccountsNeedingReconciliation);
assertEquals(expectedAccountsNeedingReconciliation, accountsNeedingReconciliation);
}
}

View File

@@ -1,6 +1,13 @@
package org.whispersystems.textsecuregcm.storage;
import com.almworks.sqlite4java.SQLite;
import com.amazonaws.ClientConfiguration;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.auth.InstanceProfileCredentialsProvider;
import com.amazonaws.client.builder.AwsClientBuilder;
import com.amazonaws.services.dynamodbv2.AmazonDynamoDB;
import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClientBuilder;
import com.amazonaws.services.dynamodbv2.local.main.ServerRunner;
import com.amazonaws.services.dynamodbv2.local.server.DynamoDBProxyServer;
import java.net.ServerSocket;
@@ -46,6 +53,7 @@ public class DynamoDbExtension implements BeforeEachCallback, AfterEachCallback
private DynamoDbClient dynamoDB2;
private DynamoDbAsyncClient dynamoAsyncDB2;
private AmazonDynamoDB legacyDynamoClient;
private DynamoDbExtension(String tableName, String hashKey, String rangeKey, List<AttributeDefinition> attributeDefinitions, List<GlobalSecondaryIndex> globalSecondaryIndexes, long readCapacityUnits,
long writeCapacityUnits) {
@@ -137,6 +145,11 @@ public class DynamoDbExtension implements BeforeEachCallback, AfterEachCallback
.credentialsProvider(StaticCredentialsProvider.create(
AwsBasicCredentials.create("accessKey", "secretKey")))
.build();
legacyDynamoClient = AmazonDynamoDBClientBuilder.standard()
.withEndpointConfiguration(
new AwsClientBuilder.EndpointConfiguration("http://localhost:" + port, "local-test-region"))
.withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials("accessKey", "secretKey")))
.build();
}
static class DynamoDbExtensionBuilder {
@@ -194,6 +207,10 @@ public class DynamoDbExtension implements BeforeEachCallback, AfterEachCallback
return dynamoAsyncDB2;
}
public AmazonDynamoDB getLegacyDynamoClient() {
return legacyDynamoClient;
}
public String getTableName() {
return tableName;
}

View File

@@ -1565,7 +1565,7 @@ class AccountControllerTest {
}
@Test
void testDeleteAccount() {
void testDeleteAccount() throws InterruptedException {
Response response =
resources.getJerseyTest()
.target("/v1/accounts/me")
@@ -1577,6 +1577,21 @@ class AccountControllerTest {
verify(accountsManager).delete(AuthHelper.VALID_ACCOUNT, AccountsManager.DeletionReason.USER_REQUEST);
}
@Test
void testDeleteAccountInterrupted() throws InterruptedException {
doThrow(InterruptedException.class).when(accountsManager).delete(any(), any());
Response response =
resources.getJerseyTest()
.target("/v1/accounts/me")
.request()
.header("Authorization", AuthHelper.getAuthHeader(AuthHelper.VALID_NUMBER, AuthHelper.VALID_PASSWORD))
.delete();
assertThat(response.getStatus()).isEqualTo(500);
verify(accountsManager).delete(AuthHelper.VALID_ACCOUNT, AccountsManager.DeletionReason.USER_REQUEST);
}
@ParameterizedTest
@MethodSource
void testSignupCaptcha(final String message, final boolean enforced, final Set<String> countryCodes, final int expectedResponseStatusCode) {

View File

@@ -72,7 +72,7 @@ public class AccountCleanerTest {
}
@Test
public void testAccounts() throws AccountDatabaseCrawlerRestartException {
public void testAccounts() throws AccountDatabaseCrawlerRestartException, InterruptedException {
AccountCleaner accountCleaner = new AccountCleaner(accountsManager);
accountCleaner.onCrawlStart();
accountCleaner.timeAndProcessCrawlChunk(Optional.empty(), Arrays.asList(deletedDisabledAccount, undeletedDisabledAccount, undeletedEnabledAccount));
@@ -86,7 +86,7 @@ public class AccountCleanerTest {
}
@Test
public void testMaxAccountUpdates() throws AccountDatabaseCrawlerRestartException {
public void testMaxAccountUpdates() throws AccountDatabaseCrawlerRestartException, InterruptedException {
List<Account> accounts = new LinkedList<>();
accounts.add(undeletedEnabledAccount);

View File

@@ -50,6 +50,7 @@ import org.whispersystems.textsecuregcm.storage.AccountsDynamoDb;
import org.whispersystems.textsecuregcm.storage.AccountsManager;
import org.whispersystems.textsecuregcm.storage.ContestedOptimisticLockException;
import org.whispersystems.textsecuregcm.storage.DeletedAccounts;
import org.whispersystems.textsecuregcm.storage.DeletedAccountsManager;
import org.whispersystems.textsecuregcm.storage.Device;
import org.whispersystems.textsecuregcm.storage.DynamicConfigurationManager;
import org.whispersystems.textsecuregcm.storage.KeysDynamoDb;
@@ -87,7 +88,7 @@ class AccountsManagerTest {
FaultTolerantRedisCluster cacheCluster = RedisClusterHelper.buildMockRedisCluster(commands);
Accounts accounts = mock(Accounts.class);
AccountsDynamoDb accountsDynamoDb = mock(AccountsDynamoDb.class);
DeletedAccounts deletedAccounts = mock(DeletedAccounts.class);
DeletedAccountsManager deletedAccounts = mock(DeletedAccountsManager.class);
DirectoryQueue directoryQueue = mock(DirectoryQueue.class);
KeysDynamoDb keysDynamoDb = mock(KeysDynamoDb.class);
MessagesManager messagesManager = mock(MessagesManager.class);
@@ -139,7 +140,7 @@ class AccountsManagerTest {
FaultTolerantRedisCluster cacheCluster = RedisClusterHelper.buildMockRedisCluster(commands);
Accounts accounts = mock(Accounts.class);
AccountsDynamoDb accountsDynamoDb = mock(AccountsDynamoDb.class);
DeletedAccounts deletedAccounts = mock(DeletedAccounts.class);
DeletedAccountsManager deletedAccounts = mock(DeletedAccountsManager.class);
DirectoryQueue directoryQueue = mock(DirectoryQueue.class);
KeysDynamoDb keysDynamoDb = mock(KeysDynamoDb.class);
MessagesManager messagesManager = mock(MessagesManager.class);
@@ -178,7 +179,7 @@ class AccountsManagerTest {
FaultTolerantRedisCluster cacheCluster = RedisClusterHelper.buildMockRedisCluster(commands);
Accounts accounts = mock(Accounts.class);
AccountsDynamoDb accountsDynamoDb = mock(AccountsDynamoDb.class);
DeletedAccounts deletedAccounts = mock(DeletedAccounts.class);
DeletedAccountsManager deletedAccounts = mock(DeletedAccountsManager.class);
DirectoryQueue directoryQueue = mock(DirectoryQueue.class);
KeysDynamoDb keysDynamoDb = mock(KeysDynamoDb.class);
MessagesManager messagesManager = mock(MessagesManager.class);
@@ -221,7 +222,7 @@ class AccountsManagerTest {
FaultTolerantRedisCluster cacheCluster = RedisClusterHelper.buildMockRedisCluster(commands);
Accounts accounts = mock(Accounts.class);
AccountsDynamoDb accountsDynamoDb = mock(AccountsDynamoDb.class);
DeletedAccounts deletedAccounts = mock(DeletedAccounts.class);
DeletedAccountsManager deletedAccounts = mock(DeletedAccountsManager.class);
DirectoryQueue directoryQueue = mock(DirectoryQueue.class);
KeysDynamoDb keysDynamoDb = mock(KeysDynamoDb.class);
MessagesManager messagesManager = mock(MessagesManager.class);
@@ -263,7 +264,7 @@ class AccountsManagerTest {
FaultTolerantRedisCluster cacheCluster = RedisClusterHelper.buildMockRedisCluster(commands);
Accounts accounts = mock(Accounts.class);
AccountsDynamoDb accountsDynamoDb = mock(AccountsDynamoDb.class);
DeletedAccounts deletedAccounts = mock(DeletedAccounts.class);
DeletedAccountsManager deletedAccounts = mock(DeletedAccountsManager.class);
DirectoryQueue directoryQueue = mock(DirectoryQueue.class);
KeysDynamoDb keysDynamoDb = mock(KeysDynamoDb.class);
MessagesManager messagesManager = mock(MessagesManager.class);
@@ -305,7 +306,7 @@ class AccountsManagerTest {
FaultTolerantRedisCluster cacheCluster = RedisClusterHelper.buildMockRedisCluster(commands);
Accounts accounts = mock(Accounts.class);
AccountsDynamoDb accountsDynamoDb = mock(AccountsDynamoDb.class);
DeletedAccounts deletedAccounts = mock(DeletedAccounts.class);
DeletedAccountsManager deletedAccounts = mock(DeletedAccountsManager.class);
DirectoryQueue directoryQueue = mock(DirectoryQueue.class);
KeysDynamoDb keysDynamoDb = mock(KeysDynamoDb.class);
MessagesManager messagesManager = mock(MessagesManager.class);
@@ -347,7 +348,7 @@ class AccountsManagerTest {
FaultTolerantRedisCluster cacheCluster = RedisClusterHelper.buildMockRedisCluster(commands);
Accounts accounts = mock(Accounts.class);
AccountsDynamoDb accountsDynamoDb = mock(AccountsDynamoDb.class);
DeletedAccounts deletedAccounts = mock(DeletedAccounts.class);
DeletedAccountsManager deletedAccountsManager = mock(DeletedAccountsManager.class);
DirectoryQueue directoryQueue = mock(DirectoryQueue.class);
KeysDynamoDb keysDynamoDb = mock(KeysDynamoDb.class);
MessagesManager messagesManager = mock(MessagesManager.class);
@@ -366,7 +367,7 @@ class AccountsManagerTest {
when(accountsDynamoDb.get(uuid)).thenReturn(Optional.of(new Account("+14152222222", uuid, new HashSet<>(), new byte[16])));
doAnswer(ACCOUNT_UPDATE_ANSWER).when(accounts).update(any(Account.class));
AccountsManager accountsManager = new AccountsManager(accounts, accountsDynamoDb, cacheCluster, deletedAccounts,
AccountsManager accountsManager = new AccountsManager(accounts, accountsDynamoDb, cacheCluster, deletedAccountsManager,
directoryQueue, keysDynamoDb, messagesManager, usernamesManager, profilesManager, secureStorageClient, secureBackupClient, experimentEnrollmentManager, dynamicConfigurationManager);
Account updatedAccount = accountsManager.update(account, a -> a.setProfileName("name"));
@@ -408,7 +409,7 @@ class AccountsManagerTest {
FaultTolerantRedisCluster cacheCluster = RedisClusterHelper.buildMockRedisCluster(commands);
Accounts accounts = mock(Accounts.class);
AccountsDynamoDb accountsDynamoDb = mock(AccountsDynamoDb.class);
DeletedAccounts deletedAccounts = mock(DeletedAccounts.class);
DeletedAccountsManager deletedAccounts = mock(DeletedAccountsManager.class);
DirectoryQueue directoryQueue = mock(DirectoryQueue.class);
KeysDynamoDb keysDynamoDb = mock(KeysDynamoDb.class);
MessagesManager messagesManager = mock(MessagesManager.class);
@@ -447,7 +448,7 @@ class AccountsManagerTest {
FaultTolerantRedisCluster cacheCluster = RedisClusterHelper.buildMockRedisCluster(commands);
Accounts accounts = mock(Accounts.class);
AccountsDynamoDb accountsDynamoDb = mock(AccountsDynamoDb.class);
DeletedAccounts deletedAccounts = mock(DeletedAccounts.class);
DeletedAccountsManager deletedAccounts = mock(DeletedAccountsManager.class);
DirectoryQueue directoryQueue = mock(DirectoryQueue.class);
KeysDynamoDb keysDynamoDb = mock(KeysDynamoDb.class);
MessagesManager messagesManager = mock(MessagesManager.class);
@@ -495,7 +496,7 @@ class AccountsManagerTest {
FaultTolerantRedisCluster cacheCluster = RedisClusterHelper.buildMockRedisCluster(commands);
Accounts accounts = mock(Accounts.class);
AccountsDynamoDb accountsDynamoDb = mock(AccountsDynamoDb.class);
DeletedAccounts deletedAccounts = mock(DeletedAccounts.class);
DeletedAccountsManager deletedAccountsManager = mock(DeletedAccountsManager.class);
DirectoryQueue directoryQueue = mock(DirectoryQueue.class);
KeysDynamoDb keysDynamoDb = mock(KeysDynamoDb.class);
MessagesManager messagesManager = mock(MessagesManager.class);
@@ -513,7 +514,7 @@ class AccountsManagerTest {
.thenReturn(Optional.of(account));
when(accountsDynamoDb.create(any())).thenThrow(ContestedOptimisticLockException.class);
AccountsManager accountsManager = new AccountsManager(accounts, accountsDynamoDb, cacheCluster, deletedAccounts, directoryQueue, keysDynamoDb, messagesManager, usernamesManager, profilesManager, secureStorageClient, secureBackupClient, experimentEnrollmentManager, dynamicConfigurationManager);
AccountsManager accountsManager = new AccountsManager(accounts, accountsDynamoDb, cacheCluster, deletedAccountsManager, directoryQueue, keysDynamoDb, messagesManager, usernamesManager, profilesManager, secureStorageClient, secureBackupClient, experimentEnrollmentManager, dynamicConfigurationManager);
accountsManager.update(account, a -> {});
@@ -530,7 +531,7 @@ class AccountsManagerTest {
FaultTolerantRedisCluster cacheCluster = RedisClusterHelper.buildMockRedisCluster(commands);
Accounts accounts = mock(Accounts.class);
AccountsDynamoDb accountsDynamoDb = mock(AccountsDynamoDb.class);
DeletedAccounts deletedAccounts = mock(DeletedAccounts.class);
DeletedAccountsManager deletedAccountsManager = mock(DeletedAccountsManager.class);
DirectoryQueue directoryQueue = mock(DirectoryQueue.class);
KeysDynamoDb keysDynamoDb = mock(KeysDynamoDb.class);
MessagesManager messagesManager = mock(MessagesManager.class);
@@ -539,7 +540,7 @@ class AccountsManagerTest {
SecureBackupClient secureBackupClient = mock(SecureBackupClient.class);
SecureStorageClient secureStorageClient = mock(SecureStorageClient.class);
AccountsManager accountsManager = new AccountsManager(accounts, accountsDynamoDb, cacheCluster, deletedAccounts, directoryQueue, keysDynamoDb, messagesManager, usernamesManager, profilesManager, secureStorageClient, secureBackupClient, experimentEnrollmentManager, dynamicConfigurationManager);
AccountsManager accountsManager = new AccountsManager(accounts, accountsDynamoDb, cacheCluster, deletedAccountsManager, directoryQueue, keysDynamoDb, messagesManager, usernamesManager, profilesManager, secureStorageClient, secureBackupClient, experimentEnrollmentManager, dynamicConfigurationManager);
assertEquals(Optional.empty(), accountsManager.compareAccounts(Optional.empty(), Optional.empty()));
@@ -580,7 +581,7 @@ class AccountsManagerTest {
FaultTolerantRedisCluster cacheCluster = RedisClusterHelper.buildMockRedisCluster(commands);
Accounts accounts = mock(Accounts.class);
AccountsDynamoDb accountsDynamoDb = mock(AccountsDynamoDb.class);
DeletedAccounts deletedAccounts = mock(DeletedAccounts.class);
DeletedAccountsManager deletedAccounts = mock(DeletedAccountsManager.class);
DirectoryQueue directoryQueue = mock(DirectoryQueue.class);
KeysDynamoDb keysDynamoDb = mock(KeysDynamoDb.class);
MessagesManager messagesManager = mock(MessagesManager.class);