Signal/Signal-Server
1
0
Fork 0
mirror of https://github.com/signalapp/Signal-Server synced 2026-04-21 21:08:03 +01:00
Code Issues Packages Projects Releases Wiki Activity

Add threadpoool to increase Accounts → Dynamo migration throughput

Browse Source
This commit is contained in:
Chris Eager
2021-04-19 14:03:33 -05:00
committed by Chris Eager
parent 166d203e8e
commit a472774734
8 changed files with 138 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
service/src/test/java/org/whispersystems/textsecuregcm/configuration/dynamic/DynamicConfigurationTest.java
View File

@@ -318,6 +318,7 @@ class DynamicConfigurationTest {
final String accountsDynamoDbMigrationConfig =
"accountsDynamoDbMigration:\n"
+ " backgroundMigrationEnabled: true\n"
+ " backgroundMigrationExecutorThreads: 100\n"
+ " deleteEnabled: true\n"
+ " readEnabled: true\n"
+ " writeEnabled: true";
@@ -327,6 +328,7 @@ class DynamicConfigurationTest {
.getAccountsDynamoDbMigrationConfiguration();
assertTrue(config.isBackgroundMigrationEnabled());
assertEquals(100, config.getBackgroundMigrationExecutorThreads());
assertTrue(config.isDeleteEnabled());
assertTrue(config.isWriteEnabled());
assertTrue(config.isReadEnabled());

19
service/src/test/java/org/whispersystems/textsecuregcm/storage/AccountsDynamoDbTest.java
View File

@@ -12,6 +12,7 @@ import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import com.amazonaws.services.dynamodbv2.AmazonDynamoDB;
import com.amazonaws.services.dynamodbv2.AmazonDynamoDBAsync;
import com.amazonaws.services.dynamodbv2.document.DynamoDB;
import com.amazonaws.services.dynamodbv2.document.Item;
import com.amazonaws.services.dynamodbv2.document.Table;
@@ -31,6 +32,10 @@ import java.util.Optional;
import java.util.Random;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import org.jdbi.v3.core.transaction.TransactionException;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled;
@@ -65,7 +70,7 @@ class AccountsDynamoDbTest {
final Table numbersTable = dynamoDbExtension.getDynamoDB().createTable(createNumbersTableRequest);
this.accountsDynamoDb = new AccountsDynamoDb(dynamoDbExtension.getClient(), dynamoDbExtension.getDynamoDB(), dynamoDbExtension.getTableName(), numbersTable.getTableName());
this.accountsDynamoDb = new AccountsDynamoDb(dynamoDbExtension.getClient(), dynamoDbExtension.getAsyncClient(), new ThreadPoolExecutor(1, 1, 0, TimeUnit.SECONDS, new LinkedBlockingDeque<>()), dynamoDbExtension.getDynamoDB(), dynamoDbExtension.getTableName(), numbersTable.getTableName());
}
@Test
@@ -244,7 +249,7 @@ class AccountsDynamoDbTest {
when(client.updateItem(any()))
.thenThrow(RuntimeException.class);
AccountsDynamoDb accounts = new AccountsDynamoDb(client, dynamoDB, ACCOUNTS_TABLE_NAME, NUMBERS_TABLE_NAME);
AccountsDynamoDb accounts = new AccountsDynamoDb(client, mock(AmazonDynamoDBAsync.class), mock(ThreadPoolExecutor.class), dynamoDB, ACCOUNTS_TABLE_NAME, NUMBERS_TABLE_NAME);
Account account = generateAccount("+14151112222", UUID.randomUUID());
try {
@@ -279,19 +284,19 @@ class AccountsDynamoDbTest {
}
@Test
void testMigrate() {
void testMigrate() throws ExecutionException, InterruptedException {
Device device = generateDevice (1 );
UUID firstUuid = UUID.randomUUID();
Account account = generateAccount("+14151112222", firstUuid, Collections.singleton(device));
boolean migrated = accountsDynamoDb.migrate(account);
boolean migrated = accountsDynamoDb.migrate(account).get();
assertThat(migrated).isTrue();
verifyStoredState("+14151112222", account.getUuid(), account);
migrated = accountsDynamoDb.migrate(account);
migrated = accountsDynamoDb.migrate(account).get();
assertThat(migrated).isFalse();
@@ -302,14 +307,14 @@ class AccountsDynamoDbTest {
device = generateDevice(1);
Account accountRemigrationWithDifferentUuid = generateAccount("+14151112222", secondUuid, Collections.singleton(device));
migrated = accountsDynamoDb.migrate(account);
migrated = accountsDynamoDb.migrate(account).get();
assertThat(migrated).isFalse();
verifyStoredState("+14151112222", firstUuid, account);
account.setDynamoDbMigrationVersion(account.getDynamoDbMigrationVersion() + 1);
migrated = accountsDynamoDb.migrate(account);
migrated = accountsDynamoDb.migrate(account).get();
assertThat(migrated).isTrue();
}

22
service/src/test/java/org/whispersystems/textsecuregcm/storage/DynamoDbExtension.java
View File

@@ -5,6 +5,8 @@ import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.client.builder.AwsClientBuilder;
import com.amazonaws.services.dynamodbv2.AmazonDynamoDB;
import com.amazonaws.services.dynamodbv2.AmazonDynamoDBAsync;
import com.amazonaws.services.dynamodbv2.AmazonDynamoDBAsyncClientBuilder;
import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClientBuilder;
import com.amazonaws.services.dynamodbv2.document.DynamoDB;
import com.amazonaws.services.dynamodbv2.local.main.ServerRunner;
@@ -41,6 +43,7 @@ public class DynamoDbExtension implements BeforeEachCallback, AfterEachCallback
private final long writeCapacityUnits;
private AmazonDynamoDB client;
private AmazonDynamoDBAsync asyncClient;
private DynamoDB dynamoDB;
private DynamoDbExtension(String tableName, String hashKey, String rangeKey, List<AttributeDefinition> attributeDefinitions, List<GlobalSecondaryIndex> globalSecondaryIndexes, long readCapacityUnits,
@@ -114,11 +117,16 @@ public class DynamoDbExtension implements BeforeEachCallback, AfterEachCallback
}
private void initializeClient() {
client = AmazonDynamoDBClientBuilder.standard()
.withEndpointConfiguration(
new AwsClientBuilder.EndpointConfiguration("http://localhost:" + port, "local-test-region"))
.withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials("accessKey", "secretKey")))
.build();
AmazonDynamoDBClientBuilder clientBuilder = AmazonDynamoDBClientBuilder.standard()
.withEndpointConfiguration(
new AwsClientBuilder.EndpointConfiguration("http://localhost:" + port, "local-test-region"))
.withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials("accessKey", "secretKey")));
client = clientBuilder.build();
asyncClient = AmazonDynamoDBAsyncClientBuilder.standard()
.withEndpointConfiguration(clientBuilder.getEndpoint())
.withCredentials(clientBuilder.getCredentials())
.build();
dynamoDB = new DynamoDB(client);
}
@@ -174,6 +182,10 @@ public class DynamoDbExtension implements BeforeEachCallback, AfterEachCallback
return client;
}
public AmazonDynamoDBAsync getAsyncClient() {
return asyncClient;
}
public DynamoDB getDynamoDB() {
return dynamoDB;
}