Signal/Signal-Server
1
0
Fork 0
mirror of https://github.com/signalapp/Signal-Server synced 2026-04-20 12:08:05 +01:00
Code Issues Packages Projects Releases Wiki Activity

Update reconciliation crawler to use secondary index

Browse Source
This commit is contained in:
Chris Eager
2021-06-24 18:52:26 -05:00
committed by Chris Eager
parent 2f88f0eedb
commit 819d59cd79
6 changed files with 58 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerConfiguration.java
View File

@@ -22,6 +22,7 @@ import org.whispersystems.textsecuregcm.configuration.AwsAttachmentsConfiguratio
import org.whispersystems.textsecuregcm.configuration.CdnConfiguration;
import org.whispersystems.textsecuregcm.configuration.DatabaseConfiguration;
import org.whispersystems.textsecuregcm.configuration.DatadogConfiguration;
import org.whispersystems.textsecuregcm.configuration.DeletedAccountsDynamoDbConfiguration;
import org.whispersystems.textsecuregcm.configuration.DirectoryConfiguration;
import org.whispersystems.textsecuregcm.configuration.DonationConfiguration;
import org.whispersystems.textsecuregcm.configuration.DynamoDbConfiguration;
@@ -160,7 +161,7 @@ public class WhisperServerConfiguration extends Configuration {
@Valid
@NotNull
@JsonProperty
private DynamoDbConfiguration deletedAccountsDynamoDb;
private DeletedAccountsDynamoDbConfiguration deletedAccountsDynamoDb;
@Valid
@NotNull
@@ -386,7 +387,7 @@ public class WhisperServerConfiguration extends Configuration {
return migrationRetryAccountsDynamoDb;
}
public DynamoDbConfiguration getDeletedAccountsDynamoDbConfiguration() {
public DeletedAccountsDynamoDbConfiguration getDeletedAccountsDynamoDbConfiguration() {
return deletedAccountsDynamoDb;
}

2
service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java
View File

@@ -355,7 +355,7 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
DynamoDbClient pendingDevicesDynamoDbClient = DynamoDbFromConfig.client(config.getPendingDevicesDynamoDbConfiguration(),
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
DeletedAccounts deletedAccounts = new DeletedAccounts(deletedAccountsDynamoDbClient, config.getDeletedAccountsDynamoDbConfiguration().getTableName());
DeletedAccounts deletedAccounts = new DeletedAccounts(deletedAccountsDynamoDbClient, config.getDeletedAccountsDynamoDbConfiguration().getTableName(), config.getDeletedAccountsDynamoDbConfiguration().getNeedsReconciliationIndexName());
MigrationDeletedAccounts migrationDeletedAccounts = new MigrationDeletedAccounts(recentlyDeletedAccountsDynamoDb, config.getMigrationDeletedAccountsDynamoDbConfiguration().getTableName());
MigrationRetryAccounts migrationRetryAccounts = new MigrationRetryAccounts(migrationRetryAccountsDynamoDb, config.getMigrationRetryAccountsDynamoDbConfiguration().getTableName());

13
service/src/main/java/org/whispersystems/textsecuregcm/configuration/DeletedAccountsDynamoDbConfiguration.java Normal file
View File

@@ -0,0 +1,13 @@
package org.whispersystems.textsecuregcm.configuration;
import javax.validation.constraints.NotNull;
public class DeletedAccountsDynamoDbConfiguration extends DynamoDbConfiguration {
@NotNull
private String needsReconciliationIndexName;
public String getNeedsReconciliationIndexName() {
return needsReconciliationIndexName;
}
}

42
service/src/main/java/org/whispersystems/textsecuregcm/storage/DeletedAccounts.java
View File

@@ -14,9 +14,8 @@ import org.whispersystems.textsecuregcm.util.AttributeValues;
import org.whispersystems.textsecuregcm.util.Pair;
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
import software.amazon.awssdk.services.dynamodb.model.PutItemRequest;
import software.amazon.awssdk.services.dynamodb.model.PutRequest;
import software.amazon.awssdk.services.dynamodb.model.ScanRequest;
import software.amazon.awssdk.services.dynamodb.model.WriteRequest;
import software.amazon.awssdk.services.dynamodb.model.UpdateItemRequest;
public class DeletedAccounts extends AbstractDynamoDbStore {
@@ -24,16 +23,18 @@ public class DeletedAccounts extends AbstractDynamoDbStore {
static final String KEY_ACCOUNT_E164 = "P";
static final String ATTR_ACCOUNT_UUID = "U";
static final String ATTR_EXPIRES = "E";
static final String ATTR_RECONCILED_IN_CDS = "R";
static final String ATTR_NEEDS_CDS_RECONCILIATION = "R";
static final Duration TIME_TO_LIVE = Duration.ofDays(30);
private final String tableName;
private final String needsReconciliationIndexName;
public DeletedAccounts(final DynamoDbClient dynamoDb, final String tableName) {
public DeletedAccounts(final DynamoDbClient dynamoDb, final String tableName, final String needsReconciliationIndexName) {
super(dynamoDb);
this.tableName = tableName;
this.needsReconciliationIndexName = needsReconciliationIndexName;
}
public void put(UUID uuid, String e164) {
@@ -43,7 +44,7 @@ public class DeletedAccounts extends AbstractDynamoDbStore {
KEY_ACCOUNT_E164, AttributeValues.fromString(e164),
ATTR_ACCOUNT_UUID, AttributeValues.fromUUID(uuid),
ATTR_EXPIRES, AttributeValues.fromLong(Instant.now().plus(TIME_TO_LIVE).getEpochSecond()),
ATTR_RECONCILED_IN_CDS, AttributeValues.fromBoolean(false)))
ATTR_NEEDS_CDS_RECONCILIATION, AttributeValues.fromInt(1)))
.build());
}
@@ -51,13 +52,7 @@ public class DeletedAccounts extends AbstractDynamoDbStore {
final ScanRequest scanRequest = ScanRequest.builder()
.tableName(tableName)
.filterExpression("#reconciled = :reconciled")
.expressionAttributeNames(Map.of(
"#reconciled", ATTR_RECONCILED_IN_CDS
))
.expressionAttributeValues(Map.of(
":reconciled", AttributeValues.fromBoolean(false)
))
.indexName(needsReconciliationIndexName)
.limit(max)
.build();
@@ -70,18 +65,19 @@ public class DeletedAccounts extends AbstractDynamoDbStore {
}
public void markReconciled(final List<String> phoneNumbersReconciled) {
writeInBatches(phoneNumbersReconciled, phoneNumbers -> {
final List<WriteRequest> updates = phoneNumbers.stream()
.map(phoneNumber -> WriteRequest.builder()
.putRequest(PutRequest.builder().item(
Map.of(KEY_ACCOUNT_E164, AttributeValues.fromString(phoneNumber),
ATTR_RECONCILED_IN_CDS, AttributeValues.fromBoolean(true))
).build()).build()
).collect(Collectors.toList());
executeTableWriteItemsUntilComplete(Map.of(tableName, updates));
});
phoneNumbersReconciled.forEach(number -> db().updateItem(
UpdateItemRequest.builder()
.tableName(tableName)
.key(Map.of(
KEY_ACCOUNT_E164, AttributeValues.fromString(number)
))
.updateExpression("REMOVE #needs_reconciliation")
.expressionAttributeNames(Map.of(
"#needs_reconciliation", ATTR_NEEDS_CDS_RECONCILIATION
))
.build()
));
}
}

2
service/src/main/java/org/whispersystems/textsecuregcm/workers/DeleteUserCommand.java
View File

@@ -133,7 +133,7 @@ public class DeleteUserCommand extends EnvironmentCommand<WhisperServerConfigura
DynamoDbClient migrationRetryAccountsDynamoDb = DynamoDbFromConfig.client(configuration.getMigrationRetryAccountsDynamoDbConfiguration(),
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
DeletedAccounts deletedAccounts = new DeletedAccounts(deletedAccountsDynamoDbClient, configuration.getDeletedAccountsDynamoDbConfiguration().getTableName());
DeletedAccounts deletedAccounts = new DeletedAccounts(deletedAccountsDynamoDbClient, configuration.getDeletedAccountsDynamoDbConfiguration().getTableName(), configuration.getDeletedAccountsDynamoDbConfiguration().getNeedsReconciliationIndexName());
MigrationDeletedAccounts migrationDeletedAccounts = new MigrationDeletedAccounts(migrationDeletedAccountsDynamoDb, configuration.getMigrationDeletedAccountsDynamoDbConfiguration().getTableName());
MigrationRetryAccounts migrationRetryAccounts = new MigrationRetryAccounts(migrationRetryAccountsDynamoDb, configuration.getMigrationRetryAccountsDynamoDbConfiguration().getTableName());