mirror of
https://github.com/signalapp/Signal-Server
synced 2026-04-23 08:58:03 +01:00
Build Dynamo DB backed Message Store (#358)
* Work in progress... * Finish first pass draft of MessagesDynamoDb * Use begins_with everywhere for destination device id * Remove now unused methods * First basic test built * Add another test case * Remove comment * Verify more of the message contents * Ensure all methods are tested * Integrate MessagesDynamoDb into the MessagesManager This change plugs the MessagesDynamoDb class into the live serving flow in MessagesManager. Tests are not yet as comprehensive for this big a change as they should be, but they now compile and pass so checkpointing here with a commit. * Put DynamoDB before RDBS when deleting specific messages * Extract method * Make aws sdk version into a property * Rename clientBuilder * Discard messages with no GUID * Unify batching logic into one function * Comment on the source of the value in this constant * Inline method * Variable name swizzle * Add timers to all public methods * Add missing return statements * Reject messages that are too large with response code 413 * Add configuration to control dynamo DB timeouts * Set server timestamp from the ReceiptSender * Change to shorter key names to optimize IOPS * Fix tests broken by changing column names * Fix broken copyright template output * Remove copyright template error text * Add experiments to control use of dynamo and rds in message storage * Specify instance profile credentials for the dynamic configuration manager * Use property for aws sdk version * Switch dynamo to instance profile credentials * Add metrics to the batch write loop * Use placeholders in logging
This commit is contained in:
Ehren Kret
GitHub
@@ -0,0 +1,197 @@
|
||||
/*
|
||||
* Copyright 2021 Signal Messenger, LLC
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
package org.whispersystems.textsecuregcm.tests.storage;
|
||||
|
||||
import com.google.protobuf.ByteString;
|
||||
import org.junit.Before;
|
||||
import org.junit.ClassRule;
|
||||
import org.junit.Test;
|
||||
import org.whispersystems.textsecuregcm.entities.MessageProtos;
|
||||
import org.whispersystems.textsecuregcm.entities.OutgoingMessageEntity;
|
||||
import org.whispersystems.textsecuregcm.storage.MessagesDynamoDb;
|
||||
import org.whispersystems.textsecuregcm.tests.util.MessagesDynamoDbRule;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.List;
|
||||
import java.util.Random;
|
||||
import java.util.UUID;
|
||||
import java.util.function.Consumer;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
public class MessagesDynamoDbTest {
|
||||
private static final Random random = new Random();
|
||||
private static final MessageProtos.Envelope MESSAGE1;
|
||||
private static final MessageProtos.Envelope MESSAGE2;
|
||||
private static final MessageProtos.Envelope MESSAGE3;
|
||||
|
||||
static {
|
||||
final long serverTimestamp = System.currentTimeMillis();
|
||||
MessageProtos.Envelope.Builder builder = MessageProtos.Envelope.newBuilder();
|
||||
builder.setType(MessageProtos.Envelope.Type.UNIDENTIFIED_SENDER);
|
||||
builder.setTimestamp(123456789L);
|
||||
builder.setContent(ByteString.copyFrom(new byte[]{(byte) 0xDE, (byte) 0xAD, (byte) 0xBE, (byte) 0xEF}));
|
||||
builder.setServerGuid(UUID.randomUUID().toString());
|
||||
builder.setServerTimestamp(serverTimestamp);
|
||||
|
||||
MESSAGE1 = builder.build();
|
||||
|
||||
builder.setType(MessageProtos.Envelope.Type.CIPHERTEXT);
|
||||
builder.setSource("12348675309");
|
||||
builder.setSourceUuid(UUID.randomUUID().toString());
|
||||
builder.setSourceDevice(1);
|
||||
builder.setContent(ByteString.copyFromUtf8("MOO"));
|
||||
builder.setServerGuid(UUID.randomUUID().toString());
|
||||
builder.setServerTimestamp(serverTimestamp + 1);
|
||||
|
||||
MESSAGE2 = builder.build();
|
||||
|
||||
builder.setType(MessageProtos.Envelope.Type.UNIDENTIFIED_SENDER);
|
||||
builder.clearSource();
|
||||
builder.clearSourceUuid();
|
||||
builder.clearSourceDevice();
|
||||
builder.setContent(ByteString.copyFromUtf8("COW"));
|
||||
builder.setServerGuid(UUID.randomUUID().toString());
|
||||
builder.setServerTimestamp(serverTimestamp); // Test same millisecond arrival for two different messages
|
||||
|
||||
MESSAGE3 = builder.build();
|
||||
}
|
||||
|
||||
private MessagesDynamoDb messagesDynamoDb;
|
||||
|
||||
@ClassRule
|
||||
public static MessagesDynamoDbRule dynamoDbRule = new MessagesDynamoDbRule();
|
||||
|
||||
@Before
|
||||
public void setup() {
|
||||
messagesDynamoDb = new MessagesDynamoDb(dynamoDbRule.getDynamoDB(), MessagesDynamoDbRule.TABLE_NAME, Duration.ofDays(7));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testServerStart() {
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSimpleFetchAfterInsert() {
|
||||
final UUID destinationUuid = UUID.randomUUID();
|
||||
final int destinationDeviceId = random.nextInt(255) + 1;
|
||||
messagesDynamoDb.store(List.of(MESSAGE1, MESSAGE2, MESSAGE3), destinationUuid, destinationDeviceId);
|
||||
|
||||
final List<OutgoingMessageEntity> messagesStored = messagesDynamoDb.load(destinationUuid, destinationDeviceId, MessagesDynamoDb.RESULT_SET_CHUNK_SIZE);
|
||||
assertThat(messagesStored).isNotNull().hasSize(3);
|
||||
final MessageProtos.Envelope firstMessage = MESSAGE1.getServerGuid().compareTo(MESSAGE3.getServerGuid()) < 0 ? MESSAGE1 : MESSAGE3;
|
||||
final MessageProtos.Envelope secondMessage = firstMessage == MESSAGE1 ? MESSAGE3 : MESSAGE1;
|
||||
assertThat(messagesStored).element(0).satisfies(verify(firstMessage));
|
||||
assertThat(messagesStored).element(1).satisfies(verify(secondMessage));
|
||||
assertThat(messagesStored).element(2).satisfies(verify(MESSAGE2));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testDeleteForDestination() {
|
||||
final UUID destinationUuid = UUID.randomUUID();
|
||||
final UUID secondDestinationUuid = UUID.randomUUID();
|
||||
messagesDynamoDb.store(List.of(MESSAGE1), destinationUuid, 1);
|
||||
messagesDynamoDb.store(List.of(MESSAGE2), secondDestinationUuid, 1);
|
||||
messagesDynamoDb.store(List.of(MESSAGE3), destinationUuid, 2);
|
||||
|
||||
assertThat(messagesDynamoDb.load(destinationUuid, 1, MessagesDynamoDb.RESULT_SET_CHUNK_SIZE)).isNotNull().hasSize(1).element(0).satisfies(verify(MESSAGE1));
|
||||
assertThat(messagesDynamoDb.load(destinationUuid, 2, MessagesDynamoDb.RESULT_SET_CHUNK_SIZE)).isNotNull().hasSize(1).element(0).satisfies(verify(MESSAGE3));
|
||||
assertThat(messagesDynamoDb.load(secondDestinationUuid, 1, MessagesDynamoDb.RESULT_SET_CHUNK_SIZE)).isNotNull().hasSize(1).element(0).satisfies(verify(MESSAGE2));
|
||||
|
||||
messagesDynamoDb.deleteAllMessagesForAccount(destinationUuid);
|
||||
|
||||
assertThat(messagesDynamoDb.load(destinationUuid, 1, MessagesDynamoDb.RESULT_SET_CHUNK_SIZE)).isNotNull().isEmpty();
|
||||
assertThat(messagesDynamoDb.load(destinationUuid, 2, MessagesDynamoDb.RESULT_SET_CHUNK_SIZE)).isNotNull().isEmpty();
|
||||
assertThat(messagesDynamoDb.load(secondDestinationUuid, 1, MessagesDynamoDb.RESULT_SET_CHUNK_SIZE)).isNotNull().hasSize(1).element(0).satisfies(verify(MESSAGE2));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testDeleteForDestinationDevice() {
|
||||
final UUID destinationUuid = UUID.randomUUID();
|
||||
final UUID secondDestinationUuid = UUID.randomUUID();
|
||||
messagesDynamoDb.store(List.of(MESSAGE1), destinationUuid, 1);
|
||||
messagesDynamoDb.store(List.of(MESSAGE2), secondDestinationUuid, 1);
|
||||
messagesDynamoDb.store(List.of(MESSAGE3), destinationUuid, 2);
|
||||
|
||||
assertThat(messagesDynamoDb.load(destinationUuid, 1, MessagesDynamoDb.RESULT_SET_CHUNK_SIZE)).isNotNull().hasSize(1).element(0).satisfies(verify(MESSAGE1));
|
||||
assertThat(messagesDynamoDb.load(destinationUuid, 2, MessagesDynamoDb.RESULT_SET_CHUNK_SIZE)).isNotNull().hasSize(1).element(0).satisfies(verify(MESSAGE3));
|
||||
assertThat(messagesDynamoDb.load(secondDestinationUuid, 1, MessagesDynamoDb.RESULT_SET_CHUNK_SIZE)).isNotNull().hasSize(1).element(0).satisfies(verify(MESSAGE2));
|
||||
|
||||
messagesDynamoDb.deleteAllMessagesForDevice(destinationUuid, 2);
|
||||
|
||||
assertThat(messagesDynamoDb.load(destinationUuid, 1, MessagesDynamoDb.RESULT_SET_CHUNK_SIZE)).isNotNull().hasSize(1).element(0).satisfies(verify(MESSAGE1));
|
||||
assertThat(messagesDynamoDb.load(destinationUuid, 2, MessagesDynamoDb.RESULT_SET_CHUNK_SIZE)).isNotNull().isEmpty();
|
||||
assertThat(messagesDynamoDb.load(secondDestinationUuid, 1, MessagesDynamoDb.RESULT_SET_CHUNK_SIZE)).isNotNull().hasSize(1).element(0).satisfies(verify(MESSAGE2));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testDeleteMessageByDestinationAndSourceAndTimestamp() {
|
||||
final UUID destinationUuid = UUID.randomUUID();
|
||||
final UUID secondDestinationUuid = UUID.randomUUID();
|
||||
messagesDynamoDb.store(List.of(MESSAGE1), destinationUuid, 1);
|
||||
messagesDynamoDb.store(List.of(MESSAGE2), secondDestinationUuid, 1);
|
||||
messagesDynamoDb.store(List.of(MESSAGE3), destinationUuid, 2);
|
||||
|
||||
assertThat(messagesDynamoDb.load(destinationUuid, 1, MessagesDynamoDb.RESULT_SET_CHUNK_SIZE)).isNotNull().hasSize(1).element(0).satisfies(verify(MESSAGE1));
|
||||
assertThat(messagesDynamoDb.load(destinationUuid, 2, MessagesDynamoDb.RESULT_SET_CHUNK_SIZE)).isNotNull().hasSize(1).element(0).satisfies(verify(MESSAGE3));
|
||||
assertThat(messagesDynamoDb.load(secondDestinationUuid, 1, MessagesDynamoDb.RESULT_SET_CHUNK_SIZE)).isNotNull().hasSize(1).element(0).satisfies(verify(MESSAGE2));
|
||||
|
||||
messagesDynamoDb.deleteMessageByDestinationAndSourceAndTimestamp(secondDestinationUuid, 1, MESSAGE2.getSource(), MESSAGE2.getTimestamp());
|
||||
|
||||
assertThat(messagesDynamoDb.load(destinationUuid, 1, MessagesDynamoDb.RESULT_SET_CHUNK_SIZE)).isNotNull().hasSize(1).element(0).satisfies(verify(MESSAGE1));
|
||||
assertThat(messagesDynamoDb.load(destinationUuid, 2, MessagesDynamoDb.RESULT_SET_CHUNK_SIZE)).isNotNull().hasSize(1).element(0).satisfies(verify(MESSAGE3));
|
||||
assertThat(messagesDynamoDb.load(secondDestinationUuid, 1, MessagesDynamoDb.RESULT_SET_CHUNK_SIZE)).isNotNull().isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testDeleteMessageByDestinationAndGuid() {
|
||||
final UUID destinationUuid = UUID.randomUUID();
|
||||
final UUID secondDestinationUuid = UUID.randomUUID();
|
||||
messagesDynamoDb.store(List.of(MESSAGE1), destinationUuid, 1);
|
||||
messagesDynamoDb.store(List.of(MESSAGE2), secondDestinationUuid, 1);
|
||||
messagesDynamoDb.store(List.of(MESSAGE3), destinationUuid, 2);
|
||||
|
||||
assertThat(messagesDynamoDb.load(destinationUuid, 1, MessagesDynamoDb.RESULT_SET_CHUNK_SIZE)).isNotNull().hasSize(1).element(0).satisfies(verify(MESSAGE1));
|
||||
assertThat(messagesDynamoDb.load(destinationUuid, 2, MessagesDynamoDb.RESULT_SET_CHUNK_SIZE)).isNotNull().hasSize(1).element(0).satisfies(verify(MESSAGE3));
|
||||
assertThat(messagesDynamoDb.load(secondDestinationUuid, 1, MessagesDynamoDb.RESULT_SET_CHUNK_SIZE)).isNotNull().hasSize(1).element(0).satisfies(verify(MESSAGE2));
|
||||
|
||||
messagesDynamoDb.deleteMessageByDestinationAndGuid(secondDestinationUuid, 1, UUID.fromString(MESSAGE2.getServerGuid()));
|
||||
|
||||
assertThat(messagesDynamoDb.load(destinationUuid, 1, MessagesDynamoDb.RESULT_SET_CHUNK_SIZE)).isNotNull().hasSize(1).element(0).satisfies(verify(MESSAGE1));
|
||||
assertThat(messagesDynamoDb.load(destinationUuid, 2, MessagesDynamoDb.RESULT_SET_CHUNK_SIZE)).isNotNull().hasSize(1).element(0).satisfies(verify(MESSAGE3));
|
||||
assertThat(messagesDynamoDb.load(secondDestinationUuid, 1, MessagesDynamoDb.RESULT_SET_CHUNK_SIZE)).isNotNull().isEmpty();
|
||||
}
|
||||
|
||||
private static void verify(OutgoingMessageEntity retrieved, MessageProtos.Envelope inserted) {
|
||||
assertThat(retrieved.getTimestamp()).isEqualTo(inserted.getTimestamp());
|
||||
assertThat(retrieved.getSource()).isEqualTo(inserted.hasSource() ? inserted.getSource() : null);
|
||||
assertThat(retrieved.getSourceUuid()).isEqualTo(inserted.hasSourceUuid() ? UUID.fromString(inserted.getSourceUuid()) : null);
|
||||
assertThat(retrieved.getSourceDevice()).isEqualTo(inserted.getSourceDevice());
|
||||
assertThat(retrieved.getRelay()).isEqualTo(inserted.hasRelay() ? inserted.getRelay() : null);
|
||||
assertThat(retrieved.getType()).isEqualTo(inserted.getType().getNumber());
|
||||
assertThat(retrieved.getContent()).isEqualTo(inserted.hasContent() ? inserted.getContent().toByteArray() : null);
|
||||
assertThat(retrieved.getMessage()).isEqualTo(inserted.hasLegacyMessage() ? inserted.getLegacyMessage().toByteArray() : null);
|
||||
assertThat(retrieved.getServerTimestamp()).isEqualTo(inserted.getServerTimestamp());
|
||||
assertThat(retrieved.getGuid()).isEqualTo(UUID.fromString(inserted.getServerGuid()));
|
||||
}
|
||||
|
||||
private static VerifyMessage verify(MessageProtos.Envelope expected) {
|
||||
return new VerifyMessage(expected);
|
||||
}
|
||||
|
||||
private static final class VerifyMessage implements Consumer<OutgoingMessageEntity> {
|
||||
private final MessageProtos.Envelope expected;
|
||||
|
||||
public VerifyMessage(MessageProtos.Envelope expected) {
|
||||
this.expected = expected;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void accept(OutgoingMessageEntity outgoingMessageEntity) {
|
||||
verify(outgoingMessageEntity, expected);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
/*
|
||||
* Copyright 2021 Signal Messenger, LLC
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
package org.whispersystems.textsecuregcm.tests.util;
|
||||
|
||||
import com.almworks.sqlite4java.SQLite;
|
||||
import com.amazonaws.auth.AWSStaticCredentialsProvider;
|
||||
import com.amazonaws.auth.BasicAWSCredentials;
|
||||
import com.amazonaws.client.builder.AwsClientBuilder;
|
||||
import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClientBuilder;
|
||||
import com.amazonaws.services.dynamodbv2.document.DynamoDB;
|
||||
import com.amazonaws.services.dynamodbv2.local.main.ServerRunner;
|
||||
import com.amazonaws.services.dynamodbv2.local.server.DynamoDBProxyServer;
|
||||
import org.junit.rules.ExternalResource;
|
||||
|
||||
import java.net.ServerSocket;
|
||||
|
||||
public class LocalDynamoDbRule extends ExternalResource {
|
||||
private DynamoDBProxyServer server;
|
||||
private int port;
|
||||
|
||||
@Override
|
||||
protected void before() throws Throwable {
|
||||
super.before();
|
||||
SQLite.setLibraryPath("target/lib"); // if you see a library failed to load error, you need to run mvn test-compile at least once first
|
||||
ServerSocket serverSocket = new ServerSocket(0);
|
||||
serverSocket.setReuseAddress(false);
|
||||
port = serverSocket.getLocalPort();
|
||||
serverSocket.close();
|
||||
server = ServerRunner.createServerFromCommandLineArgs(new String[]{"-inMemory", "-port", String.valueOf(port)});
|
||||
server.start();
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void after() {
|
||||
try {
|
||||
server.stop();
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
super.after();
|
||||
}
|
||||
|
||||
public DynamoDB getDynamoDB() {
|
||||
AmazonDynamoDBClientBuilder clientBuilder =
|
||||
AmazonDynamoDBClientBuilder.standard()
|
||||
.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration("http://localhost:" + port, "local-test-region"))
|
||||
.withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials("accessKey", "secretKey")));
|
||||
return new DynamoDB(clientBuilder.build());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
/*
|
||||
* Copyright 2021 Signal Messenger, LLC
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
package org.whispersystems.textsecuregcm.tests.util;
|
||||
|
||||
import com.amazonaws.services.dynamodbv2.document.DynamoDB;
|
||||
import com.amazonaws.services.dynamodbv2.model.AttributeDefinition;
|
||||
import com.amazonaws.services.dynamodbv2.model.CreateTableRequest;
|
||||
import com.amazonaws.services.dynamodbv2.model.KeySchemaElement;
|
||||
import com.amazonaws.services.dynamodbv2.model.LocalSecondaryIndex;
|
||||
import com.amazonaws.services.dynamodbv2.model.Projection;
|
||||
import com.amazonaws.services.dynamodbv2.model.ProjectionType;
|
||||
import com.amazonaws.services.dynamodbv2.model.ProvisionedThroughput;
|
||||
import com.amazonaws.services.dynamodbv2.model.ScalarAttributeType;
|
||||
|
||||
public class MessagesDynamoDbRule extends LocalDynamoDbRule {
|
||||
|
||||
public static final String TABLE_NAME = "Signal_Messages_UnitTest";
|
||||
|
||||
@Override
|
||||
protected void before() throws Throwable {
|
||||
super.before();
|
||||
DynamoDB dynamoDB = getDynamoDB();
|
||||
CreateTableRequest createTableRequest = new CreateTableRequest()
|
||||
.withTableName(TABLE_NAME)
|
||||
.withKeySchema(new KeySchemaElement("H", "HASH"),
|
||||
new KeySchemaElement("S", "RANGE"))
|
||||
.withAttributeDefinitions(new AttributeDefinition("H", ScalarAttributeType.B),
|
||||
new AttributeDefinition("S", ScalarAttributeType.B),
|
||||
new AttributeDefinition("U", ScalarAttributeType.B))
|
||||
.withProvisionedThroughput(new ProvisionedThroughput(20L, 20L))
|
||||
.withLocalSecondaryIndexes(new LocalSecondaryIndex().withIndexName("Message_UUID_Index")
|
||||
.withKeySchema(new KeySchemaElement("H", "HASH"),
|
||||
new KeySchemaElement("U", "RANGE"))
|
||||
.withProjection(new Projection().withProjectionType(ProjectionType.KEYS_ONLY)));
|
||||
dynamoDB.createTable(createTableRequest);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void after() {
|
||||
super.after();
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user