Remove message database from the codebase (#395)

* Remove message database from the codebase

* Remove unused ExperimentEnrollmentManager in test

* Be more stylish
This commit is contained in:
Ehren Kret
2021-02-11 10:50:03 -06:00
committed by GitHub
parent 477615fc66
commit be8a1acca9
21 changed files with 258 additions and 1049 deletions

View File

@@ -5,6 +5,10 @@
package org.whispersystems.textsecuregcm.storage;
import static org.junit.Assert.assertEquals;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import com.amazonaws.services.dynamodbv2.document.DynamoDB;
import com.amazonaws.services.dynamodbv2.document.Item;
import com.amazonaws.services.dynamodbv2.document.ItemCollection;
@@ -12,23 +16,7 @@ import com.amazonaws.services.dynamodbv2.document.ScanOutcome;
import com.amazonaws.services.dynamodbv2.document.Table;
import com.amazonaws.services.dynamodbv2.document.spec.ScanSpec;
import com.google.protobuf.ByteString;
import com.opentable.db.postgres.embedded.LiquibasePreparer;
import com.opentable.db.postgres.junit.EmbeddedPostgresRules;
import com.opentable.db.postgres.junit.PreparedDbRule;
import io.lettuce.core.cluster.SlotHash;
import org.apache.commons.lang3.RandomStringUtils;
import org.jdbi.v3.core.Jdbi;
import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.whispersystems.textsecuregcm.configuration.CircuitBreakerConfiguration;
import org.whispersystems.textsecuregcm.entities.MessageProtos;
import org.whispersystems.textsecuregcm.experiment.ExperimentEnrollmentManager;
import org.whispersystems.textsecuregcm.metrics.PushLatencyManager;
import org.whispersystems.textsecuregcm.redis.AbstractRedisClusterTest;
import org.whispersystems.textsecuregcm.tests.util.MessagesDynamoDbRule;
import java.nio.ByteBuffer;
import java.time.Duration;
import java.time.Instant;
@@ -40,18 +28,18 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import static org.junit.Assert.assertEquals;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import org.apache.commons.lang3.RandomStringUtils;
import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.whispersystems.textsecuregcm.entities.MessageProtos;
import org.whispersystems.textsecuregcm.metrics.PushLatencyManager;
import org.whispersystems.textsecuregcm.redis.AbstractRedisClusterTest;
import org.whispersystems.textsecuregcm.tests.util.MessagesDynamoDbRule;
public class MessagePersisterIntegrationTest extends AbstractRedisClusterTest {
@Rule
public PreparedDbRule db = EmbeddedPostgresRules.preparedDatabase(LiquibasePreparer.forClasspathLocation("messagedb.xml"));
@Rule
public MessagesDynamoDbRule messagesDynamoDbRule = new MessagesDynamoDbRule();
@@ -60,7 +48,6 @@ public class MessagePersisterIntegrationTest extends AbstractRedisClusterTest {
private MessagesManager messagesManager;
private MessagePersister messagePersister;
private Account account;
private ExperimentEnrollmentManager experimentEnrollmentManager;
private static final Duration PERSIST_DELAY = Duration.ofMinutes(10);
@@ -74,16 +61,12 @@ public class MessagePersisterIntegrationTest extends AbstractRedisClusterTest {
connection.sync().masters().commands().configSet("notify-keyspace-events", "K$glz");
});
final Messages messages = new Messages(new FaultTolerantDatabase("messages-test", Jdbi.create(db.getTestDatabase()), new CircuitBreakerConfiguration()));
final MessagesDynamoDb messagesDynamoDb = new MessagesDynamoDb(messagesDynamoDbRule.getDynamoDB(), MessagesDynamoDbRule.TABLE_NAME, Duration.ofDays(7));
final AccountsManager accountsManager = mock(AccountsManager.class);
experimentEnrollmentManager = mock(ExperimentEnrollmentManager.class);
when(experimentEnrollmentManager.isEnrolled(any(UUID.class), anyString())).thenReturn(Boolean.TRUE);
notificationExecutorService = Executors.newSingleThreadExecutor();
messagesCache = new MessagesCache(getRedisCluster(), getRedisCluster(), notificationExecutorService);
messagesManager = new MessagesManager(messages, messagesDynamoDb, messagesCache, mock(PushLatencyManager.class), experimentEnrollmentManager);
messagesManager = new MessagesManager(messagesDynamoDb, messagesCache, mock(PushLatencyManager.class));
messagePersister = new MessagePersister(messagesCache, messagesManager, accountsManager, mock(FeatureFlagsManager.class), PERSIST_DELAY);
account = mock(Account.class);

View File

@@ -5,16 +5,19 @@
package org.whispersystems.textsecuregcm.storage;
import static org.junit.Assert.assertEquals;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.atLeastOnce;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import com.google.protobuf.ByteString;
import io.lettuce.core.cluster.SlotHash;
import org.apache.commons.lang3.RandomStringUtils;
import org.junit.Before;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
import org.mockito.stubbing.Answer;
import org.whispersystems.textsecuregcm.entities.MessageProtos;
import org.whispersystems.textsecuregcm.redis.AbstractRedisClusterTest;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.time.Instant;
@@ -24,24 +27,19 @@ import java.util.UUID;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import static org.junit.Assert.assertEquals;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.atLeastOnce;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import org.apache.commons.lang3.RandomStringUtils;
import org.junit.Before;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
import org.mockito.stubbing.Answer;
import org.whispersystems.textsecuregcm.entities.MessageProtos;
import org.whispersystems.textsecuregcm.redis.AbstractRedisClusterTest;
public class MessagePersisterTest extends AbstractRedisClusterTest {
private ExecutorService notificationExecutorService;
private MessagesCache messagesCache;
private Messages messagesDatabase;
private MessagesDynamoDb messagesDynamoDb;
private MessagePersister messagePersister;
private AccountsManager accountsManager;
@@ -58,7 +56,7 @@ public class MessagePersisterTest extends AbstractRedisClusterTest {
final MessagesManager messagesManager = mock(MessagesManager.class);
messagesDatabase = mock(Messages.class);
messagesDynamoDb = mock(MessagesDynamoDb.class);
accountsManager = mock(AccountsManager.class);
final Account account = mock(Account.class);
@@ -71,19 +69,18 @@ public class MessagePersisterTest extends AbstractRedisClusterTest {
messagePersister = new MessagePersister(messagesCache, messagesManager, accountsManager, mock(FeatureFlagsManager.class), PERSIST_DELAY);
doAnswer(invocation -> {
final String destination = invocation.getArgument(0, String.class);
final UUID destinationUuid = invocation.getArgument(1, UUID.class);
final long deviceId = invocation.getArgument(2, Long.class);
final List<MessageProtos.Envelope> messages = invocation.getArgument(3, List.class);
final UUID destinationUuid = invocation.getArgument(0);
final long destinationDeviceId = invocation.getArgument(1);
final List<MessageProtos.Envelope> messages = invocation.getArgument(2);
messagesDatabase.store(messages, destination, deviceId);
messagesDynamoDb.store(messages, destinationUuid, destinationDeviceId);
for (final MessageProtos.Envelope message : messages) {
messagesCache.remove(destinationUuid, deviceId, UUID.fromString(message.getServerGuid()));
messagesCache.remove(destinationUuid, destinationDeviceId, UUID.fromString(message.getServerGuid()));
}
return null;
}).when(messagesManager).persistMessages(anyString(), any(UUID.class), anyLong(), any());
}).when(messagesManager).persistMessages(any(UUID.class), anyLong(), any());
}
@Override
@@ -114,7 +111,7 @@ public class MessagePersisterTest extends AbstractRedisClusterTest {
final ArgumentCaptor<List<MessageProtos.Envelope>> messagesCaptor = ArgumentCaptor.forClass(List.class);
verify(messagesDatabase, atLeastOnce()).store(messagesCaptor.capture(), eq(DESTINATION_ACCOUNT_NUMBER), eq(DESTINATION_DEVICE_ID));
verify(messagesDynamoDb, atLeastOnce()).store(messagesCaptor.capture(), eq(DESTINATION_ACCOUNT_UUID), eq(DESTINATION_DEVICE_ID));
assertEquals(messageCount, messagesCaptor.getAllValues().stream().mapToInt(List::size).sum());
}
@@ -129,7 +126,7 @@ public class MessagePersisterTest extends AbstractRedisClusterTest {
messagePersister.persistNextQueues(now);
verify(messagesDatabase, never()).store(any(), anyString(), anyLong());
verify(messagesDynamoDb, never()).store(any(), any(), anyLong());
}
@Test
@@ -159,7 +156,7 @@ public class MessagePersisterTest extends AbstractRedisClusterTest {
final ArgumentCaptor<List<MessageProtos.Envelope>> messagesCaptor = ArgumentCaptor.forClass(List.class);
verify(messagesDatabase, atLeastOnce()).store(messagesCaptor.capture(), anyString(), anyLong());
verify(messagesDynamoDb, atLeastOnce()).store(messagesCaptor.capture(), any(UUID.class), anyLong());
assertEquals(queueCount * messagesPerQueue, messagesCaptor.getAllValues().stream().mapToInt(List::size).sum());
}
@@ -174,7 +171,7 @@ public class MessagePersisterTest extends AbstractRedisClusterTest {
doAnswer((Answer<Void>)invocation -> {
throw new RuntimeException("OH NO.");
}).when(messagesDatabase).store(any(), eq(DESTINATION_ACCOUNT_NUMBER), eq(DESTINATION_DEVICE_ID));
}).when(messagesDynamoDb).store(any(), eq(DESTINATION_ACCOUNT_UUID), eq(DESTINATION_DEVICE_ID));
messagePersister.persistNextQueues(now.plus(messagePersister.getPersistDelay()));

View File

@@ -4,9 +4,25 @@
*/
package org.whispersystems.textsecuregcm.tests.controllers;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.Assert.assertEquals;
import static org.mockito.Mockito.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoMoreInteractions;
import static org.mockito.Mockito.when;
import com.google.common.collect.ImmutableSet;
import io.dropwizard.auth.PolymorphicAuthValueFactoryProvider;
import io.dropwizard.testing.junit.ResourceTestRule;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
import javax.ws.rs.Path;
import javax.ws.rs.client.Entity;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import junitparams.JUnitParamsRunner;
import junitparams.Parameters;
import org.glassfish.jersey.test.grizzly.GrizzlyWebTestContainerFactory;
@@ -31,23 +47,6 @@ import org.whispersystems.textsecuregcm.storage.PendingDevicesManager;
import org.whispersystems.textsecuregcm.tests.util.AuthHelper;
import org.whispersystems.textsecuregcm.util.VerificationCode;
import javax.ws.rs.Path;
import javax.ws.rs.client.Entity;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.Assert.assertEquals;
import static org.mockito.Mockito.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoMoreInteractions;
import static org.mockito.Mockito.when;
@RunWith(JUnitParamsRunner.class)
public class DeviceControllerTest {
@Path("/v1/devices")
@@ -143,7 +142,7 @@ public class DeviceControllerTest {
assertThat(response.getDeviceId()).isEqualTo(42L);
verify(pendingDevicesManager).remove(AuthHelper.VALID_NUMBER);
verify(messagesManager).clear(eq(AuthHelper.VALID_NUMBER), eq(AuthHelper.VALID_UUID), eq(42L));
verify(messagesManager).clear(eq(AuthHelper.VALID_UUID), eq(42L));
}
@Test

View File

@@ -66,7 +66,6 @@ import org.whispersystems.textsecuregcm.push.ReceiptSender;
import org.whispersystems.textsecuregcm.storage.Account;
import org.whispersystems.textsecuregcm.storage.AccountsManager;
import org.whispersystems.textsecuregcm.storage.Device;
import org.whispersystems.textsecuregcm.storage.FeatureFlagsManager;
import org.whispersystems.textsecuregcm.storage.MessagesManager;
import org.whispersystems.textsecuregcm.tests.util.AuthHelper;
import org.whispersystems.textsecuregcm.util.Base64;
@@ -87,7 +86,6 @@ public class MessageControllerTest {
private final RateLimiter rateLimiter = mock(RateLimiter.class);
private final CardinalityRateLimiter unsealedSenderLimiter = mock(CardinalityRateLimiter.class);
private final ApnFallbackManager apnFallbackManager = mock(ApnFallbackManager.class);
private final FeatureFlagsManager featureFlagsManager = mock(FeatureFlagsManager.class);
private final ObjectMapper mapper = new ObjectMapper();
@@ -281,7 +279,7 @@ public class MessageControllerTest {
OutgoingMessageEntityList messagesList = new OutgoingMessageEntityList(messages, false);
when(messagesManager.getMessagesForDevice(eq(AuthHelper.VALID_NUMBER), eq(AuthHelper.VALID_UUID), eq(1L), anyString(), anyBoolean())).thenReturn(messagesList);
when(messagesManager.getMessagesForDevice(eq(AuthHelper.VALID_UUID), eq(1L), anyString(), anyBoolean())).thenReturn(messagesList);
OutgoingMessageEntityList response =
resources.getJerseyTest().target("/v1/messages/")
@@ -318,7 +316,7 @@ public class MessageControllerTest {
OutgoingMessageEntityList messagesList = new OutgoingMessageEntityList(messages, false);
when(messagesManager.getMessagesForDevice(eq(AuthHelper.VALID_NUMBER), eq(AuthHelper.VALID_UUID), eq(1L), anyString(), anyBoolean())).thenReturn(messagesList);
when(messagesManager.getMessagesForDevice(eq(AuthHelper.VALID_UUID), eq(1L), anyString(), anyBoolean())).thenReturn(messagesList);
Response response =
resources.getJerseyTest().target("/v1/messages/")
@@ -336,20 +334,20 @@ public class MessageControllerTest {
UUID sourceUuid = UUID.randomUUID();
when(messagesManager.delete(AuthHelper.VALID_NUMBER, AuthHelper.VALID_UUID, 1, "+14152222222", 31337))
when(messagesManager.delete(AuthHelper.VALID_UUID, 1, "+14152222222", 31337))
.thenReturn(Optional.of(new OutgoingMessageEntity(31337L, true, null,
Envelope.Type.CIPHERTEXT_VALUE,
null, timestamp,
"+14152222222", sourceUuid, 1, "hi".getBytes(), null, 0)));
when(messagesManager.delete(AuthHelper.VALID_NUMBER, AuthHelper.VALID_UUID, 1, "+14152222222", 31338))
when(messagesManager.delete(AuthHelper.VALID_UUID, 1, "+14152222222", 31338))
.thenReturn(Optional.of(new OutgoingMessageEntity(31337L, true, null,
Envelope.Type.RECEIPT_VALUE,
null, System.currentTimeMillis(),
"+14152222222", sourceUuid, 1, null, null, 0)));
when(messagesManager.delete(AuthHelper.VALID_NUMBER, AuthHelper.VALID_UUID, 1, "+14152222222", 31339))
when(messagesManager.delete(AuthHelper.VALID_UUID, 1, "+14152222222", 31339))
.thenReturn(Optional.empty());
Response response = resources.getJerseyTest()

View File

@@ -1,317 +0,0 @@
/*
* Copyright 2013-2020 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.tests.storage;
import com.google.protobuf.ByteString;
import com.opentable.db.postgres.embedded.LiquibasePreparer;
import com.opentable.db.postgres.junit.EmbeddedPostgresRules;
import com.opentable.db.postgres.junit.PreparedDbRule;
import junitparams.JUnitParamsRunner;
import junitparams.Parameters;
import org.jdbi.v3.core.Jdbi;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.whispersystems.textsecuregcm.configuration.CircuitBreakerConfiguration;
import org.whispersystems.textsecuregcm.entities.MessageProtos.Envelope;
import org.whispersystems.textsecuregcm.entities.OutgoingMessageEntity;
import org.whispersystems.textsecuregcm.storage.FaultTolerantDatabase;
import org.whispersystems.textsecuregcm.storage.Messages;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.List;
import java.util.Optional;
import java.util.Queue;
import java.util.Random;
import java.util.UUID;
import java.util.concurrent.ThreadLocalRandom;
import java.util.stream.Collectors;
import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
@RunWith(JUnitParamsRunner.class)
public class MessagesTest {
@Rule
public PreparedDbRule db = EmbeddedPostgresRules.preparedDatabase(LiquibasePreparer.forClasspathLocation("messagedb.xml"));
private Messages messages;
private long serialTimestamp = 0;
@Before
public void setupAccountsDao() {
this.messages = new Messages(new FaultTolerantDatabase("messages-test", Jdbi.create(db.getTestDatabase()), new CircuitBreakerConfiguration()));
}
@Test
public void testStore() throws SQLException {
Envelope envelope = generateEnvelope();
messages.store(List.of(envelope), "+14151112222", 1);
PreparedStatement statement = db.getTestDatabase().getConnection().prepareStatement("SELECT * FROM messages WHERE destination = ?");
statement.setString(1, "+14151112222");
ResultSet resultSet = statement.executeQuery();
assertThat(resultSet.next()).isTrue();
assertThat(resultSet.getString("guid")).isEqualTo(envelope.getServerGuid());
assertThat(resultSet.getInt("type")).isEqualTo(envelope.getType().getNumber());
assertThat(resultSet.getString("relay")).isNullOrEmpty();
assertThat(resultSet.getLong("timestamp")).isEqualTo(envelope.getTimestamp());
assertThat(resultSet.getLong("server_timestamp")).isEqualTo(envelope.getServerTimestamp());
assertThat(resultSet.getString("source")).isEqualTo(envelope.getSource());
assertThat(resultSet.getLong("source_device")).isEqualTo(envelope.getSourceDevice());
assertThat(resultSet.getBytes("message")).isEqualTo(envelope.getLegacyMessage().toByteArray());
assertThat(resultSet.getBytes("content")).isEqualTo(envelope.getContent().toByteArray());
assertThat(resultSet.getString("destination")).isEqualTo("+14151112222");
assertThat(resultSet.getLong("destination_device")).isEqualTo(1);
assertThat(resultSet.next()).isFalse();
}
@Test
@Parameters(method = "argumentsForTestStoreSealedSenderBatch")
public void testStoreSealedSenderBatch(final List<Boolean> sealedSenderSequence) throws Exception {
final String destinationNumber = "+14151234567";
final List<Envelope> envelopes = sealedSenderSequence.stream()
.map(sealedSender -> {
if (sealedSender) {
return generateEnvelope().toBuilder().clearSourceUuid().clearSource().clearSourceDevice().build();
} else {
return generateEnvelope().toBuilder().setSourceUuid(UUID.randomUUID().toString()).setSource("+18005551234").setSourceDevice(4).build();
}
}).collect(Collectors.toList());
messages.store(envelopes, destinationNumber, 1);
final Queue<Envelope> expectedMessages = new ArrayDeque<>(envelopes);
try (final PreparedStatement statement = db.getTestDatabase().getConnection().prepareStatement("SELECT * FROM messages WHERE destination = ?")) {
statement.setString(1, destinationNumber);
try (final ResultSet resultSet = statement.executeQuery()) {
while (resultSet.next() && !expectedMessages.isEmpty()) {
assertRowEqualsEnvelope(resultSet, destinationNumber, expectedMessages.poll());
}
assertThat(resultSet.next()).isFalse();
assertThat(expectedMessages.isEmpty());
}
}
}
private static Object argumentsForTestStoreSealedSenderBatch() {
return new Object[] {
List.of(true),
List.of(false),
List.of(true, false),
List.of(false, true)
};
}
private void assertRowEqualsEnvelope(final ResultSet resultSet, final String expectedDestination, final Envelope expectedMessage) throws SQLException {
assertThat(resultSet.getString("guid")).isEqualTo(expectedMessage.getServerGuid());
assertThat(resultSet.getInt("type")).isEqualTo(expectedMessage.getType().getNumber());
assertThat(resultSet.getString("relay")).isNullOrEmpty();
assertThat(resultSet.getLong("timestamp")).isEqualTo(expectedMessage.getTimestamp());
assertThat(resultSet.getLong("server_timestamp")).isEqualTo(expectedMessage.getServerTimestamp());
assertThat(resultSet.getBytes("message")).isEqualTo(expectedMessage.getLegacyMessage().toByteArray());
assertThat(resultSet.getBytes("content")).isEqualTo(expectedMessage.getContent().toByteArray());
assertThat(resultSet.getString("destination")).isEqualTo(expectedDestination);
assertThat(resultSet.getLong("destination_device")).isEqualTo(1);
if (expectedMessage.hasSource()) {
assertThat(resultSet.getString("source")).isEqualTo(expectedMessage.getSource());
} else {
assertThat(resultSet.getString("source")).isNullOrEmpty();
}
if (expectedMessage.hasSourceDevice()) {
assertThat(resultSet.getLong("source_device")).isEqualTo(expectedMessage.getSourceDevice());
} else {
assertThat(resultSet.getLong("source_device")).isEqualTo(0);
}
if (expectedMessage.hasSourceUuid()) {
assertThat(resultSet.getString("source_uuid")).isEqualTo(expectedMessage.getSourceUuid());
} else {
assertThat(resultSet.getString("source_uuid")).isNull();
}
}
@Test
public void testLoad() {
List<Envelope> inserted = insertRandom("+14151112222", 1);
inserted.sort(Comparator.comparingLong(Envelope::getTimestamp));
List<OutgoingMessageEntity> retrieved = messages.load("+14151112222", 1);
assertThat(retrieved.size()).isEqualTo(inserted.size());
for (int i=0;i<retrieved.size();i++) {
verifyExpected(retrieved.get(i), inserted.get(i), UUID.fromString(inserted.get(i).getServerGuid()));
}
}
@Test
public void removeBySourceDestinationTimestamp() {
List<Envelope> inserted = insertRandom("+14151112222", 1);
List<Envelope> unrelated = insertRandom("+14151114444", 3);
Envelope toRemove = inserted.remove(new Random(System.currentTimeMillis()).nextInt(inserted.size() - 1));
Optional<OutgoingMessageEntity> removed = messages.remove("+14151112222", 1, toRemove.getSource(), toRemove.getTimestamp());
assertThat(removed.isPresent()).isTrue();
verifyExpected(removed.get(), toRemove, UUID.fromString(toRemove.getServerGuid()));
verifyInTact(inserted, "+14151112222", 1);
verifyInTact(unrelated, "+14151114444", 3);
}
@Test
public void removeByDestinationGuid() {
List<Envelope> unrelated = insertRandom("+14151113333", 2);
List<Envelope> inserted = insertRandom("+14151112222", 1);
Envelope toRemove = inserted.remove(new Random(System.currentTimeMillis()).nextInt(inserted.size() - 1));
Optional<OutgoingMessageEntity> removed = messages.remove("+14151112222", UUID.fromString(toRemove.getServerGuid()));
assertThat(removed.isPresent()).isTrue();
verifyExpected(removed.get(), toRemove, UUID.fromString(toRemove.getServerGuid()));
verifyInTact(inserted, "+14151112222", 1);
verifyInTact(unrelated, "+14151113333", 2);
}
@Test
public void removeByDestinationRowId() {
List<Envelope> unrelatedInserted = insertRandom("+14151111111", 1);
List<Envelope> inserted = insertRandom("+14151112222", 1);
inserted.sort(Comparator.comparingLong(Envelope::getTimestamp));
List<OutgoingMessageEntity> retrieved = messages.load("+14151112222", 1);
int toRemoveIndex = new Random(System.currentTimeMillis()).nextInt(inserted.size() - 1);
inserted.remove(toRemoveIndex);
messages.remove("+14151112222", retrieved.get(toRemoveIndex).getId());
verifyInTact(inserted, "+14151112222", 1);
verifyInTact(unrelatedInserted, "+14151111111", 1);
}
@Test
public void testLoadEmpty() {
insertRandom("+14151112222", 1);
assertThat(messages.load("+14159999999", 1).isEmpty()).isTrue();
}
@Test
public void testClearDestination() {
insertRandom("+14151112222", 1);
insertRandom("+14151112222", 2);
List<Envelope> unrelated = insertRandom("+14151111111", 1);
messages.clear("+14151112222");
assertThat(messages.load("+14151112222", 1).isEmpty()).isTrue();
verifyInTact(unrelated, "+14151111111", 1);
}
@Test
public void testClearDestinationDevice() {
insertRandom("+14151112222", 1);
List<Envelope> inserted = insertRandom("+14151112222", 2);
List<Envelope> unrelated = insertRandom("+14151111111", 1);
messages.clear("+14151112222", 1);
assertThat(messages.load("+14151112222", 1).isEmpty()).isTrue();
verifyInTact(inserted, "+14151112222", 2);
verifyInTact(unrelated, "+14151111111", 1);
}
@Test
public void testVacuum() {
List<Envelope> inserted = insertRandom("+14151112222", 2);
messages.vacuum();
verifyInTact(inserted, "+14151112222", 2);
}
private List<Envelope> insertRandom(String destination, int destinationDevice) {
List<Envelope> inserted = new ArrayList<>(50);
for (int i=0;i<50;i++) {
inserted.add(generateEnvelope());
}
messages.store(inserted, destination, destinationDevice);
return inserted;
}
private void verifyInTact(List<Envelope> inserted, String destination, int destinationDevice) {
inserted.sort(Comparator.comparingLong(Envelope::getTimestamp));
List<OutgoingMessageEntity> retrieved = messages.load(destination, destinationDevice);
assertThat(retrieved.size()).isEqualTo(inserted.size());
for (int i=0;i<retrieved.size();i++) {
verifyExpected(retrieved.get(i), inserted.get(i), UUID.fromString(inserted.get(i).getServerGuid()));
}
}
private void verifyExpected(OutgoingMessageEntity retrieved, Envelope inserted, UUID guid) {
assertThat(retrieved.getTimestamp()).isEqualTo(inserted.getTimestamp());
assertThat(retrieved.getSource()).isEqualTo(inserted.getSource());
assertThat(retrieved.getRelay()).isEqualTo(inserted.getRelay());
assertThat(retrieved.getType()).isEqualTo(inserted.getType().getNumber());
assertThat(retrieved.getContent()).isEqualTo(inserted.getContent().toByteArray());
assertThat(retrieved.getMessage()).isEqualTo(inserted.getLegacyMessage().toByteArray());
assertThat(retrieved.getServerTimestamp()).isEqualTo(inserted.getServerTimestamp());
assertThat(retrieved.getGuid()).isEqualTo(guid);
assertThat(retrieved.getSourceDevice()).isEqualTo(inserted.getSourceDevice());
}
private Envelope generateEnvelope() {
Random random = new Random();
byte[] content = new byte[256];
byte[] legacy = new byte[200];
Arrays.fill(content, (byte)random.nextInt(255));
Arrays.fill(legacy, (byte)random.nextInt(255));
return Envelope.newBuilder()
.setServerGuid(UUID.randomUUID().toString())
.setSourceDevice(random.nextInt(10000))
.setSource("testSource" + random.nextInt())
.setTimestamp(serialTimestamp++)
.setServerTimestamp(serialTimestamp++)
.setLegacyMessage(ByteString.copyFrom(legacy))
.setContent(ByteString.copyFrom(content))
.setType(Envelope.Type.CIPHERTEXT)
.setServerGuid(UUID.randomUUID().toString())
.build();
}
}

View File

@@ -5,36 +5,20 @@
package org.whispersystems.textsecuregcm.websocket;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyList;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.atMost;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import com.google.protobuf.ByteString;
import com.google.protobuf.InvalidProtocolBufferException;
import com.opentable.db.postgres.embedded.LiquibasePreparer;
import com.opentable.db.postgres.junit.EmbeddedPostgresRules;
import com.opentable.db.postgres.junit.PreparedDbRule;
import org.apache.commons.lang3.RandomStringUtils;
import org.jdbi.v3.core.Jdbi;
import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
import org.mockito.stubbing.Answer;
import org.whispersystems.textsecuregcm.configuration.CircuitBreakerConfiguration;
import org.whispersystems.textsecuregcm.entities.MessageProtos;
import org.whispersystems.textsecuregcm.experiment.ExperimentEnrollmentManager;
import org.whispersystems.textsecuregcm.metrics.PushLatencyManager;
import org.whispersystems.textsecuregcm.push.ReceiptSender;
import org.whispersystems.textsecuregcm.redis.AbstractRedisClusterTest;
import org.whispersystems.textsecuregcm.storage.Account;
import org.whispersystems.textsecuregcm.storage.Device;
import org.whispersystems.textsecuregcm.storage.FaultTolerantDatabase;
import org.whispersystems.textsecuregcm.storage.Messages;
import org.whispersystems.textsecuregcm.storage.MessagesCache;
import org.whispersystems.textsecuregcm.storage.MessagesDynamoDb;
import org.whispersystems.textsecuregcm.storage.MessagesManager;
import org.whispersystems.textsecuregcm.tests.util.MessagesDynamoDbRule;
import org.whispersystems.websocket.WebSocketClient;
import org.whispersystems.websocket.messages.WebSocketResponseMessage;
import java.io.IOException;
import java.time.Duration;
import java.util.ArrayList;
@@ -46,39 +30,40 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyList;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.atMost;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import org.apache.commons.lang3.RandomStringUtils;
import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
import org.mockito.stubbing.Answer;
import org.whispersystems.textsecuregcm.entities.MessageProtos;
import org.whispersystems.textsecuregcm.metrics.PushLatencyManager;
import org.whispersystems.textsecuregcm.push.ReceiptSender;
import org.whispersystems.textsecuregcm.redis.AbstractRedisClusterTest;
import org.whispersystems.textsecuregcm.storage.Account;
import org.whispersystems.textsecuregcm.storage.Device;
import org.whispersystems.textsecuregcm.storage.MessagesCache;
import org.whispersystems.textsecuregcm.storage.MessagesDynamoDb;
import org.whispersystems.textsecuregcm.storage.MessagesManager;
import org.whispersystems.textsecuregcm.tests.util.MessagesDynamoDbRule;
import org.whispersystems.websocket.WebSocketClient;
import org.whispersystems.websocket.messages.WebSocketResponseMessage;
public class WebSocketConnectionIntegrationTest extends AbstractRedisClusterTest {
@Rule
public PreparedDbRule db = EmbeddedPostgresRules.preparedDatabase(LiquibasePreparer.forClasspathLocation("messagedb.xml"));
@Rule
public MessagesDynamoDbRule messagesDynamoDbRule = new MessagesDynamoDbRule();
private ExecutorService executorService;
private Messages messages;
private MessagesDynamoDb messagesDynamoDb;
private MessagesCache messagesCache;
private Account account;
private Device device;
private WebSocketClient webSocketClient;
private WebSocketConnection webSocketConnection;
private ExperimentEnrollmentManager experimentEnrollmentManager;
private long serialTimestamp = System.currentTimeMillis();
private long serialTimestamp = System.currentTimeMillis();
@Before
public void setupAccountsDao() {
@@ -90,22 +75,19 @@ public class WebSocketConnectionIntegrationTest extends AbstractRedisClusterTest
super.setUp();
executorService = Executors.newSingleThreadExecutor();
messages = new Messages(new FaultTolerantDatabase("messages-test", Jdbi.create(db.getTestDatabase()), new CircuitBreakerConfiguration()));
messagesCache = new MessagesCache(getRedisCluster(), getRedisCluster(), executorService);
messagesDynamoDb = new MessagesDynamoDb(messagesDynamoDbRule.getDynamoDB(), MessagesDynamoDbRule.TABLE_NAME, Duration.ofDays(7));
account = mock(Account.class);
device = mock(Device.class);
webSocketClient = mock(WebSocketClient.class);
experimentEnrollmentManager = mock(ExperimentEnrollmentManager.class);
when(account.getNumber()).thenReturn("+18005551234");
when(account.getUuid()).thenReturn(UUID.randomUUID());
when(device.getId()).thenReturn(1L);
when(experimentEnrollmentManager.isEnrolled(any(UUID.class), anyString())).thenReturn(Boolean.FALSE);
webSocketConnection = new WebSocketConnection(
mock(ReceiptSender.class),
new MessagesManager(messages, messagesDynamoDb, messagesCache, mock(PushLatencyManager.class), experimentEnrollmentManager),
new MessagesManager(messagesDynamoDb, messagesCache, mock(PushLatencyManager.class)),
account,
device,
webSocketClient);
@@ -137,7 +119,7 @@ public class WebSocketConnectionIntegrationTest extends AbstractRedisClusterTest
expectedMessages.add(envelope.toBuilder().clearServerGuid().build());
}
messages.store(persistedMessages, account.getNumber(), device.getId());
messagesDynamoDb.store(persistedMessages, account.getUuid(), device.getId());
}
for (int i = 0; i < cachedMessageCount; i++) {
@@ -171,6 +153,7 @@ public class WebSocketConnectionIntegrationTest extends AbstractRedisClusterTest
}
}
@SuppressWarnings("unchecked")
final ArgumentCaptor<Optional<byte[]>> messageBodyCaptor = ArgumentCaptor.forClass(Optional.class);
verify(webSocketClient, times(persistedMessageCount + cachedMessageCount)).sendRequest(eq("PUT"), eq("/api/v1/message"), anyList(), messageBodyCaptor.capture());
@@ -203,7 +186,7 @@ public class WebSocketConnectionIntegrationTest extends AbstractRedisClusterTest
persistedMessages.add(generateRandomMessage(UUID.randomUUID()));
}
messages.store(persistedMessages, account.getNumber(), device.getId());
messagesDynamoDb.store(persistedMessages, account.getUuid(), device.getId());
}
for (int i = 0; i < cachedMessageCount; i++) {

View File

@@ -160,7 +160,7 @@ public class WebSocketConnectionTest {
String userAgent = "user-agent";
when(storedMessages.getMessagesForDevice(account.getNumber(), account.getUuid(), device.getId(), userAgent, false))
when(storedMessages.getMessagesForDevice(account.getUuid(), device.getId(), userAgent, false))
.thenReturn(outgoingMessagesList);
final List<CompletableFuture<WebSocketResponseMessage>> futures = new LinkedList<>();
@@ -192,7 +192,7 @@ public class WebSocketConnectionTest {
futures.get(0).completeExceptionally(new IOException());
futures.get(2).completeExceptionally(new IOException());
verify(storedMessages, times(1)).delete(eq(account.getNumber()), eq(accountUuid), eq(2L), eq(outgoingMessages.get(1).getGuid()));
verify(storedMessages, times(1)).delete(eq(accountUuid), eq(2L), eq(outgoingMessages.get(1).getGuid()));
verify(receiptSender, times(1)).sendReceipt(eq(account), eq("sender1"), eq(2222L));
connection.stop();
@@ -212,7 +212,7 @@ public class WebSocketConnectionTest {
when(device.getId()).thenReturn(1L);
when(client.getUserAgent()).thenReturn("Test-UA");
when(messagesManager.getMessagesForDevice(eq("+18005551234"), eq(accountUuid), eq(1L), eq("Test-UA"), anyBoolean()))
when(messagesManager.getMessagesForDevice(eq(accountUuid), eq(1L), eq("Test-UA"), anyBoolean()))
.thenReturn(new OutgoingMessageEntityList(Collections.emptyList(), false))
.thenReturn(new OutgoingMessageEntityList(List.of(createMessage(1L, false, "sender1", UUID.randomUUID(), 1111, false, "first")), false))
.thenReturn(new OutgoingMessageEntityList(List.of(createMessage(2L, false, "sender1", UUID.randomUUID(), 2222, false, "second")), false));
@@ -312,7 +312,7 @@ public class WebSocketConnectionTest {
String userAgent = "user-agent";
when(storedMessages.getMessagesForDevice(account.getNumber(), account.getUuid(), device.getId(), userAgent, false))
when(storedMessages.getMessagesForDevice(account.getUuid(), device.getId(), userAgent, false))
.thenReturn(pendingMessagesList);
final List<CompletableFuture<WebSocketResponseMessage>> futures = new LinkedList<>();
@@ -363,7 +363,7 @@ public class WebSocketConnectionTest {
final AtomicBoolean threadWaiting = new AtomicBoolean(false);
final AtomicBoolean returnMessageList = new AtomicBoolean(false);
when(messagesManager.getMessagesForDevice(account.getNumber(), account.getUuid(), 1L, client.getUserAgent(), false)).thenAnswer((Answer<OutgoingMessageEntityList>)invocation -> {
when(messagesManager.getMessagesForDevice(account.getUuid(), 1L, client.getUserAgent(), false)).thenAnswer((Answer<OutgoingMessageEntityList>)invocation -> {
synchronized (threadWaiting) {
threadWaiting.set(true);
threadWaiting.notifyAll();
@@ -407,7 +407,7 @@ public class WebSocketConnectionTest {
thread.join();
}
verify(messagesManager).getMessagesForDevice(anyString(), any(UUID.class), anyLong(), anyString(), eq(false));
verify(messagesManager).getMessagesForDevice(any(UUID.class), anyLong(), anyString(), eq(false));
}
@Test(timeout = 5000L)
@@ -431,7 +431,7 @@ public class WebSocketConnectionTest {
final OutgoingMessageEntityList firstPage = new OutgoingMessageEntityList(firstPageMessages, true);
final OutgoingMessageEntityList secondPage = new OutgoingMessageEntityList(secondPageMessages, false);
when(messagesManager.getMessagesForDevice(account.getNumber(), account.getUuid(), 1L, client.getUserAgent(), false))
when(messagesManager.getMessagesForDevice(account.getUuid(), 1L, client.getUserAgent(), false))
.thenReturn(firstPage)
.thenReturn(secondPage);
@@ -468,7 +468,7 @@ public class WebSocketConnectionTest {
final List<OutgoingMessageEntity> messages = List.of(createMessage(1L, false, "senderE164", senderUuid, 1111L, false, "message the first"));
final OutgoingMessageEntityList firstPage = new OutgoingMessageEntityList(messages, false);
when(messagesManager.getMessagesForDevice(account.getNumber(), account.getUuid(), 1L, client.getUserAgent(), false)).thenReturn(firstPage);
when(messagesManager.getMessagesForDevice(account.getUuid(), 1L, client.getUserAgent(), false)).thenReturn(firstPage);
final WebSocketResponseMessage successResponse = mock(WebSocketResponseMessage.class);
when(successResponse.getStatus()).thenReturn(200);
@@ -516,7 +516,7 @@ public class WebSocketConnectionTest {
when(device.getId()).thenReturn(1L);
when(client.getUserAgent()).thenReturn("Test-UA");
when(messagesManager.getMessagesForDevice(eq("+18005551234"), eq(accountUuid), eq(1L), eq("Test-UA"), anyBoolean()))
when(messagesManager.getMessagesForDevice(eq(accountUuid), eq(1L), eq("Test-UA"), anyBoolean()))
.thenReturn(new OutgoingMessageEntityList(Collections.emptyList(), false));
final WebSocketResponseMessage successResponse = mock(WebSocketResponseMessage.class);
@@ -554,7 +554,7 @@ public class WebSocketConnectionTest {
final OutgoingMessageEntityList firstPage = new OutgoingMessageEntityList(firstPageMessages, false);
final OutgoingMessageEntityList secondPage = new OutgoingMessageEntityList(secondPageMessages, false);
when(messagesManager.getMessagesForDevice(eq("+18005551234"), eq(accountUuid), eq(1L), eq("Test-UA"), anyBoolean()))
when(messagesManager.getMessagesForDevice(eq(accountUuid), eq(1L), eq("Test-UA"), anyBoolean()))
.thenReturn(firstPage)
.thenReturn(secondPage)
.thenReturn(new OutgoingMessageEntityList(Collections.emptyList(), false));
@@ -592,7 +592,7 @@ public class WebSocketConnectionTest {
when(device.getId()).thenReturn(1L);
when(client.getUserAgent()).thenReturn("Test-UA");
when(messagesManager.getMessagesForDevice(eq("+18005551234"), eq(accountUuid), eq(1L), eq("Test-UA"), anyBoolean()))
when(messagesManager.getMessagesForDevice(eq(accountUuid), eq(1L), eq("Test-UA"), anyBoolean()))
.thenReturn(new OutgoingMessageEntityList(Collections.emptyList(), false));
final WebSocketResponseMessage successResponse = mock(WebSocketResponseMessage.class);
@@ -604,11 +604,11 @@ public class WebSocketConnectionTest {
// anything.
connection.processStoredMessages();
verify(messagesManager).getMessagesForDevice(account.getNumber(), account.getUuid(), device.getId(), client.getUserAgent(), false);
verify(messagesManager).getMessagesForDevice(account.getUuid(), device.getId(), client.getUserAgent(), false);
connection.handleNewMessagesAvailable();
verify(messagesManager).getMessagesForDevice(account.getNumber(), account.getUuid(), device.getId(), client.getUserAgent(), true);
verify(messagesManager).getMessagesForDevice(account.getUuid(), device.getId(), client.getUserAgent(), true);
}
@Test
@@ -624,7 +624,7 @@ public class WebSocketConnectionTest {
when(device.getId()).thenReturn(1L);
when(client.getUserAgent()).thenReturn("Test-UA");
when(messagesManager.getMessagesForDevice(eq("+18005551234"), eq(accountUuid), eq(1L), eq("Test-UA"), anyBoolean()))
when(messagesManager.getMessagesForDevice(eq(accountUuid), eq(1L), eq("Test-UA"), anyBoolean()))
.thenReturn(new OutgoingMessageEntityList(Collections.emptyList(), false));
final WebSocketResponseMessage successResponse = mock(WebSocketResponseMessage.class);
@@ -637,7 +637,7 @@ public class WebSocketConnectionTest {
connection.processStoredMessages();
connection.handleMessagesPersisted();
verify(messagesManager, times(2)).getMessagesForDevice(account.getNumber(), account.getUuid(), device.getId(), client.getUserAgent(), false);
verify(messagesManager, times(2)).getMessagesForDevice(account.getUuid(), device.getId(), client.getUserAgent(), false);
}
@Test
@@ -676,7 +676,7 @@ public class WebSocketConnectionTest {
String userAgent = "Signal-Desktop/1.2.3";
when(storedMessages.getMessagesForDevice(account.getNumber(), account.getUuid(), device.getId(), userAgent, false))
when(storedMessages.getMessagesForDevice(account.getUuid(), device.getId(), userAgent, false))
.thenReturn(outgoingMessagesList);
final List<CompletableFuture<WebSocketResponseMessage>> futures = new LinkedList<>();
@@ -707,7 +707,7 @@ public class WebSocketConnectionTest {
// We should delete all three messages even though we only sent two; one got discarded because it was too big for
// desktop clients.
verify(storedMessages, times(3)).delete(eq(account.getNumber()), eq(accountUuid), eq(2L), any(UUID.class));
verify(storedMessages, times(3)).delete(eq(accountUuid), eq(2L), any(UUID.class));
connection.stop();
verify(client).close(anyInt(), anyString());
@@ -749,7 +749,7 @@ public class WebSocketConnectionTest {
String userAgent = "Signal-Android/4.68.3";
when(storedMessages.getMessagesForDevice(account.getNumber(), account.getUuid(), device.getId(), userAgent, false))
when(storedMessages.getMessagesForDevice(account.getUuid(), device.getId(), userAgent, false))
.thenReturn(outgoingMessagesList);
final List<CompletableFuture<WebSocketResponseMessage>> futures = new LinkedList<>();
@@ -779,7 +779,7 @@ public class WebSocketConnectionTest {
futures.get(1).complete(response);
futures.get(2).complete(response);
verify(storedMessages, times(3)).delete(eq(account.getNumber()), eq(accountUuid), eq(2L), any(UUID.class));
verify(storedMessages, times(3)).delete(eq(accountUuid), eq(2L), any(UUID.class));
connection.stop();
verify(client).close(anyInt(), anyString());