Signal/Signal-Server
1
0
Fork 0
mirror of https://github.com/signalapp/Signal-Server synced 2026-04-21 06:28:04 +01:00
Code Issues Packages Projects Releases Wiki Activity

Combine the read/write breakers for Redis clusters.

Browse Source
This commit is contained in:
Jon Chambers
2020-08-14 11:23:10 -04:00
committed by Jon Chambers
parent ae0f8df11b
commit 5717dc294e
24 changed files with 105 additions and 160 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
service/src/test/java/org/whispersystems/textsecuregcm/push/ClientPresenceManagerTest.java
View File

@@ -30,7 +30,7 @@ public class ClientPresenceManagerTest extends AbstractRedisClusterTest {
public void setUp() throws Exception {
super.setUp();
getRedisCluster().useWriteCluster(connection -> {
getRedisCluster().useCluster(connection -> {
connection.sync().flushall();
connection.sync().masters().commands().configSet("notify-keyspace-events", "K$z");
});
@@ -93,7 +93,7 @@ public class ClientPresenceManagerTest extends AbstractRedisClusterTest {
}
});
getRedisCluster().useWriteCluster(connection -> connection.sync().set(ClientPresenceManager.getPresenceKey(accountUuid, deviceId),
getRedisCluster().useCluster(connection -> connection.sync().set(ClientPresenceManager.getPresenceKey(accountUuid, deviceId),
UUID.randomUUID().toString()));
synchronized (displaced) {
@@ -125,7 +125,7 @@ public class ClientPresenceManagerTest extends AbstractRedisClusterTest {
clientPresenceManager.getPubSubConnection().usePubSubConnection(connection -> connection.getResources().eventBus().publish(new ClusterTopologyChangedEvent(List.of(), List.of())));
getRedisCluster().useWriteCluster(connection -> connection.sync().set(ClientPresenceManager.getPresenceKey(accountUuid, deviceId),
getRedisCluster().useCluster(connection -> connection.sync().set(ClientPresenceManager.getPresenceKey(accountUuid, deviceId),
UUID.randomUUID().toString()));
synchronized (displaced) {
@@ -149,7 +149,7 @@ public class ClientPresenceManagerTest extends AbstractRedisClusterTest {
assertTrue(clientPresenceManager.clearPresence(accountUuid, deviceId));
clientPresenceManager.setPresent(accountUuid, deviceId, NO_OP);
getRedisCluster().useWriteCluster(connection -> connection.sync().set(ClientPresenceManager.getPresenceKey(accountUuid, deviceId),
getRedisCluster().useCluster(connection -> connection.sync().set(ClientPresenceManager.getPresenceKey(accountUuid, deviceId),
UUID.randomUUID().toString()));
assertFalse(clientPresenceManager.clearPresence(accountUuid, deviceId));
@@ -160,7 +160,7 @@ public class ClientPresenceManagerTest extends AbstractRedisClusterTest {
final String presentPeerId = UUID.randomUUID().toString();
final String missingPeerId = UUID.randomUUID().toString();
getRedisCluster().useWriteCluster(connection -> {
getRedisCluster().useCluster(connection -> {
connection.sync().sadd(ClientPresenceManager.MANAGER_SET_KEY, presentPeerId);
connection.sync().sadd(ClientPresenceManager.MANAGER_SET_KEY, missingPeerId);
});
@@ -173,17 +173,17 @@ public class ClientPresenceManagerTest extends AbstractRedisClusterTest {
clientPresenceManager.getPubSubConnection().usePubSubConnection(connection -> connection.sync().masters().commands().subscribe(ClientPresenceManager.getManagerPresenceChannel(presentPeerId)));
clientPresenceManager.pruneMissingPeers();
assertEquals(1, (long)getRedisCluster().withWriteCluster(connection -> connection.sync().exists(ClientPresenceManager.getConnectedClientSetKey(presentPeerId))));
assertTrue(getRedisCluster().withReadCluster(connection -> connection.sync().sismember(ClientPresenceManager.MANAGER_SET_KEY, presentPeerId)));
assertEquals(1, (long)getRedisCluster().withCluster(connection -> connection.sync().exists(ClientPresenceManager.getConnectedClientSetKey(presentPeerId))));
assertTrue(getRedisCluster().withCluster(connection -> connection.sync().sismember(ClientPresenceManager.MANAGER_SET_KEY, presentPeerId)));
assertEquals(0, (long)getRedisCluster().withReadCluster(connection -> connection.sync().exists(ClientPresenceManager.getConnectedClientSetKey(missingPeerId))));
assertFalse(getRedisCluster().withReadCluster(connection -> connection.sync().sismember(ClientPresenceManager.MANAGER_SET_KEY, missingPeerId)));
assertEquals(0, (long)getRedisCluster().withCluster(connection -> connection.sync().exists(ClientPresenceManager.getConnectedClientSetKey(missingPeerId))));
assertFalse(getRedisCluster().withCluster(connection -> connection.sync().sismember(ClientPresenceManager.MANAGER_SET_KEY, missingPeerId)));
}
private void addClientPresence(final String managerId) {
final String clientPresenceKey = ClientPresenceManager.getPresenceKey(UUID.randomUUID(), 7);
getRedisCluster().useWriteCluster(connection -> {
getRedisCluster().useCluster(connection -> {
connection.sync().set(clientPresenceKey, managerId);
connection.sync().sadd(ClientPresenceManager.getConnectedClientSetKey(managerId), clientPresenceKey);
});
@@ -206,7 +206,7 @@ public class ClientPresenceManagerTest extends AbstractRedisClusterTest {
final long displacedAccountDeviceId = 7;
clientPresenceManager.setPresent(displacedAccountUuid, displacedAccountDeviceId, NO_OP);
getRedisCluster().useWriteCluster(connection -> connection.sync().set(ClientPresenceManager.getPresenceKey(displacedAccountUuid, displacedAccountDeviceId),
getRedisCluster().useCluster(connection -> connection.sync().set(ClientPresenceManager.getPresenceKey(displacedAccountUuid, displacedAccountDeviceId),
UUID.randomUUID().toString()));
clientPresenceManager.stop();

4
service/src/test/java/org/whispersystems/textsecuregcm/redis/AbstractRedisClusterTest.java
View File

@@ -58,7 +58,7 @@ public abstract class AbstractRedisClusterTest {
redisCluster = new FaultTolerantRedisCluster("test-cluster", urls, Duration.ofSeconds(2), new CircuitBreakerConfiguration());
redisCluster.useWriteCluster(connection -> {
redisCluster.useCluster(connection -> {
boolean setAll = false;
final String[] keys = new String[NODE_COUNT];
@@ -84,7 +84,7 @@ public abstract class AbstractRedisClusterTest {
}
});
redisCluster.useWriteCluster(connection -> connection.sync().flushall());
redisCluster.useCluster(connection -> connection.sync().flushall());
}
protected FaultTolerantRedisCluster getRedisCluster() {

17
service/src/test/java/org/whispersystems/textsecuregcm/redis/ClusterLuaScriptTest.java
View File

@@ -15,7 +15,6 @@ import java.util.List;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
@@ -33,35 +32,35 @@ public class ClusterLuaScriptTest extends AbstractRedisClusterTest {
final ClusterLuaScript script = new ClusterLuaScript(redisCluster, "return redis.call(\"SET\", KEYS[1], ARGV[1])", ScriptOutputType.VALUE);
assertEquals("OK", script.execute(List.of(key), List.of(value)));
assertEquals(value, redisCluster.withReadCluster(connection -> connection.sync().get(key)));
assertEquals(value, redisCluster.withCluster(connection -> connection.sync().get(key)));
final int slot = SlotHash.getSlot(key);
final int sourcePort = redisCluster.withWriteCluster(connection -> connection.sync().nodes(node -> node.hasSlot(slot) && node.is(RedisClusterNode.NodeFlag.MASTER)).node(0).getUri().getPort());
final RedisCommands<String, String> sourceCommands = redisCluster.withWriteCluster(connection -> connection.sync().nodes(node -> node.hasSlot(slot) && node.is(RedisClusterNode.NodeFlag.MASTER)).commands(0));
final RedisCommands<String, String> destinationCommands = redisCluster.withWriteCluster(connection -> connection.sync().nodes(node -> !node.hasSlot(slot) && node.is(RedisClusterNode.NodeFlag.MASTER)).commands(0));
final int sourcePort = redisCluster.withCluster(connection -> connection.sync().nodes(node -> node.hasSlot(slot) && node.is(RedisClusterNode.NodeFlag.MASTER)).node(0).getUri().getPort());
final RedisCommands<String, String> sourceCommands = redisCluster.withCluster(connection -> connection.sync().nodes(node -> node.hasSlot(slot) && node.is(RedisClusterNode.NodeFlag.MASTER)).commands(0));
final RedisCommands<String, String> destinationCommands = redisCluster.withCluster(connection -> connection.sync().nodes(node -> !node.hasSlot(slot) && node.is(RedisClusterNode.NodeFlag.MASTER)).commands(0));
destinationCommands.clusterSetSlotImporting(slot, sourceCommands.clusterMyId());
assertEquals("OK", script.execute(List.of(key), List.of(value)));
assertEquals(value, redisCluster.withReadCluster(connection -> connection.sync().get(key)));
assertEquals(value, redisCluster.withCluster(connection -> connection.sync().get(key)));
sourceCommands.clusterSetSlotMigrating(slot, destinationCommands.clusterMyId());
assertEquals("OK", script.execute(List.of(key), List.of(value)));
assertEquals(value, redisCluster.withReadCluster(connection -> connection.sync().get(key)));
assertEquals(value, redisCluster.withCluster(connection -> connection.sync().get(key)));
for (final String migrateKey : sourceCommands.clusterGetKeysInSlot(slot, Integer.MAX_VALUE)) {
destinationCommands.migrate("127.0.0.1", sourcePort, migrateKey, 0, 1000);
}
assertEquals("OK", script.execute(List.of(key), List.of(value)));
assertEquals(value, redisCluster.withReadCluster(connection -> connection.sync().get(key)));
assertEquals(value, redisCluster.withCluster(connection -> connection.sync().get(key)));
destinationCommands.clusterSetSlotNode(slot, destinationCommands.clusterMyId());
assertEquals("OK", script.execute(List.of(key), List.of(value)));
assertEquals(value, redisCluster.withReadCluster(connection -> connection.sync().get(key)));
assertEquals(value, redisCluster.withCluster(connection -> connection.sync().get(key)));
}
@Test

38
service/src/test/java/org/whispersystems/textsecuregcm/redis/FaultTolerantRedisClusterTest.java
View File

@@ -44,47 +44,17 @@ public class FaultTolerantRedisClusterTest {
}
@Test
public void testReadBreaker() {
public void testBreaker() {
when(clusterCommands.get(anyString()))
.thenReturn("value")
.thenThrow(new RedisException("Badness has ensued."));
assertEquals("value", faultTolerantCluster.withReadCluster(connection -> connection.sync().get("key")));
assertEquals("value", faultTolerantCluster.withCluster(connection -> connection.sync().get("key")));
assertThrows(RedisException.class,
() -> faultTolerantCluster.withReadCluster(connection -> connection.sync().get("OH NO")));
() -> faultTolerantCluster.withCluster(connection -> connection.sync().get("OH NO")));
assertThrows(CircuitBreakerOpenException.class,
() -> faultTolerantCluster.withReadCluster(connection -> connection.sync().get("OH NO")));
}
@Test
public void testReadsContinueWhileWriteBreakerOpen() {
when(clusterCommands.set(anyString(), anyString())).thenThrow(new RedisException("Badness has ensued."));
assertThrows(RedisException.class,
() -> faultTolerantCluster.useWriteCluster(connection -> connection.sync().set("OH", "NO")));
assertThrows(CircuitBreakerOpenException.class,
() -> faultTolerantCluster.useWriteCluster(connection -> connection.sync().set("OH", "NO")));
when(clusterCommands.get("key")).thenReturn("value");
assertEquals("value", faultTolerantCluster.withReadCluster(connection -> connection.sync().get("key")));
}
@Test
public void testWriteBreaker() {
when(clusterCommands.get(anyString()))
.thenReturn("value")
.thenThrow(new RedisException("Badness has ensued."));
assertEquals("value", faultTolerantCluster.withWriteCluster(connection -> connection.sync().get("key")));
assertThrows(RedisException.class,
() -> faultTolerantCluster.withWriteCluster(connection -> connection.sync().get("OH NO")));
assertThrows(CircuitBreakerOpenException.class,
() -> faultTolerantCluster.withWriteCluster(connection -> connection.sync().get("OH NO")));
() -> faultTolerantCluster.withCluster(connection -> connection.sync().get("OH NO")));
}
}

2
service/src/test/java/org/whispersystems/textsecuregcm/storage/RedisClusterMessagesCacheTest.java
View File

@@ -31,7 +31,7 @@ public class RedisClusterMessagesCacheTest extends AbstractMessagesCacheTest {
public void setUp() throws Exception {
super.setUp();
getRedisCluster().useWriteCluster(connection -> connection.sync().masters().commands().configSet("notify-keyspace-events", "K$gz"));
getRedisCluster().useCluster(connection -> connection.sync().masters().commands().configSet("notify-keyspace-events", "K$gz"));
notificationExecutorService = Executors.newSingleThreadExecutor();
messagesCache = new RedisClusterMessagesCache(getRedisCluster(), notificationExecutorService);

18
service/src/test/java/org/whispersystems/textsecuregcm/tests/storage/ActiveUserCounterTest.java
View File

@@ -19,7 +19,6 @@ package org.whispersystems.textsecuregcm.tests.storage;
import io.lettuce.core.cluster.api.sync.RedisAdvancedClusterCommands;
import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster;
import org.whispersystems.textsecuregcm.redis.ReplicatedJedisPool;
import org.whispersystems.textsecuregcm.storage.Account;
import org.whispersystems.textsecuregcm.storage.ActiveUserCounter;
import org.whispersystems.textsecuregcm.storage.AccountDatabaseCrawlerRestartException;
@@ -30,7 +29,6 @@ import com.google.common.collect.ImmutableList;
import io.dropwizard.metrics.MetricsFactory;
import org.junit.Before;
import org.junit.Test;
import redis.clients.jedis.Jedis;
import java.util.Arrays;
import java.util.UUID;
@@ -105,7 +103,7 @@ public class ActiveUserCounterTest {
public void testCrawlStart() {
activeUserCounter.onCrawlStart();
verify(cacheCluster, times(1)).useWriteCluster(any());
verify(cacheCluster, times(1)).useCluster(any());
verify(commands, times(1)).del(any(String.class));
verifyZeroInteractions(iosDevice);
@@ -122,7 +120,7 @@ public class ActiveUserCounterTest {
public void testCrawlEnd() {
activeUserCounter.onCrawlEnd(Optional.empty());
verify(cacheCluster, times(1)).withReadCluster(any());
verify(cacheCluster, times(1)).withCluster(any());
verify(commands, times(1)).get(any(String.class));
verify(metricsFactory, times(1)).getReporters();
@@ -150,8 +148,8 @@ public class ActiveUserCounterTest {
verify(iosDevice, times(1)).getApnId();
verify(iosDevice, times(0)).getGcmId();
verify(cacheCluster, times(1)).withReadCluster(any());
verify(cacheCluster, times(1)).useWriteCluster(any());
verify(cacheCluster, times(1)).withCluster(any());
verify(cacheCluster, times(1)).useCluster(any());
verify(commands, times(1)).get(any(String.class));
verify(commands, times(1)).set(any(String.class), eq("{\"fromUuid\":\""+UUID_IOS.toString()+"\",\"platforms\":{\"ios\":[1,1,1,1,1]},\"countries\":{\"1\":[1,1,1,1,1]}}"));
@@ -174,8 +172,8 @@ public class ActiveUserCounterTest {
verify(noDeviceAccount, times(1)).getMasterDevice();
verify(cacheCluster, times(1)).withReadCluster(any());
verify(cacheCluster, times(1)).useWriteCluster(any());
verify(cacheCluster, times(1)).withCluster(any());
verify(cacheCluster, times(1)).useCluster(any());
verify(commands, times(1)).get(eq(TALLY_KEY));
verify(commands, times(1)).set(any(String.class), eq("{\"fromUuid\":\""+UUID_NODEVICE+"\",\"platforms\":{},\"countries\":{}}"));
@@ -210,8 +208,8 @@ public class ActiveUserCounterTest {
verify(androidDevice, times(1)).getApnId();
verify(androidDevice, times(1)).getGcmId();
verify(cacheCluster, times(1)).withReadCluster(any());
verify(cacheCluster, times(1)).useWriteCluster(any());
verify(cacheCluster, times(1)).withCluster(any());
verify(cacheCluster, times(1)).useCluster(any());
verify(commands, times(1)).get(eq(TALLY_KEY));
verify(commands, times(1)).set(any(String.class), eq("{\"fromUuid\":\""+UUID_IOS+"\",\"platforms\":{\"android\":[0,0,0,1,1],\"ios\":[1,1,1,1,1]},\"countries\":{\"55\":[0,0,0,1,1],\"1\":[1,1,1,1,1]}}"));

16
service/src/test/java/org/whispersystems/textsecuregcm/tests/util/RedisClusterHelper.java
View File

@@ -28,41 +28,41 @@ public class RedisClusterHelper {
when(stringConnection.sync()).thenReturn(stringCommands);
when(binaryConnection.sync()).thenReturn(binaryCommands);
when(cluster.withReadCluster(any(Function.class))).thenAnswer(invocation -> {
when(cluster.withCluster(any(Function.class))).thenAnswer(invocation -> {
return invocation.getArgument(0, Function.class).apply(stringConnection);
});
doAnswer(invocation -> {
invocation.getArgument(0, Consumer.class).accept(stringConnection);
return null;
}).when(cluster).useReadCluster(any(Consumer.class));
}).when(cluster).useCluster(any(Consumer.class));
when(cluster.withWriteCluster(any(Function.class))).thenAnswer(invocation -> {
when(cluster.withCluster(any(Function.class))).thenAnswer(invocation -> {
return invocation.getArgument(0, Function.class).apply(stringConnection);
});
doAnswer(invocation -> {
invocation.getArgument(0, Consumer.class).accept(stringConnection);
return null;
}).when(cluster).useWriteCluster(any(Consumer.class));
}).when(cluster).useCluster(any(Consumer.class));
when(cluster.withBinaryReadCluster(any(Function.class))).thenAnswer(invocation -> {
when(cluster.withBinaryCluster(any(Function.class))).thenAnswer(invocation -> {
return invocation.getArgument(0, Function.class).apply(binaryConnection);
});
doAnswer(invocation -> {
invocation.getArgument(0, Consumer.class).accept(binaryConnection);
return null;
}).when(cluster).useBinaryReadCluster(any(Consumer.class));
}).when(cluster).useBinaryCluster(any(Consumer.class));
when(cluster.withBinaryWriteCluster(any(Function.class))).thenAnswer(invocation -> {
when(cluster.withBinaryCluster(any(Function.class))).thenAnswer(invocation -> {
return invocation.getArgument(0, Function.class).apply(binaryConnection);
});
doAnswer(invocation -> {
invocation.getArgument(0, Consumer.class).accept(binaryConnection);
return null;
}).when(cluster).useBinaryWriteCluster(any(Consumer.class));
}).when(cluster).useBinaryCluster(any(Consumer.class));
return cluster;
}