Introduce FaultTolerantRedisClient

service/src/test/java/org/whispersystems/textsecuregcm/redis/ClusterLuaScriptTest.java

@@ -43,7 +43,7 @@ class ClusterLuaScriptTest {
   @Test
   void testExecute() {
     final RedisAdvancedClusterCommands<String, String> commands = mock(RedisAdvancedClusterCommands.class);
-    final FaultTolerantRedisCluster mockCluster = RedisClusterHelper.builder().stringCommands(commands).build();
+    final FaultTolerantRedisClusterClient mockCluster = RedisClusterHelper.builder().stringCommands(commands).build();
 
     final String script = "return redis.call(\"SET\", KEYS[1], ARGV[1])";
     final ScriptOutputType scriptOutputType = ScriptOutputType.VALUE;
@@ -62,7 +62,7 @@ class ClusterLuaScriptTest {
   @Test
   void testExecuteScriptNotLoaded() {
     final RedisAdvancedClusterCommands<String, String> commands = mock(RedisAdvancedClusterCommands.class);
-    final FaultTolerantRedisCluster mockCluster = RedisClusterHelper.builder().stringCommands(commands).build();
+    final FaultTolerantRedisClusterClient mockCluster = RedisClusterHelper.builder().stringCommands(commands).build();
 
     final String script = "return redis.call(\"SET\", KEYS[1], ARGV[1])";
     final ScriptOutputType scriptOutputType = ScriptOutputType.VALUE;
@@ -82,7 +82,7 @@ class ClusterLuaScriptTest {
   void testExecuteBinaryScriptNotLoaded() {
     final RedisAdvancedClusterCommands<String, String> stringCommands = mock(RedisAdvancedClusterCommands.class);
     final RedisAdvancedClusterCommands<byte[], byte[]> binaryCommands = mock(RedisAdvancedClusterCommands.class);
-    final FaultTolerantRedisCluster mockCluster = RedisClusterHelper.builder()
+    final FaultTolerantRedisClusterClient mockCluster = RedisClusterHelper.builder()
         .stringCommands(stringCommands)
         .binaryCommands(binaryCommands)
         .build();
@@ -106,7 +106,7 @@ class ClusterLuaScriptTest {
   void testExecuteBinaryAsyncScriptNotLoaded() throws Exception {
     final RedisAdvancedClusterAsyncCommands<byte[], byte[]> binaryAsyncCommands =
         mock(RedisAdvancedClusterAsyncCommands.class);
-    final FaultTolerantRedisCluster mockCluster =
+    final FaultTolerantRedisClusterClient mockCluster =
         RedisClusterHelper.builder().binaryAsyncCommands(binaryAsyncCommands).build();
 
     final String script = "return redis.call(\"SET\", KEYS[1], ARGV[1])";
@@ -136,7 +136,7 @@ class ClusterLuaScriptTest {
   void testExecuteBinaryReactiveScriptNotLoaded() {
     final RedisAdvancedClusterReactiveCommands<byte[], byte[]> binaryReactiveCommands =
         mock(RedisAdvancedClusterReactiveCommands.class);
-    final FaultTolerantRedisCluster mockCluster = RedisClusterHelper.builder()
+    final FaultTolerantRedisClusterClient mockCluster = RedisClusterHelper.builder()
         .binaryReactiveCommands(binaryReactiveCommands).build();
 
     final String script = "return redis.call(\"SET\", KEYS[1], ARGV[1])";

service/src/test/java/org/whispersystems/textsecuregcm/redis/FaultTolerantPubSubClusterConnectionTest.java

@@ -40,11 +40,11 @@ import reactor.core.publisher.Flux;
 import reactor.core.scheduler.Schedulers;
 import reactor.test.publisher.TestPublisher;
 
-class FaultTolerantPubSubConnectionTest {
+class FaultTolerantPubSubClusterConnectionTest {
 
   private StatefulRedisClusterPubSubConnection<String, String> pubSubConnection;
   private RedisClusterPubSubCommands<String, String> pubSubCommands;
-  private FaultTolerantPubSubConnection<String, String> faultTolerantPubSubConnection;
+  private FaultTolerantPubSubClusterConnection<String, String> faultTolerantPubSubConnection;
 
 
   @SuppressWarnings("unchecked")
@@ -68,7 +68,7 @@ class FaultTolerantPubSubConnectionTest {
         .build();
     final Retry resubscribeRetry = Retry.of("test-resubscribe", resubscribeRetryConfiguration);
 
-    faultTolerantPubSubConnection = new FaultTolerantPubSubConnection<>("test", pubSubConnection,
+    faultTolerantPubSubConnection = new FaultTolerantPubSubClusterConnection<>("test", pubSubConnection,
         retry, resubscribeRetry, Schedulers.newSingle("test"));
   }
 

service/src/test/java/org/whispersystems/textsecuregcm/redis/FaultTolerantRedisClientTest.java

@@ -0,0 +1,102 @@
+package org.whispersystems.textsecuregcm.redis;
+
+import io.github.resilience4j.circuitbreaker.CallNotPermittedException;
+import io.lettuce.core.RedisCommandTimeoutException;
+import io.lettuce.core.RedisException;
+import io.lettuce.core.resource.ClientResources;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
+import org.junit.jupiter.api.extension.RegisterExtension;
+import org.whispersystems.textsecuregcm.configuration.CircuitBreakerConfiguration;
+import org.whispersystems.textsecuregcm.configuration.RetryConfiguration;
+
+import javax.annotation.Nullable;
+import java.time.Duration;
+import java.util.Optional;
+import java.util.concurrent.ExecutionException;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+// ThreadMode.SEPARATE_THREAD protects against hangs in the remote Redis calls, as this mode allows the test code to be
+// preempted by the timeout check
+@Timeout(value = 5, threadMode = Timeout.ThreadMode.SEPARATE_THREAD)
+class FaultTolerantRedisClientTest {
+
+  private static final Duration TIMEOUT = Duration.ofMillis(50);
+
+  private static final RetryConfiguration RETRY_CONFIGURATION = new RetryConfiguration();
+
+  static {
+    RETRY_CONFIGURATION.setMaxAttempts(1);
+    RETRY_CONFIGURATION.setWaitDuration(50);
+  }
+
+  @RegisterExtension
+  static final RedisServerExtension REDIS_SERVER_EXTENSION = RedisServerExtension.builder().build();
+
+  private FaultTolerantRedisClient faultTolerantRedisClient;
+
+  private static FaultTolerantRedisClient buildRedisClient(
+      @Nullable final CircuitBreakerConfiguration circuitBreakerConfiguration,
+      final ClientResources.Builder clientResourcesBuilder) {
+
+    return new FaultTolerantRedisClient("test", clientResourcesBuilder,
+        RedisServerExtension.getRedisURI(), TIMEOUT,
+        Optional.ofNullable(circuitBreakerConfiguration).orElseGet(CircuitBreakerConfiguration::new),
+        RETRY_CONFIGURATION);
+  }
+
+  @AfterEach
+  void tearDown() {
+    faultTolerantRedisClient.shutdown();
+  }
+
+  @Test
+  void testTimeout() {
+    faultTolerantRedisClient = buildRedisClient(null, ClientResources.builder());
+
+    final ExecutionException asyncException = assertThrows(ExecutionException.class,
+        () -> faultTolerantRedisClient.withConnection(connection -> connection.async().blpop(2 * TIMEOUT.toMillis() / 1000d, "key"))
+            .get());
+
+    assertInstanceOf(RedisCommandTimeoutException.class, asyncException.getCause());
+
+    assertThrows(RedisCommandTimeoutException.class,
+        () -> faultTolerantRedisClient.withConnection(connection -> connection.sync().blpop(2 * TIMEOUT.toMillis() / 1000d, "key")));
+  }
+
+  @Test
+  void testTimeoutCircuitBreaker() throws Exception {
+    // because we’re using a single key, and blpop involves *Redis* also blocking, the breaker wait duration must be
+    // longer than the sum of the remote timeouts
+    final Duration breakerWaitDuration = TIMEOUT.multipliedBy(5);
+
+    final CircuitBreakerConfiguration circuitBreakerConfig = new CircuitBreakerConfiguration();
+    circuitBreakerConfig.setFailureRateThreshold(1);
+    circuitBreakerConfig.setSlidingWindowMinimumNumberOfCalls(1);
+    circuitBreakerConfig.setSlidingWindowSize(1);
+    circuitBreakerConfig.setWaitDurationInOpenState(breakerWaitDuration);
+
+    faultTolerantRedisClient = buildRedisClient(circuitBreakerConfig, ClientResources.builder());
+
+    final String key = "key";
+
+    // the first call should time out and open the breaker
+    assertThrows(RedisCommandTimeoutException.class,
+        () -> faultTolerantRedisClient.withConnection(connection -> connection.sync().blpop(2 * TIMEOUT.toMillis() / 1000d, key)));
+
+    // the second call gets blocked by the breaker
+    final RedisException e = assertThrows(RedisException.class,
+        () -> faultTolerantRedisClient.withConnection(connection -> connection.sync().blpop(2 * TIMEOUT.toMillis() / 1000d, key)));
+    assertInstanceOf(CallNotPermittedException.class, e.getCause());
+
+    // wait for breaker to be half-open
+    Thread.sleep(breakerWaitDuration.toMillis() * 2);
+
+    assertEquals(0, (Long) faultTolerantRedisClient.withConnection(connection -> connection.sync().llen(key)));
+  }
+}

service/src/test/java/org/whispersystems/textsecuregcm/redis/FaultTolerantRedisClusterClientTest.java

@@ -68,7 +68,7 @@ import org.whispersystems.textsecuregcm.util.RedisClusterUtil;
 // ThreadMode.SEPARATE_THREAD protects against hangs in the remote Redis calls, as this mode allows the test code to be
 // preempted by the timeout check
 @Timeout(value = 5, threadMode = Timeout.ThreadMode.SEPARATE_THREAD)
-class FaultTolerantRedisClusterTest {
+class FaultTolerantRedisClusterClientTest {
 
   private static final Duration TIMEOUT = Duration.ofMillis(50);
 
@@ -85,13 +85,13 @@ class FaultTolerantRedisClusterTest {
       .timeout(TIMEOUT)
       .build();
 
-  private FaultTolerantRedisCluster cluster;
+  private FaultTolerantRedisClusterClient cluster;
 
-  private static FaultTolerantRedisCluster buildCluster(
+  private static FaultTolerantRedisClusterClient buildCluster(
       @Nullable final CircuitBreakerConfiguration circuitBreakerConfiguration,
       final ClientResources.Builder clientResourcesBuilder) {
 
-    return new FaultTolerantRedisCluster("test", clientResourcesBuilder,
+    return new FaultTolerantRedisClusterClient("test", clientResourcesBuilder,
         RedisClusterExtension.getRedisURIs(), TIMEOUT,
         Optional.ofNullable(circuitBreakerConfiguration).orElseGet(CircuitBreakerConfiguration::new),
         RETRY_CONFIGURATION);
@@ -235,7 +235,7 @@ class FaultTolerantRedisClusterTest {
     final int availableSlot = availableNode.getSlots().getFirst();
     final String availableKey = "key::{%s}".formatted(RedisClusterUtil.getMinimalHashTag(availableSlot));
 
-    final FaultTolerantPubSubConnection<String, String> pubSubConnection = cluster.createPubSubConnection();
+    final FaultTolerantPubSubClusterConnection<String, String> pubSubConnection = cluster.createPubSubConnection();
 
     // Keyspace notifications are delivered on a different thread, so we use a CountDownLatch to wait for the
     // expected number of notifications to arrive

service/src/test/java/org/whispersystems/textsecuregcm/redis/RedisClusterExtension.java

@@ -42,7 +42,7 @@ public class RedisClusterExtension implements BeforeAllCallback, BeforeEachCallb
 
   private final Duration timeout;
   private final RetryConfiguration retryConfiguration;
-  private FaultTolerantRedisCluster redisCluster;
+  private FaultTolerantRedisClusterClient redisCluster;
   private ClientResources redisClientResources;
 
   public RedisClusterExtension(final Duration timeout, final RetryConfiguration retryConfiguration) {
@@ -87,7 +87,7 @@ public class RedisClusterExtension implements BeforeAllCallback, BeforeEachCallb
     redisClientResources = ClientResources.builder().build();
     final CircuitBreakerConfiguration circuitBreakerConfig = new CircuitBreakerConfiguration();
     circuitBreakerConfig.setWaitDurationInOpenState(Duration.ofMillis(500));
-    redisCluster = new FaultTolerantRedisCluster("test-cluster",
+    redisCluster = new FaultTolerantRedisClusterClient("test-cluster",
         redisClientResources.mutate(),
         getRedisURIs(),
         timeout,
@@ -130,7 +130,7 @@ public class RedisClusterExtension implements BeforeAllCallback, BeforeEachCallb
         .toList();
   }
 
-  public FaultTolerantRedisCluster getRedisCluster() {
+  public FaultTolerantRedisClusterClient getRedisCluster() {
     return redisCluster;
   }
 

service/src/test/java/org/whispersystems/textsecuregcm/redis/RedisServerExtension.java

@@ -0,0 +1,99 @@
+/*
+ * Copyright 2022 Signal Messenger, LLC
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+package org.whispersystems.textsecuregcm.redis;
+
+import static org.junit.jupiter.api.Assumptions.assumeFalse;
+
+import io.lettuce.core.RedisURI;
+import java.io.IOException;
+import java.net.ServerSocket;
+import java.time.Duration;
+import io.lettuce.core.resource.ClientResources;
+import org.junit.jupiter.api.extension.AfterAllCallback;
+import org.junit.jupiter.api.extension.AfterEachCallback;
+import org.junit.jupiter.api.extension.BeforeAllCallback;
+import org.junit.jupiter.api.extension.BeforeEachCallback;
+import org.junit.jupiter.api.extension.ExtensionContext;
+import org.whispersystems.textsecuregcm.configuration.CircuitBreakerConfiguration;
+import org.whispersystems.textsecuregcm.configuration.RetryConfiguration;
+import redis.embedded.RedisServer;
+
+public class RedisServerExtension implements BeforeAllCallback, BeforeEachCallback, AfterAllCallback, AfterEachCallback {
+
+  private static RedisServer redisServer;
+  private FaultTolerantRedisClient faultTolerantRedisClient;
+  private ClientResources redisClientResources;
+
+  public static class RedisServerExtensionBuilder {
+
+    private RedisServerExtensionBuilder() {
+    }
+
+    public RedisServerExtension build() {
+      return new RedisServerExtension();
+    }
+  }
+
+  public static RedisServerExtensionBuilder builder() {
+    return new RedisServerExtensionBuilder();
+  }
+
+  @Override
+  public void beforeAll(final ExtensionContext context) throws Exception {
+    assumeFalse(System.getProperty("os.name").equalsIgnoreCase("windows"));
+
+    redisServer = RedisServer.builder()
+        .setting("appendonly no")
+        .setting("save \"\"")
+        .setting("dir " + System.getProperty("java.io.tmpdir"))
+        .port(getAvailablePort())
+        .build();
+
+    redisServer.start();
+  }
+
+  public static RedisURI getRedisURI() {
+    return RedisURI.create("redis://127.0.0.1:%d".formatted(redisServer.ports().getFirst()));
+  }
+
+  @Override
+  public void beforeEach(final ExtensionContext context) {
+    redisClientResources = ClientResources.builder().build();
+    final CircuitBreakerConfiguration circuitBreakerConfig = new CircuitBreakerConfiguration();
+    circuitBreakerConfig.setWaitDurationInOpenState(Duration.ofMillis(500));
+    faultTolerantRedisClient = new FaultTolerantRedisClient("test-redis-client",
+        redisClientResources.mutate(),
+        getRedisURI(),
+        Duration.ofSeconds(2),
+        circuitBreakerConfig,
+        new RetryConfiguration());
+
+    faultTolerantRedisClient.useConnection(connection -> connection.sync().flushall());
+  }
+
+  @Override
+  public void afterEach(final ExtensionContext context) throws InterruptedException {
+    redisClientResources.shutdown().await();
+  }
+
+  @Override
+  public void afterAll(final ExtensionContext context) {
+    if (redisServer != null) {
+      redisServer.stop();
+    }
+  }
+
+  public FaultTolerantRedisClient getRedisClient() {
+    return faultTolerantRedisClient;
+  }
+
+  private static int getAvailablePort() throws IOException {
+    try (ServerSocket socket = new ServerSocket(0)) {
+      socket.setReuseAddress(false);
+      return socket.getLocalPort();
+    }
+  }
+}

service/src/test/java/org/whispersystems/textsecuregcm/redis/RedisSingletonExtension.java

@@ -1,92 +0,0 @@
-/*
- * Copyright 2013-2022 Signal Messenger, LLC
- * SPDX-License-Identifier: AGPL-3.0-only
- */
-
-package org.whispersystems.textsecuregcm.redis;
-
-import static org.junit.jupiter.api.Assumptions.assumeFalse;
-
-import io.lettuce.core.RedisClient;
-import io.lettuce.core.RedisURI;
-import io.lettuce.core.api.StatefulRedisConnection;
-import java.io.IOException;
-import java.net.ServerSocket;
-import org.junit.jupiter.api.extension.AfterAllCallback;
-import org.junit.jupiter.api.extension.AfterEachCallback;
-import org.junit.jupiter.api.extension.BeforeAllCallback;
-import org.junit.jupiter.api.extension.BeforeEachCallback;
-import org.junit.jupiter.api.extension.ExtensionContext;
-import redis.embedded.RedisServer;
-
-public class RedisSingletonExtension implements BeforeAllCallback, BeforeEachCallback, AfterAllCallback, AfterEachCallback {
-
-  private static RedisServer redisServer;
-  private RedisClient redisClient;
-  private RedisURI redisUri;
-
-  public static class RedisSingletonExtensionBuilder {
-
-    private RedisSingletonExtensionBuilder() {
-    }
-
-    public RedisSingletonExtension build() {
-      return new RedisSingletonExtension();
-    }
-  }
-
-  public static RedisSingletonExtensionBuilder builder() {
-    return new RedisSingletonExtensionBuilder();
-  }
-
-  @Override
-  public void beforeAll(final ExtensionContext context) throws Exception {
-    assumeFalse(System.getProperty("os.name").equalsIgnoreCase("windows"));
-
-    redisServer = RedisServer.builder()
-        .setting("appendonly no")
-        .setting("save \"\"")
-        .setting("dir " + System.getProperty("java.io.tmpdir"))
-        .port(getAvailablePort())
-        .build();
-
-    redisServer.start();
-  }
-
-  @Override
-  public void beforeEach(final ExtensionContext context) {
-    redisUri = RedisURI.create("redis://127.0.0.1:%d".formatted(redisServer.ports().get(0)));
-    redisClient = RedisClient.create(redisUri);
-
-    try (final StatefulRedisConnection<String, String> connection = redisClient.connect()) {
-      connection.sync().flushall();
-    }
-  }
-
-  @Override
-  public void afterEach(final ExtensionContext context) {
-    redisClient.shutdown();
-  }
-
-  @Override
-  public void afterAll(final ExtensionContext context) {
-    if (redisServer != null) {
-      redisServer.stop();
-    }
-  }
-
-  public RedisClient getRedisClient() {
-    return redisClient;
-  }
-
-  public RedisURI getRedisUri() {
-    return redisUri;
-  }
-
-  private static int getAvailablePort() throws IOException {
-    try (ServerSocket socket = new ServerSocket(0)) {
-      socket.setReuseAddress(false);
-      return socket.getLocalPort();
-    }
-  }
-}