Introduce FaultTolerantRedisClient

service/src/test/java/org/whispersystems/textsecuregcm/configuration/LocalFaultTolerantRedisClientFactory.java

@@ -0,0 +1,49 @@
+/*
+ * Copyright 2024 Signal Messenger, LLC
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+package org.whispersystems.textsecuregcm.configuration;
+
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import io.lettuce.core.resource.ClientResources;
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisClient;
+import org.whispersystems.textsecuregcm.redis.RedisServerExtension;
+
+@JsonTypeName("local")
+public class LocalFaultTolerantRedisClientFactory implements FaultTolerantRedisClientFactory {
+
+  private static final RedisServerExtension REDIS_SERVER_EXTENSION = RedisServerExtension.builder().build();
+
+  private final AtomicBoolean shutdownHookConfigured = new AtomicBoolean();
+
+  private LocalFaultTolerantRedisClientFactory() {
+    try {
+      REDIS_SERVER_EXTENSION.beforeAll(null);
+      REDIS_SERVER_EXTENSION.beforeEach(null);
+    } catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  @Override
+  public FaultTolerantRedisClient build(final String name, final ClientResources clientResources) {
+
+    if (shutdownHookConfigured.compareAndSet(false, true)) {
+      Runtime.getRuntime().addShutdownHook(new Thread(() -> {
+        try {
+          REDIS_SERVER_EXTENSION.afterEach(null);
+          REDIS_SERVER_EXTENSION.afterAll(null);
+        } catch (Exception e) {
+          throw new RuntimeException(e);
+        }
+      }));
+    }
+
+    final RedisConfiguration config = new RedisConfiguration();
+    config.setUri(RedisServerExtension.getRedisURI().toString());
+
+    return new FaultTolerantRedisClient(name, config, clientResources.mutate());
+  }
+}

service/src/test/java/org/whispersystems/textsecuregcm/configuration/LocalFaultTolerantRedisClusterFactory.java

@@ -8,7 +8,7 @@ package org.whispersystems.textsecuregcm.configuration;
 import com.fasterxml.jackson.annotation.JsonTypeName;
 import io.lettuce.core.resource.ClientResources;
 import java.util.concurrent.atomic.AtomicBoolean;
-import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster;
+import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisClusterClient;
 import org.whispersystems.textsecuregcm.redis.RedisClusterExtension;
 
 @JsonTypeName("local")
@@ -31,7 +31,7 @@ public class LocalFaultTolerantRedisClusterFactory implements FaultTolerantRedis
   }
 
   @Override
-  public FaultTolerantRedisCluster build(final String name, final ClientResources.Builder clientResourcesBuilder) {
+  public FaultTolerantRedisClusterClient build(final String name, final ClientResources.Builder clientResourcesBuilder) {
 
     if (shutdownHookConfigured.compareAndSet(false, true)) {
       Runtime.getRuntime().addShutdownHook(new Thread(() -> {
@@ -47,7 +47,7 @@ public class LocalFaultTolerantRedisClusterFactory implements FaultTolerantRedis
     final RedisClusterConfiguration config = new RedisClusterConfiguration();
     config.setConfigurationUri(RedisClusterExtension.getRedisURIs().getFirst().toString());
 
-    return new FaultTolerantRedisCluster(name, config, clientResourcesBuilder);
+    return new FaultTolerantRedisClusterClient(name, config, clientResourcesBuilder);
   }
 
 }

service/src/test/java/org/whispersystems/textsecuregcm/configuration/LocalSingletonRedisClientFactory.java

@@ -1,52 +0,0 @@
-/*
- * Copyright 2024 Signal Messenger, LLC
- * SPDX-License-Identifier: AGPL-3.0-only
- */
-
-package org.whispersystems.textsecuregcm.configuration;
-
-import com.fasterxml.jackson.annotation.JsonTypeName;
-import io.dropwizard.lifecycle.Managed;
-import io.lettuce.core.RedisClient;
-import io.lettuce.core.resource.ClientResources;
-import java.util.concurrent.atomic.AtomicBoolean;
-import org.whispersystems.textsecuregcm.redis.RedisSingletonExtension;
-
-@JsonTypeName("local")
-public class LocalSingletonRedisClientFactory implements SingletonRedisClientFactory, Managed {
-
-  private static final RedisSingletonExtension redisSingletonExtension = RedisSingletonExtension.builder().build();
-
-  private final AtomicBoolean shutdownHookConfigured = new AtomicBoolean();
-
-  private LocalSingletonRedisClientFactory() {
-    try {
-      redisSingletonExtension.beforeAll(null);
-      redisSingletonExtension.beforeEach(null);
-    } catch (Exception e) {
-      throw new RuntimeException(e);
-    }
-  }
-
-  @Override
-  public RedisClient build(final ClientResources clientResources) {
-
-    if (shutdownHookConfigured.compareAndSet(false, true)) {
-      Runtime.getRuntime().addShutdownHook(new Thread(() -> {
-        try {
-          this.stop();
-        } catch (Exception e) {
-          throw new RuntimeException(e);
-        }
-      }));
-    }
-
-    return RedisClient.create(clientResources, redisSingletonExtension.getRedisUri());
-  }
-
-  @Override
-  public void stop() throws Exception {
-    redisSingletonExtension.afterEach(null);
-    redisSingletonExtension.afterAll(null);
-  }
-}

service/src/test/java/org/whispersystems/textsecuregcm/currency/CurrencyConversionManagerTest.java

@@ -25,7 +25,7 @@ import java.util.concurrent.ScheduledExecutorService;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.RegisterExtension;
 import org.whispersystems.textsecuregcm.entities.CurrencyConversionEntityList;
-import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster;
+import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisClusterClient;
 import org.whispersystems.textsecuregcm.redis.RedisClusterExtension;
 
 class CurrencyConversionManagerTest {
@@ -240,7 +240,7 @@ class CurrencyConversionManagerTest {
   void convertToUsd() {
     final CurrencyConversionManager currencyConversionManager = new CurrencyConversionManager(mock(FixerClient.class),
         mock(CoinMarketCapClient.class),
-        mock(FaultTolerantRedisCluster.class),
+        mock(FaultTolerantRedisClusterClient.class),
         Collections.emptyList(),
         EXECUTOR,
         Clock.systemUTC());

service/src/test/java/org/whispersystems/textsecuregcm/limits/CardinalityEstimatorTest.java

@@ -9,7 +9,7 @@ import static org.assertj.core.api.Assertions.assertThat;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.Timeout;
 import org.junit.jupiter.api.extension.RegisterExtension;
-import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster;
+import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisClusterClient;
 import org.whispersystems.textsecuregcm.redis.RedisClusterExtension;
 import java.time.Duration;
 
@@ -20,7 +20,7 @@ public class CardinalityEstimatorTest {
 
   @Test
   public void testAdd() throws Exception {
-    final FaultTolerantRedisCluster redisCluster = REDIS_CLUSTER_EXTENSION.getRedisCluster();
+    final FaultTolerantRedisClusterClient redisCluster = REDIS_CLUSTER_EXTENSION.getRedisCluster();
     final CardinalityEstimator estimator = new CardinalityEstimator(redisCluster, "test", Duration.ofSeconds(1));
 
     estimator.add("1");
@@ -40,7 +40,7 @@ public class CardinalityEstimatorTest {
   @Test
   @Timeout(5)
   public void testEventuallyExpires() throws InterruptedException {
-    final FaultTolerantRedisCluster redisCluster = REDIS_CLUSTER_EXTENSION.getRedisCluster();
+    final FaultTolerantRedisClusterClient redisCluster = REDIS_CLUSTER_EXTENSION.getRedisCluster();
     final CardinalityEstimator estimator = new CardinalityEstimator(redisCluster, "test", Duration.ofMillis(100));
     estimator.add("1");
     long count;

service/src/test/java/org/whispersystems/textsecuregcm/limits/RateLimitersLuaScriptTest.java

@@ -25,7 +25,7 @@ import org.junit.jupiter.api.extension.RegisterExtension;
 import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicConfiguration;
 import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicRateLimitPolicy;
 import org.whispersystems.textsecuregcm.controllers.RateLimitExceededException;
-import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster;
+import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisClusterClient;
 import org.whispersystems.textsecuregcm.redis.RedisClusterExtension;
 import org.whispersystems.textsecuregcm.storage.DynamicConfigurationManager;
 import org.whispersystems.textsecuregcm.util.MockUtils;
@@ -55,7 +55,7 @@ public class RateLimitersLuaScriptTest {
   @Test
   public void testWithEmbeddedRedis() throws Exception {
     final RateLimiters.For descriptor = RateLimiters.For.REGISTRATION;
-    final FaultTolerantRedisCluster redisCluster = REDIS_CLUSTER_EXTENSION.getRedisCluster();
+    final FaultTolerantRedisClusterClient redisCluster = REDIS_CLUSTER_EXTENSION.getRedisCluster();
     final RateLimiters limiters = new RateLimiters(
         Map.of(descriptor.id(), new RateLimiterConfig(60, Duration.ofSeconds(1))),
         dynamicConfig,
@@ -72,7 +72,7 @@ public class RateLimitersLuaScriptTest {
   @Test
   public void testTtl() throws Exception {
     final RateLimiters.For descriptor = RateLimiters.For.REGISTRATION;
-    final FaultTolerantRedisCluster redisCluster = REDIS_CLUSTER_EXTENSION.getRedisCluster();
+    final FaultTolerantRedisClusterClient redisCluster = REDIS_CLUSTER_EXTENSION.getRedisCluster();
     final RateLimiters limiters = new RateLimiters(
         Map.of(descriptor.id(), new RateLimiterConfig(1000, Duration.ofSeconds(1))),
         dynamicConfig,
@@ -123,7 +123,7 @@ public class RateLimitersLuaScriptTest {
   public void testFailOpen() throws Exception {
     when(configuration.getRateLimitPolicy()).thenReturn(new DynamicRateLimitPolicy(true));
     final RateLimiters.For descriptor = RateLimiters.For.REGISTRATION;
-    final FaultTolerantRedisCluster redisCluster = mock(FaultTolerantRedisCluster.class);
+    final FaultTolerantRedisClusterClient redisCluster = mock(FaultTolerantRedisClusterClient.class);
     final RateLimiters limiters = new RateLimiters(
         Map.of(descriptor.id(), new RateLimiterConfig(1000, Duration.ofSeconds(1))),
         dynamicConfig,

service/src/test/java/org/whispersystems/textsecuregcm/limits/RateLimitersTest.java

@@ -22,7 +22,7 @@ import org.junit.jupiter.api.Test;
 import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicConfiguration;
 import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicRateLimitPolicy;
 import org.whispersystems.textsecuregcm.redis.ClusterLuaScript;
-import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster;
+import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisClusterClient;
 import org.whispersystems.textsecuregcm.storage.DynamicConfigurationManager;
 import org.whispersystems.textsecuregcm.util.MockUtils;
 import org.whispersystems.textsecuregcm.util.MutableClock;
@@ -37,7 +37,7 @@ public class RateLimitersTest {
 
   private final ClusterLuaScript validateScript = mock(ClusterLuaScript.class);
 
-  private final FaultTolerantRedisCluster redisCluster = mock(FaultTolerantRedisCluster.class);
+  private final FaultTolerantRedisClusterClient redisCluster = mock(FaultTolerantRedisClusterClient.class);
 
   private final MutableClock clock = MockUtils.mutableClock(0);
 

service/src/test/java/org/whispersystems/textsecuregcm/push/ProvisioningManagerTest.java

@@ -15,8 +15,7 @@ import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.RegisterExtension;
 import org.mockito.ArgumentCaptor;
-import org.whispersystems.textsecuregcm.configuration.CircuitBreakerConfiguration;
-import org.whispersystems.textsecuregcm.redis.RedisSingletonExtension;
+import org.whispersystems.textsecuregcm.redis.RedisServerExtension;
 import org.whispersystems.textsecuregcm.storage.PubSubProtos;
 import org.whispersystems.textsecuregcm.util.TestRandomUtil;
 
@@ -25,13 +24,13 @@ class ProvisioningManagerTest {
   private ProvisioningManager provisioningManager;
 
   @RegisterExtension
-  static final RedisSingletonExtension REDIS_EXTENSION = RedisSingletonExtension.builder().build();
+  static final RedisServerExtension REDIS_EXTENSION = RedisServerExtension.builder().build();
 
   private static final long PUBSUB_TIMEOUT_MILLIS = 1_000;
 
   @BeforeEach
   void setUp() throws Exception {
-    provisioningManager = new ProvisioningManager(REDIS_EXTENSION.getRedisClient(), new CircuitBreakerConfiguration());
+    provisioningManager = new ProvisioningManager(REDIS_EXTENSION.getRedisClient());
     provisioningManager.start();
   }
 

service/src/test/java/org/whispersystems/textsecuregcm/push/PushNotificationSchedulerTest.java

@@ -31,7 +31,7 @@ import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.CsvSource;
 import org.mockito.ArgumentCaptor;
 import org.whispersystems.textsecuregcm.identity.IdentityType;
-import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster;
+import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisClusterClient;
 import org.whispersystems.textsecuregcm.redis.RedisClusterExtension;
 import org.whispersystems.textsecuregcm.storage.Account;
 import org.whispersystems.textsecuregcm.storage.AccountsManager;
@@ -247,7 +247,7 @@ class PushNotificationSchedulerTest {
   void testDedicatedProcessDynamicConfiguration(final int dedicatedThreadCount, final boolean expectActivity)
       throws Exception {
 
-    final FaultTolerantRedisCluster redisCluster = mock(FaultTolerantRedisCluster.class);
+    final FaultTolerantRedisClusterClient redisCluster = mock(FaultTolerantRedisClusterClient.class);
     when(redisCluster.withCluster(any())).thenReturn(0L);
 
     final AccountsManager accountsManager = mock(AccountsManager.class);

service/src/test/java/org/whispersystems/textsecuregcm/redis/ClusterLuaScriptTest.java

@@ -43,7 +43,7 @@ class ClusterLuaScriptTest {
   @Test
   void testExecute() {
     final RedisAdvancedClusterCommands<String, String> commands = mock(RedisAdvancedClusterCommands.class);
-    final FaultTolerantRedisCluster mockCluster = RedisClusterHelper.builder().stringCommands(commands).build();
+    final FaultTolerantRedisClusterClient mockCluster = RedisClusterHelper.builder().stringCommands(commands).build();
 
     final String script = "return redis.call(\"SET\", KEYS[1], ARGV[1])";
     final ScriptOutputType scriptOutputType = ScriptOutputType.VALUE;
@@ -62,7 +62,7 @@ class ClusterLuaScriptTest {
   @Test
   void testExecuteScriptNotLoaded() {
     final RedisAdvancedClusterCommands<String, String> commands = mock(RedisAdvancedClusterCommands.class);
-    final FaultTolerantRedisCluster mockCluster = RedisClusterHelper.builder().stringCommands(commands).build();
+    final FaultTolerantRedisClusterClient mockCluster = RedisClusterHelper.builder().stringCommands(commands).build();
 
     final String script = "return redis.call(\"SET\", KEYS[1], ARGV[1])";
     final ScriptOutputType scriptOutputType = ScriptOutputType.VALUE;
@@ -82,7 +82,7 @@ class ClusterLuaScriptTest {
   void testExecuteBinaryScriptNotLoaded() {
     final RedisAdvancedClusterCommands<String, String> stringCommands = mock(RedisAdvancedClusterCommands.class);
     final RedisAdvancedClusterCommands<byte[], byte[]> binaryCommands = mock(RedisAdvancedClusterCommands.class);
-    final FaultTolerantRedisCluster mockCluster = RedisClusterHelper.builder()
+    final FaultTolerantRedisClusterClient mockCluster = RedisClusterHelper.builder()
         .stringCommands(stringCommands)
         .binaryCommands(binaryCommands)
         .build();
@@ -106,7 +106,7 @@ class ClusterLuaScriptTest {
   void testExecuteBinaryAsyncScriptNotLoaded() throws Exception {
     final RedisAdvancedClusterAsyncCommands<byte[], byte[]> binaryAsyncCommands =
         mock(RedisAdvancedClusterAsyncCommands.class);
-    final FaultTolerantRedisCluster mockCluster =
+    final FaultTolerantRedisClusterClient mockCluster =
         RedisClusterHelper.builder().binaryAsyncCommands(binaryAsyncCommands).build();
 
     final String script = "return redis.call(\"SET\", KEYS[1], ARGV[1])";
@@ -136,7 +136,7 @@ class ClusterLuaScriptTest {
   void testExecuteBinaryReactiveScriptNotLoaded() {
     final RedisAdvancedClusterReactiveCommands<byte[], byte[]> binaryReactiveCommands =
         mock(RedisAdvancedClusterReactiveCommands.class);
-    final FaultTolerantRedisCluster mockCluster = RedisClusterHelper.builder()
+    final FaultTolerantRedisClusterClient mockCluster = RedisClusterHelper.builder()
         .binaryReactiveCommands(binaryReactiveCommands).build();
 
     final String script = "return redis.call(\"SET\", KEYS[1], ARGV[1])";

service/src/test/java/org/whispersystems/textsecuregcm/redis/FaultTolerantPubSubClusterConnectionTest.java

@@ -40,11 +40,11 @@ import reactor.core.publisher.Flux;
 import reactor.core.scheduler.Schedulers;
 import reactor.test.publisher.TestPublisher;
 
-class FaultTolerantPubSubConnectionTest {
+class FaultTolerantPubSubClusterConnectionTest {
 
   private StatefulRedisClusterPubSubConnection<String, String> pubSubConnection;
   private RedisClusterPubSubCommands<String, String> pubSubCommands;
-  private FaultTolerantPubSubConnection<String, String> faultTolerantPubSubConnection;
+  private FaultTolerantPubSubClusterConnection<String, String> faultTolerantPubSubConnection;
 
 
   @SuppressWarnings("unchecked")
@@ -68,7 +68,7 @@ class FaultTolerantPubSubConnectionTest {
         .build();
     final Retry resubscribeRetry = Retry.of("test-resubscribe", resubscribeRetryConfiguration);
 
-    faultTolerantPubSubConnection = new FaultTolerantPubSubConnection<>("test", pubSubConnection,
+    faultTolerantPubSubConnection = new FaultTolerantPubSubClusterConnection<>("test", pubSubConnection,
         retry, resubscribeRetry, Schedulers.newSingle("test"));
   }
 

service/src/test/java/org/whispersystems/textsecuregcm/redis/FaultTolerantRedisClientTest.java

@@ -0,0 +1,102 @@
+package org.whispersystems.textsecuregcm.redis;
+
+import io.github.resilience4j.circuitbreaker.CallNotPermittedException;
+import io.lettuce.core.RedisCommandTimeoutException;
+import io.lettuce.core.RedisException;
+import io.lettuce.core.resource.ClientResources;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
+import org.junit.jupiter.api.extension.RegisterExtension;
+import org.whispersystems.textsecuregcm.configuration.CircuitBreakerConfiguration;
+import org.whispersystems.textsecuregcm.configuration.RetryConfiguration;
+
+import javax.annotation.Nullable;
+import java.time.Duration;
+import java.util.Optional;
+import java.util.concurrent.ExecutionException;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+// ThreadMode.SEPARATE_THREAD protects against hangs in the remote Redis calls, as this mode allows the test code to be
+// preempted by the timeout check
+@Timeout(value = 5, threadMode = Timeout.ThreadMode.SEPARATE_THREAD)
+class FaultTolerantRedisClientTest {
+
+  private static final Duration TIMEOUT = Duration.ofMillis(50);
+
+  private static final RetryConfiguration RETRY_CONFIGURATION = new RetryConfiguration();
+
+  static {
+    RETRY_CONFIGURATION.setMaxAttempts(1);
+    RETRY_CONFIGURATION.setWaitDuration(50);
+  }
+
+  @RegisterExtension
+  static final RedisServerExtension REDIS_SERVER_EXTENSION = RedisServerExtension.builder().build();
+
+  private FaultTolerantRedisClient faultTolerantRedisClient;
+
+  private static FaultTolerantRedisClient buildRedisClient(
+      @Nullable final CircuitBreakerConfiguration circuitBreakerConfiguration,
+      final ClientResources.Builder clientResourcesBuilder) {
+
+    return new FaultTolerantRedisClient("test", clientResourcesBuilder,
+        RedisServerExtension.getRedisURI(), TIMEOUT,
+        Optional.ofNullable(circuitBreakerConfiguration).orElseGet(CircuitBreakerConfiguration::new),
+        RETRY_CONFIGURATION);
+  }
+
+  @AfterEach
+  void tearDown() {
+    faultTolerantRedisClient.shutdown();
+  }
+
+  @Test
+  void testTimeout() {
+    faultTolerantRedisClient = buildRedisClient(null, ClientResources.builder());
+
+    final ExecutionException asyncException = assertThrows(ExecutionException.class,
+        () -> faultTolerantRedisClient.withConnection(connection -> connection.async().blpop(2 * TIMEOUT.toMillis() / 1000d, "key"))
+            .get());
+
+    assertInstanceOf(RedisCommandTimeoutException.class, asyncException.getCause());
+
+    assertThrows(RedisCommandTimeoutException.class,
+        () -> faultTolerantRedisClient.withConnection(connection -> connection.sync().blpop(2 * TIMEOUT.toMillis() / 1000d, "key")));
+  }
+
+  @Test
+  void testTimeoutCircuitBreaker() throws Exception {
+    // because we’re using a single key, and blpop involves *Redis* also blocking, the breaker wait duration must be
+    // longer than the sum of the remote timeouts
+    final Duration breakerWaitDuration = TIMEOUT.multipliedBy(5);
+
+    final CircuitBreakerConfiguration circuitBreakerConfig = new CircuitBreakerConfiguration();
+    circuitBreakerConfig.setFailureRateThreshold(1);
+    circuitBreakerConfig.setSlidingWindowMinimumNumberOfCalls(1);
+    circuitBreakerConfig.setSlidingWindowSize(1);
+    circuitBreakerConfig.setWaitDurationInOpenState(breakerWaitDuration);
+
+    faultTolerantRedisClient = buildRedisClient(circuitBreakerConfig, ClientResources.builder());
+
+    final String key = "key";
+
+    // the first call should time out and open the breaker
+    assertThrows(RedisCommandTimeoutException.class,
+        () -> faultTolerantRedisClient.withConnection(connection -> connection.sync().blpop(2 * TIMEOUT.toMillis() / 1000d, key)));
+
+    // the second call gets blocked by the breaker
+    final RedisException e = assertThrows(RedisException.class,
+        () -> faultTolerantRedisClient.withConnection(connection -> connection.sync().blpop(2 * TIMEOUT.toMillis() / 1000d, key)));
+    assertInstanceOf(CallNotPermittedException.class, e.getCause());
+
+    // wait for breaker to be half-open
+    Thread.sleep(breakerWaitDuration.toMillis() * 2);
+
+    assertEquals(0, (Long) faultTolerantRedisClient.withConnection(connection -> connection.sync().llen(key)));
+  }
+}

service/src/test/java/org/whispersystems/textsecuregcm/redis/FaultTolerantRedisClusterClientTest.java

@@ -68,7 +68,7 @@ import org.whispersystems.textsecuregcm.util.RedisClusterUtil;
 // ThreadMode.SEPARATE_THREAD protects against hangs in the remote Redis calls, as this mode allows the test code to be
 // preempted by the timeout check
 @Timeout(value = 5, threadMode = Timeout.ThreadMode.SEPARATE_THREAD)
-class FaultTolerantRedisClusterTest {
+class FaultTolerantRedisClusterClientTest {
 
   private static final Duration TIMEOUT = Duration.ofMillis(50);
 
@@ -85,13 +85,13 @@ class FaultTolerantRedisClusterTest {
       .timeout(TIMEOUT)
       .build();
 
-  private FaultTolerantRedisCluster cluster;
+  private FaultTolerantRedisClusterClient cluster;
 
-  private static FaultTolerantRedisCluster buildCluster(
+  private static FaultTolerantRedisClusterClient buildCluster(
       @Nullable final CircuitBreakerConfiguration circuitBreakerConfiguration,
       final ClientResources.Builder clientResourcesBuilder) {
 
-    return new FaultTolerantRedisCluster("test", clientResourcesBuilder,
+    return new FaultTolerantRedisClusterClient("test", clientResourcesBuilder,
         RedisClusterExtension.getRedisURIs(), TIMEOUT,
         Optional.ofNullable(circuitBreakerConfiguration).orElseGet(CircuitBreakerConfiguration::new),
         RETRY_CONFIGURATION);
@@ -235,7 +235,7 @@ class FaultTolerantRedisClusterTest {
     final int availableSlot = availableNode.getSlots().getFirst();
     final String availableKey = "key::{%s}".formatted(RedisClusterUtil.getMinimalHashTag(availableSlot));
 
-    final FaultTolerantPubSubConnection<String, String> pubSubConnection = cluster.createPubSubConnection();
+    final FaultTolerantPubSubClusterConnection<String, String> pubSubConnection = cluster.createPubSubConnection();
 
     // Keyspace notifications are delivered on a different thread, so we use a CountDownLatch to wait for the
     // expected number of notifications to arrive

service/src/test/java/org/whispersystems/textsecuregcm/redis/RedisClusterExtension.java

@@ -42,7 +42,7 @@ public class RedisClusterExtension implements BeforeAllCallback, BeforeEachCallb
 
   private final Duration timeout;
   private final RetryConfiguration retryConfiguration;
-  private FaultTolerantRedisCluster redisCluster;
+  private FaultTolerantRedisClusterClient redisCluster;
   private ClientResources redisClientResources;
 
   public RedisClusterExtension(final Duration timeout, final RetryConfiguration retryConfiguration) {
@@ -87,7 +87,7 @@ public class RedisClusterExtension implements BeforeAllCallback, BeforeEachCallb
     redisClientResources = ClientResources.builder().build();
     final CircuitBreakerConfiguration circuitBreakerConfig = new CircuitBreakerConfiguration();
     circuitBreakerConfig.setWaitDurationInOpenState(Duration.ofMillis(500));
-    redisCluster = new FaultTolerantRedisCluster("test-cluster",
+    redisCluster = new FaultTolerantRedisClusterClient("test-cluster",
         redisClientResources.mutate(),
         getRedisURIs(),
         timeout,
@@ -130,7 +130,7 @@ public class RedisClusterExtension implements BeforeAllCallback, BeforeEachCallb
         .toList();
   }
 
-  public FaultTolerantRedisCluster getRedisCluster() {
+  public FaultTolerantRedisClusterClient getRedisCluster() {
     return redisCluster;
   }
 

service/src/test/java/org/whispersystems/textsecuregcm/redis/RedisServerExtension.java

@@ -0,0 +1,99 @@
+/*
+ * Copyright 2022 Signal Messenger, LLC
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+package org.whispersystems.textsecuregcm.redis;
+
+import static org.junit.jupiter.api.Assumptions.assumeFalse;
+
+import io.lettuce.core.RedisURI;
+import java.io.IOException;
+import java.net.ServerSocket;
+import java.time.Duration;
+import io.lettuce.core.resource.ClientResources;
+import org.junit.jupiter.api.extension.AfterAllCallback;
+import org.junit.jupiter.api.extension.AfterEachCallback;
+import org.junit.jupiter.api.extension.BeforeAllCallback;
+import org.junit.jupiter.api.extension.BeforeEachCallback;
+import org.junit.jupiter.api.extension.ExtensionContext;
+import org.whispersystems.textsecuregcm.configuration.CircuitBreakerConfiguration;
+import org.whispersystems.textsecuregcm.configuration.RetryConfiguration;
+import redis.embedded.RedisServer;
+
+public class RedisServerExtension implements BeforeAllCallback, BeforeEachCallback, AfterAllCallback, AfterEachCallback {
+
+  private static RedisServer redisServer;
+  private FaultTolerantRedisClient faultTolerantRedisClient;
+  private ClientResources redisClientResources;
+
+  public static class RedisServerExtensionBuilder {
+
+    private RedisServerExtensionBuilder() {
+    }
+
+    public RedisServerExtension build() {
+      return new RedisServerExtension();
+    }
+  }
+
+  public static RedisServerExtensionBuilder builder() {
+    return new RedisServerExtensionBuilder();
+  }
+
+  @Override
+  public void beforeAll(final ExtensionContext context) throws Exception {
+    assumeFalse(System.getProperty("os.name").equalsIgnoreCase("windows"));
+
+    redisServer = RedisServer.builder()
+        .setting("appendonly no")
+        .setting("save \"\"")
+        .setting("dir " + System.getProperty("java.io.tmpdir"))
+        .port(getAvailablePort())
+        .build();
+
+    redisServer.start();
+  }
+
+  public static RedisURI getRedisURI() {
+    return RedisURI.create("redis://127.0.0.1:%d".formatted(redisServer.ports().getFirst()));
+  }
+
+  @Override
+  public void beforeEach(final ExtensionContext context) {
+    redisClientResources = ClientResources.builder().build();
+    final CircuitBreakerConfiguration circuitBreakerConfig = new CircuitBreakerConfiguration();
+    circuitBreakerConfig.setWaitDurationInOpenState(Duration.ofMillis(500));
+    faultTolerantRedisClient = new FaultTolerantRedisClient("test-redis-client",
+        redisClientResources.mutate(),
+        getRedisURI(),
+        Duration.ofSeconds(2),
+        circuitBreakerConfig,
+        new RetryConfiguration());
+
+    faultTolerantRedisClient.useConnection(connection -> connection.sync().flushall());
+  }
+
+  @Override
+  public void afterEach(final ExtensionContext context) throws InterruptedException {
+    redisClientResources.shutdown().await();
+  }
+
+  @Override
+  public void afterAll(final ExtensionContext context) {
+    if (redisServer != null) {
+      redisServer.stop();
+    }
+  }
+
+  public FaultTolerantRedisClient getRedisClient() {
+    return faultTolerantRedisClient;
+  }
+
+  private static int getAvailablePort() throws IOException {
+    try (ServerSocket socket = new ServerSocket(0)) {
+      socket.setReuseAddress(false);
+      return socket.getLocalPort();
+    }
+  }
+}

service/src/test/java/org/whispersystems/textsecuregcm/redis/RedisSingletonExtension.java

@@ -1,92 +0,0 @@
-/*
- * Copyright 2013-2022 Signal Messenger, LLC
- * SPDX-License-Identifier: AGPL-3.0-only
- */
-
-package org.whispersystems.textsecuregcm.redis;
-
-import static org.junit.jupiter.api.Assumptions.assumeFalse;
-
-import io.lettuce.core.RedisClient;
-import io.lettuce.core.RedisURI;
-import io.lettuce.core.api.StatefulRedisConnection;
-import java.io.IOException;
-import java.net.ServerSocket;
-import org.junit.jupiter.api.extension.AfterAllCallback;
-import org.junit.jupiter.api.extension.AfterEachCallback;
-import org.junit.jupiter.api.extension.BeforeAllCallback;
-import org.junit.jupiter.api.extension.BeforeEachCallback;
-import org.junit.jupiter.api.extension.ExtensionContext;
-import redis.embedded.RedisServer;
-
-public class RedisSingletonExtension implements BeforeAllCallback, BeforeEachCallback, AfterAllCallback, AfterEachCallback {
-
-  private static RedisServer redisServer;
-  private RedisClient redisClient;
-  private RedisURI redisUri;
-
-  public static class RedisSingletonExtensionBuilder {
-
-    private RedisSingletonExtensionBuilder() {
-    }
-
-    public RedisSingletonExtension build() {
-      return new RedisSingletonExtension();
-    }
-  }
-
-  public static RedisSingletonExtensionBuilder builder() {
-    return new RedisSingletonExtensionBuilder();
-  }
-
-  @Override
-  public void beforeAll(final ExtensionContext context) throws Exception {
-    assumeFalse(System.getProperty("os.name").equalsIgnoreCase("windows"));
-
-    redisServer = RedisServer.builder()
-        .setting("appendonly no")
-        .setting("save \"\"")
-        .setting("dir " + System.getProperty("java.io.tmpdir"))
-        .port(getAvailablePort())
-        .build();
-
-    redisServer.start();
-  }
-
-  @Override
-  public void beforeEach(final ExtensionContext context) {
-    redisUri = RedisURI.create("redis://127.0.0.1:%d".formatted(redisServer.ports().get(0)));
-    redisClient = RedisClient.create(redisUri);
-
-    try (final StatefulRedisConnection<String, String> connection = redisClient.connect()) {
-      connection.sync().flushall();
-    }
-  }
-
-  @Override
-  public void afterEach(final ExtensionContext context) {
-    redisClient.shutdown();
-  }
-
-  @Override
-  public void afterAll(final ExtensionContext context) {
-    if (redisServer != null) {
-      redisServer.stop();
-    }
-  }
-
-  public RedisClient getRedisClient() {
-    return redisClient;
-  }
-
-  public RedisURI getRedisUri() {
-    return redisUri;
-  }
-
-  private static int getAvailablePort() throws IOException {
-    try (ServerSocket socket = new ServerSocket(0)) {
-      socket.setReuseAddress(false);
-      return socket.getLocalPort();
-    }
-  }
-}

service/src/test/java/org/whispersystems/textsecuregcm/storage/AccountsManagerTest.java

@@ -77,7 +77,7 @@ import org.whispersystems.textsecuregcm.identity.AciServiceIdentifier;
 import org.whispersystems.textsecuregcm.identity.IdentityType;
 import org.whispersystems.textsecuregcm.identity.PniServiceIdentifier;
 import org.whispersystems.textsecuregcm.push.ClientPresenceManager;
-import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster;
+import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisClusterClient;
 import org.whispersystems.textsecuregcm.securestorage.SecureStorageClient;
 import org.whispersystems.textsecuregcm.securevaluerecovery.SecureValueRecovery2Client;
 import org.whispersystems.textsecuregcm.securevaluerecovery.SecureValueRecoveryException;
@@ -230,7 +230,7 @@ class AccountsManagerTest {
 
     CLOCK = TestClock.now();
 
-    final FaultTolerantRedisCluster redisCluster = RedisClusterHelper.builder()
+    final FaultTolerantRedisClusterClient redisCluster = RedisClusterHelper.builder()
         .stringCommands(commands)
         .stringAsyncCommands(asyncCommands)
         .build();

service/src/test/java/org/whispersystems/textsecuregcm/storage/MessagesCacheTest.java

@@ -72,7 +72,7 @@ import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicMessagesCon
 import org.whispersystems.textsecuregcm.entities.MessageProtos;
 import org.whispersystems.textsecuregcm.identity.AciServiceIdentifier;
 import org.whispersystems.textsecuregcm.identity.ServiceIdentifier;
-import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster;
+import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisClusterClient;
 import org.whispersystems.textsecuregcm.redis.RedisClusterExtension;
 import org.whispersystems.textsecuregcm.tests.util.RedisClusterHelper;
 import reactor.core.publisher.Flux;
@@ -690,7 +690,7 @@ class MessagesCacheTest {
     void setup() throws Exception {
       reactiveCommands = mock(RedisAdvancedClusterReactiveCommands.class);
       asyncCommands = mock(RedisAdvancedClusterAsyncCommands.class);
-      final FaultTolerantRedisCluster mockCluster = RedisClusterHelper.builder()
+      final FaultTolerantRedisClusterClient mockCluster = RedisClusterHelper.builder()
           .binaryReactiveCommands(reactiveCommands)
           .binaryAsyncCommands(asyncCommands)
           .build();

service/src/test/java/org/whispersystems/textsecuregcm/storage/ProfilesManagerTest.java

@@ -30,7 +30,7 @@ import org.junit.jupiter.api.Timeout;
 import org.signal.libsignal.protocol.ServiceId;
 import org.signal.libsignal.zkgroup.InvalidInputException;
 import org.signal.libsignal.zkgroup.profiles.ProfileKey;
-import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster;
+import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisClusterClient;
 import org.whispersystems.textsecuregcm.tests.util.MockRedisFuture;
 import org.whispersystems.textsecuregcm.tests.util.ProfileTestHelper;
 import org.whispersystems.textsecuregcm.tests.util.RedisClusterHelper;
@@ -50,7 +50,7 @@ public class ProfilesManagerTest {
     //noinspection unchecked
     commands = mock(RedisAdvancedClusterCommands.class);
     asyncCommands = mock(RedisAdvancedClusterAsyncCommands.class);
-    final FaultTolerantRedisCluster cacheCluster = RedisClusterHelper.builder()
+    final FaultTolerantRedisClusterClient cacheCluster = RedisClusterHelper.builder()
         .stringCommands(commands)
         .stringAsyncCommands(asyncCommands)
         .build();

service/src/test/java/org/whispersystems/textsecuregcm/tests/util/RedisClusterHelper.java

@@ -16,7 +16,7 @@ import io.lettuce.core.cluster.api.reactive.RedisAdvancedClusterReactiveCommands
 import io.lettuce.core.cluster.api.sync.RedisAdvancedClusterCommands;
 import java.util.function.Consumer;
 import java.util.function.Function;
-import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster;
+import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisClusterClient;
 
 public class RedisClusterHelper {
 
@@ -25,13 +25,13 @@ public class RedisClusterHelper {
   }
 
   @SuppressWarnings("unchecked")
-  private static FaultTolerantRedisCluster buildMockRedisCluster(
+  private static FaultTolerantRedisClusterClient buildMockRedisCluster(
       final RedisAdvancedClusterCommands<String, String> stringCommands,
       final RedisAdvancedClusterAsyncCommands<String, String> stringAsyncCommands,
       final RedisAdvancedClusterCommands<byte[], byte[]> binaryCommands,
       final RedisAdvancedClusterAsyncCommands<byte[], byte[]> binaryAsyncCommands,
       final RedisAdvancedClusterReactiveCommands<byte[], byte[]> binaryReactiveCommands) {
-    final FaultTolerantRedisCluster cluster = mock(FaultTolerantRedisCluster.class);
+    final FaultTolerantRedisClusterClient cluster = mock(FaultTolerantRedisClusterClient.class);
     final StatefulRedisClusterConnection<String, String> stringConnection = mock(StatefulRedisClusterConnection.class);
     final StatefulRedisClusterConnection<byte[], byte[]> binaryConnection = mock(StatefulRedisClusterConnection.class);
 
@@ -107,7 +107,7 @@ public class RedisClusterHelper {
       return this;
     }
 
-    public FaultTolerantRedisCluster build() {
+    public FaultTolerantRedisClusterClient build() {
       return RedisClusterHelper.buildMockRedisCluster(stringCommands, stringAsyncCommands, binaryCommands, binaryAsyncCommands,
           binaryReactiveCommands);
     }