Coalesce all Redis clusters to per-shard circuit breakers

service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java

@@ -20,6 +20,7 @@ import io.dropwizard.core.server.DefaultServerFactory;
 import io.dropwizard.core.setup.Bootstrap;
 import io.dropwizard.core.setup.Environment;
 import io.dropwizard.jetty.HttpsConnectorFactory;
+import io.dropwizard.lifecycle.Managed;
 import io.grpc.ServerBuilder;
 import io.lettuce.core.metrics.MicrometerCommandLatencyRecorder;
 import io.lettuce.core.metrics.MicrometerOptions;
@@ -176,10 +177,8 @@ import org.whispersystems.textsecuregcm.push.ProvisioningManager;
 import org.whispersystems.textsecuregcm.push.PushLatencyManager;
 import org.whispersystems.textsecuregcm.push.PushNotificationManager;
 import org.whispersystems.textsecuregcm.push.ReceiptSender;
-import org.whispersystems.textsecuregcm.redis.ClusterFaultTolerantRedisCluster;
 import org.whispersystems.textsecuregcm.redis.ConnectionEventLogger;
 import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster;
-import org.whispersystems.textsecuregcm.redis.ShardFaultTolerantRedisCluster;
 import org.whispersystems.textsecuregcm.registration.RegistrationServiceClient;
 import org.whispersystems.textsecuregcm.s3.PolicySigner;
 import org.whispersystems.textsecuregcm.s3.PostPolicyGenerator;
@@ -414,25 +413,24 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
     final VerificationSessions verificationSessions = new VerificationSessions(dynamoDbAsyncClient,
         config.getDynamoDbTables().getVerificationSessions().getTableName(), clock);
 
-    final ClientResources.Builder redisClientResourcesBuilder = ClientResources.builder()
+    final ClientResources sharedClientResources = ClientResources.builder()
         .commandLatencyRecorder(
-            new MicrometerCommandLatencyRecorder(Metrics.globalRegistry, MicrometerOptions.builder().build()));
-    final ClientResources redisClientResources = redisClientResourcesBuilder.build();
+            new MicrometerCommandLatencyRecorder(Metrics.globalRegistry, MicrometerOptions.builder().build()))
+        .build();
+    ConnectionEventLogger.logConnectionEvents(sharedClientResources);
 
-    ConnectionEventLogger.logConnectionEvents(redisClientResources);
-
-    FaultTolerantRedisCluster cacheCluster = new ShardFaultTolerantRedisCluster("main_cache",
-        config.getCacheClusterConfiguration(), redisClientResourcesBuilder);
-    FaultTolerantRedisCluster messagesCluster = new ClusterFaultTolerantRedisCluster("messages_cluster",
-        config.getMessageCacheConfiguration().getRedisClusterConfiguration(), redisClientResources);
-    FaultTolerantRedisCluster clientPresenceCluster = new ShardFaultTolerantRedisCluster("client_presence",
-        config.getClientPresenceClusterConfiguration(), redisClientResourcesBuilder);
-    FaultTolerantRedisCluster metricsCluster = new ShardFaultTolerantRedisCluster("metrics",
-        config.getMetricsClusterConfiguration(), redisClientResourcesBuilder);
-    FaultTolerantRedisCluster pushSchedulerCluster = new ShardFaultTolerantRedisCluster("push_scheduler",
-        config.getPushSchedulerCluster(), redisClientResourcesBuilder);
-    FaultTolerantRedisCluster rateLimitersCluster = new ShardFaultTolerantRedisCluster("rate_limiters",
-        config.getRateLimitersCluster(), redisClientResourcesBuilder);
+    FaultTolerantRedisCluster cacheCluster = new FaultTolerantRedisCluster("main_cache",
+        config.getCacheClusterConfiguration(), sharedClientResources.mutate());
+    FaultTolerantRedisCluster messagesCluster = new FaultTolerantRedisCluster("messages",
+        config.getMessageCacheConfiguration().getRedisClusterConfiguration(), sharedClientResources.mutate());
+    FaultTolerantRedisCluster clientPresenceCluster = new FaultTolerantRedisCluster("client_presence",
+        config.getClientPresenceClusterConfiguration(), sharedClientResources.mutate());
+    FaultTolerantRedisCluster metricsCluster = new FaultTolerantRedisCluster("metrics",
+        config.getMetricsClusterConfiguration(), sharedClientResources.mutate());
+    FaultTolerantRedisCluster pushSchedulerCluster = new FaultTolerantRedisCluster("push_scheduler",
+        config.getPushSchedulerCluster(), sharedClientResources.mutate());
+    FaultTolerantRedisCluster rateLimitersCluster = new FaultTolerantRedisCluster("rate_limiters",
+        config.getRateLimitersCluster(), sharedClientResources.mutate());
 
     final BlockingQueue<Runnable> keyspaceNotificationDispatchQueue = new ArrayBlockingQueue<>(100_000);
     Metrics.gaugeCollectionSize(name(getClass(), "keyspaceNotificationDispatchQueueSize"), Collections.emptyList(),
@@ -598,7 +596,7 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
     RateLimiters rateLimiters = RateLimiters.createAndValidate(config.getLimitsConfiguration(),
         dynamicConfigurationManager, rateLimitersCluster);
     ProvisioningManager provisioningManager = new ProvisioningManager(config.getPubsubCacheConfiguration().getUri(),
-        redisClientResources, config.getPubsubCacheConfiguration().getTimeout(),
+        sharedClientResources, config.getPubsubCacheConfiguration().getTimeout(),
         config.getPubsubCacheConfiguration().getCircuitBreakerConfiguration());
     IssuedReceiptsManager issuedReceiptsManager = new IssuedReceiptsManager(
         config.getDynamoDbTables().getIssuedReceipts().getTableName(),

service/src/main/java/org/whispersystems/textsecuregcm/redis/ClusterFaultTolerantPubSubConnection.java

@@ -1,106 +0,0 @@
-/*
- * Copyright 2013-2020 Signal Messenger, LLC
- * SPDX-License-Identifier: AGPL-3.0-only
- */
-
-package org.whispersystems.textsecuregcm.redis;
-
-import static org.whispersystems.textsecuregcm.metrics.MetricsUtil.name;
-
-import io.github.resilience4j.circuitbreaker.CircuitBreaker;
-import io.github.resilience4j.retry.Retry;
-import io.lettuce.core.RedisException;
-import io.lettuce.core.cluster.event.ClusterTopologyChangedEvent;
-import io.lettuce.core.cluster.pubsub.StatefulRedisClusterPubSubConnection;
-import io.micrometer.core.instrument.Metrics;
-import io.micrometer.core.instrument.Tags;
-import io.micrometer.core.instrument.Timer;
-import java.util.function.Consumer;
-import java.util.function.Function;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.whispersystems.textsecuregcm.util.CircuitBreakerUtil;
-import reactor.core.scheduler.Scheduler;
-
-public class ClusterFaultTolerantPubSubConnection<K, V> implements FaultTolerantPubSubConnection<K, V> {
-
-  private static final Logger logger = LoggerFactory.getLogger(ClusterFaultTolerantPubSubConnection.class);
-
-
-  private final String name;
-  private final StatefulRedisClusterPubSubConnection<K, V> pubSubConnection;
-
-  private final CircuitBreaker circuitBreaker;
-  private final Retry retry;
-  private final Retry resubscribeRetry;
-  private final Scheduler topologyChangedEventScheduler;
-
-  private final Timer executeTimer;
-
-  public ClusterFaultTolerantPubSubConnection(final String name,
-      final StatefulRedisClusterPubSubConnection<K, V> pubSubConnection, final CircuitBreaker circuitBreaker,
-      final Retry retry, final Retry resubscribeRetry, final Scheduler topologyChangedEventScheduler) {
-    this.name = name;
-    this.pubSubConnection = pubSubConnection;
-    this.circuitBreaker = circuitBreaker;
-    this.retry = retry;
-    this.resubscribeRetry = resubscribeRetry;
-    this.topologyChangedEventScheduler = topologyChangedEventScheduler;
-
-    this.pubSubConnection.setNodeMessagePropagation(true);
-
-    this.executeTimer = Metrics.timer(name(getClass(), "execute"), "clusterName", name + "-pubsub");
-
-    CircuitBreakerUtil.registerMetrics(circuitBreaker, ClusterFaultTolerantPubSubConnection.class, Tags.empty());
-  }
-
-  @Override
-  public void usePubSubConnection(final Consumer<StatefulRedisClusterPubSubConnection<K, V>> consumer) {
-    try {
-      circuitBreaker.executeCheckedRunnable(
-          () -> retry.executeRunnable(() -> executeTimer.record(() -> consumer.accept(pubSubConnection))));
-    } catch (final Throwable t) {
-      if (t instanceof RedisException) {
-        throw (RedisException) t;
-      } else {
-        throw new RedisException(t);
-      }
-    }
-  }
-
-  @Override
-  public <T> T withPubSubConnection(final Function<StatefulRedisClusterPubSubConnection<K, V>, T> function) {
-    try {
-      return circuitBreaker.executeCheckedSupplier(
-          () -> retry.executeCallable(() -> executeTimer.record(() -> function.apply(pubSubConnection))));
-    } catch (final Throwable t) {
-      if (t instanceof RedisException) {
-        throw (RedisException) t;
-      } else {
-        throw new RedisException(t);
-      }
-    }
-  }
-
-
-  @Override
-  public void subscribeToClusterTopologyChangedEvents(final Runnable eventHandler) {
-
-    usePubSubConnection(connection -> connection.getResources().eventBus().get()
-        .filter(event -> event instanceof ClusterTopologyChangedEvent)
-        .subscribeOn(topologyChangedEventScheduler)
-        .subscribe(event -> {
-          logger.info("Got topology change event for {}, resubscribing all keyspace notifications", name);
-
-          resubscribeRetry.executeRunnable(() -> {
-            try {
-              eventHandler.run();
-            } catch (final RuntimeException e) {
-              logger.warn("Resubscribe for {} failed", name, e);
-              throw e;
-            }
-          });
-        }));
-  }
-
-}

service/src/main/java/org/whispersystems/textsecuregcm/redis/ClusterFaultTolerantRedisCluster.java

@@ -1,196 +0,0 @@
-/*
- * Copyright 2013-2020 Signal Messenger, LLC
- * SPDX-License-Identifier: AGPL-3.0-only
- */
-
-package org.whispersystems.textsecuregcm.redis;
-
-import com.google.common.annotations.VisibleForTesting;
-import io.github.resilience4j.circuitbreaker.CircuitBreaker;
-import io.github.resilience4j.core.IntervalFunction;
-import io.github.resilience4j.reactor.circuitbreaker.operator.CircuitBreakerOperator;
-import io.github.resilience4j.reactor.retry.RetryOperator;
-import io.github.resilience4j.retry.Retry;
-import io.github.resilience4j.retry.RetryConfig;
-import io.lettuce.core.ClientOptions.DisconnectedBehavior;
-import io.lettuce.core.RedisCommandTimeoutException;
-import io.lettuce.core.RedisException;
-import io.lettuce.core.TimeoutOptions;
-import io.lettuce.core.cluster.ClusterClientOptions;
-import io.lettuce.core.cluster.ClusterTopologyRefreshOptions;
-import io.lettuce.core.cluster.RedisClusterClient;
-import io.lettuce.core.cluster.api.StatefulRedisClusterConnection;
-import io.lettuce.core.cluster.pubsub.StatefulRedisClusterPubSubConnection;
-import io.lettuce.core.codec.ByteArrayCodec;
-import io.lettuce.core.resource.ClientResources;
-import java.time.Duration;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.function.Consumer;
-import java.util.function.Function;
-import io.micrometer.core.instrument.Tags;
-import org.reactivestreams.Publisher;
-import org.whispersystems.textsecuregcm.configuration.CircuitBreakerConfiguration;
-import org.whispersystems.textsecuregcm.configuration.RedisClusterConfiguration;
-import org.whispersystems.textsecuregcm.configuration.RetryConfiguration;
-import org.whispersystems.textsecuregcm.util.CircuitBreakerUtil;
-import reactor.core.publisher.Flux;
-import reactor.core.scheduler.Schedulers;
-
-/**
- * A fault-tolerant access manager for a Redis cluster. A single circuit breaker protects all cluster
- * calls.
- */
-public class ClusterFaultTolerantRedisCluster implements FaultTolerantRedisCluster {
-
-  private final String name;
-
-  private final RedisClusterClient clusterClient;
-
-  private final StatefulRedisClusterConnection<String, String> stringConnection;
-  private final StatefulRedisClusterConnection<byte[], byte[]> binaryConnection;
-
-  private final List<StatefulRedisClusterPubSubConnection<?, ?>> pubSubConnections = new ArrayList<>();
-
-  private final CircuitBreaker circuitBreaker;
-  private final Retry retry;
-  private final Retry topologyChangedEventRetry;
-
-  public ClusterFaultTolerantRedisCluster(final String name, final RedisClusterConfiguration clusterConfiguration,
-      final ClientResources clientResources) {
-    this(name,
-        RedisClusterClient.create(clientResources,
-            RedisUriUtil.createRedisUriWithTimeout(clusterConfiguration.getConfigurationUri(),
-                clusterConfiguration.getTimeout())),
-        clusterConfiguration.getTimeout(),
-        clusterConfiguration.getCircuitBreakerConfiguration(),
-        clusterConfiguration.getRetryConfiguration());
-  }
-
-  @VisibleForTesting
-  ClusterFaultTolerantRedisCluster(final String name, final RedisClusterClient clusterClient,
-      final Duration commandTimeout,
-      final CircuitBreakerConfiguration circuitBreakerConfiguration, final RetryConfiguration retryConfiguration) {
-    this.name = name;
-
-    this.clusterClient = clusterClient;
-    this.clusterClient.setOptions(ClusterClientOptions.builder()
-        .disconnectedBehavior(DisconnectedBehavior.REJECT_COMMANDS)
-        .validateClusterNodeMembership(false)
-        .topologyRefreshOptions(ClusterTopologyRefreshOptions.builder()
-            .enableAllAdaptiveRefreshTriggers()
-            .build())
-        // for asynchronous commands
-        .timeoutOptions(TimeoutOptions.builder()
-            .fixedTimeout(commandTimeout)
-            .build())
-        .publishOnScheduler(true)
-        .build());
-
-    this.stringConnection = clusterClient.connect();
-    this.binaryConnection = clusterClient.connect(ByteArrayCodec.INSTANCE);
-
-    this.circuitBreaker = CircuitBreaker.of(name + "-breaker", circuitBreakerConfiguration.toCircuitBreakerConfig());
-    this.retry = Retry.of(name + "-retry", retryConfiguration.toRetryConfigBuilder()
-        .retryOnException(exception -> exception instanceof RedisCommandTimeoutException).build());
-    final RetryConfig topologyChangedEventRetryConfig = RetryConfig.custom()
-        .maxAttempts(Integer.MAX_VALUE)
-        .intervalFunction(
-            IntervalFunction.ofExponentialRandomBackoff(Duration.ofSeconds(1), 1.5, Duration.ofSeconds(30)))
-        .build();
-
-    this.topologyChangedEventRetry = Retry.of(name + "-topologyChangedRetry", topologyChangedEventRetryConfig);
-
-    CircuitBreakerUtil.registerMetrics(circuitBreaker, FaultTolerantRedisCluster.class, Tags.empty());
-    CircuitBreakerUtil.registerMetrics(retry, FaultTolerantRedisCluster.class);
-  }
-
-  @Override
-  public void shutdown() {
-    stringConnection.close();
-    binaryConnection.close();
-
-    for (final StatefulRedisClusterPubSubConnection<?, ?> pubSubConnection : pubSubConnections) {
-      pubSubConnection.close();
-    }
-
-    clusterClient.shutdown();
-  }
-
-  @Override
-  public String getName() {
-    return name;
-  }
-
-  @Override
-  public void useCluster(final Consumer<StatefulRedisClusterConnection<String, String>> consumer) {
-    useConnection(stringConnection, consumer);
-  }
-
-  @Override
-  public <T> T withCluster(final Function<StatefulRedisClusterConnection<String, String>, T> function) {
-    return withConnection(stringConnection, function);
-  }
-
-  @Override
-  public void useBinaryCluster(final Consumer<StatefulRedisClusterConnection<byte[], byte[]>> consumer) {
-    useConnection(binaryConnection, consumer);
-  }
-
-  @Override
-  public <T> T withBinaryCluster(final Function<StatefulRedisClusterConnection<byte[], byte[]>, T> function) {
-    return withConnection(binaryConnection, function);
-  }
-
-  @Override
-  public <T> Publisher<T> withBinaryClusterReactive(
-      final Function<StatefulRedisClusterConnection<byte[], byte[]>, Publisher<T>> function) {
-    return withConnectionReactive(binaryConnection, function);
-  }
-
-  @Override
-  public <K, V> void useConnection(final StatefulRedisClusterConnection<K, V> connection,
-      final Consumer<StatefulRedisClusterConnection<K, V>> consumer) {
-    try {
-      circuitBreaker.executeCheckedRunnable(() -> retry.executeRunnable(() -> consumer.accept(connection)));
-    } catch (final Throwable t) {
-      if (t instanceof RedisException) {
-        throw (RedisException) t;
-      } else {
-        throw new RedisException(t);
-      }
-    }
-  }
-
-  @Override
-  public <T, K, V> T withConnection(final StatefulRedisClusterConnection<K, V> connection,
-      final Function<StatefulRedisClusterConnection<K, V>, T> function) {
-    try {
-      return circuitBreaker.executeCheckedSupplier(() -> retry.executeCallable(() -> function.apply(connection)));
-    } catch (final Throwable t) {
-      if (t instanceof RedisException) {
-        throw (RedisException) t;
-      } else {
-        throw new RedisException(t);
-      }
-    }
-  }
-
-  @Override
-  public <T, K, V> Publisher<T> withConnectionReactive(final StatefulRedisClusterConnection<K, V> connection,
-      final Function<StatefulRedisClusterConnection<K, V>, Publisher<T>> function) {
-
-    return Flux.from(function.apply(connection))
-        .transformDeferred(RetryOperator.of(retry))
-        .transformDeferred(CircuitBreakerOperator.of(circuitBreaker));
-  }
-
-  public FaultTolerantPubSubConnection<String, String> createPubSubConnection() {
-    final StatefulRedisClusterPubSubConnection<String, String> pubSubConnection = clusterClient.connectPubSub();
-    pubSubConnections.add(pubSubConnection);
-
-    return new ClusterFaultTolerantPubSubConnection<>(name, pubSubConnection, circuitBreaker, retry,
-        topologyChangedEventRetry,
-        Schedulers.newSingle(name + "-redisPubSubEvents", true));
-  }
-}

service/src/main/java/org/whispersystems/textsecuregcm/redis/FaultTolerantPubSubConnection.java

@@ -5,15 +5,90 @@
 
 package org.whispersystems.textsecuregcm.redis;
 
+import static org.whispersystems.textsecuregcm.metrics.MetricsUtil.name;
+
+import io.github.resilience4j.retry.Retry;
+import io.lettuce.core.RedisException;
+import io.lettuce.core.cluster.event.ClusterTopologyChangedEvent;
 import io.lettuce.core.cluster.pubsub.StatefulRedisClusterPubSubConnection;
+import io.micrometer.core.instrument.Metrics;
+import io.micrometer.core.instrument.Timer;
 import java.util.function.Consumer;
 import java.util.function.Function;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import reactor.core.scheduler.Scheduler;
 
-public interface FaultTolerantPubSubConnection<K, V> {
+public class FaultTolerantPubSubConnection<K, V> {
 
-  void usePubSubConnection(Consumer<StatefulRedisClusterPubSubConnection<K, V>> consumer);
+  private static final Logger logger = LoggerFactory.getLogger(FaultTolerantPubSubConnection.class);
 
-  <T> T withPubSubConnection(Function<StatefulRedisClusterPubSubConnection<K, V>, T> function);
 
-  void subscribeToClusterTopologyChangedEvents(Runnable eventHandler);
+  private final String name;
+  private final StatefulRedisClusterPubSubConnection<K, V> pubSubConnection;
+
+  private final Retry retry;
+  private final Retry resubscribeRetry;
+  private final Scheduler topologyChangedEventScheduler;
+
+  private final Timer executeTimer;
+
+  public FaultTolerantPubSubConnection(final String name,
+      final StatefulRedisClusterPubSubConnection<K, V> pubSubConnection,
+      final Retry retry, final Retry resubscribeRetry, final Scheduler topologyChangedEventScheduler) {
+    this.name = name;
+    this.pubSubConnection = pubSubConnection;
+    this.retry = retry;
+    this.resubscribeRetry = resubscribeRetry;
+    this.topologyChangedEventScheduler = topologyChangedEventScheduler;
+
+    this.pubSubConnection.setNodeMessagePropagation(true);
+
+    this.executeTimer = Metrics.timer(name(getClass(), "execute"), "clusterName", name + "-pubsub");
+  }
+
+  public void usePubSubConnection(final Consumer<StatefulRedisClusterPubSubConnection<K, V>> consumer) {
+    try {
+      retry.executeRunnable(() -> executeTimer.record(() -> consumer.accept(pubSubConnection)));
+    } catch (final Throwable t) {
+      if (t instanceof RedisException) {
+        throw (RedisException) t;
+      } else {
+        throw new RedisException(t);
+      }
+    }
+  }
+
+  public <T> T withPubSubConnection(final Function<StatefulRedisClusterPubSubConnection<K, V>, T> function) {
+    try {
+      return retry.executeCallable(() -> executeTimer.record(() -> function.apply(pubSubConnection)));
+    } catch (final Throwable t) {
+      if (t instanceof RedisException) {
+        throw (RedisException) t;
+      } else {
+        throw new RedisException(t);
+      }
+    }
+  }
+
+
+  public void subscribeToClusterTopologyChangedEvents(final Runnable eventHandler) {
+
+    usePubSubConnection(connection -> connection.getResources().eventBus().get()
+        .filter(event -> event instanceof ClusterTopologyChangedEvent)
+        .subscribeOn(topologyChangedEventScheduler)
+        .subscribe(event -> {
+          logger.info("Got topology change event for {}, resubscribing all keyspace notifications", name);
+
+          resubscribeRetry.executeRunnable(() -> {
+            try {
+              eventHandler.run();
+            } catch (final RuntimeException e) {
+              logger.warn("Resubscribe for {} failed", name, e);
+              throw e;
+            }
+          });
+        }));
+  }
+
 }

service/src/main/java/org/whispersystems/textsecuregcm/redis/FaultTolerantRedisCluster.java

@@ -5,36 +5,183 @@
 
 package org.whispersystems.textsecuregcm.redis;
 
+import io.github.resilience4j.core.IntervalFunction;
+import io.github.resilience4j.reactor.retry.RetryOperator;
+import io.github.resilience4j.retry.Retry;
+import io.github.resilience4j.retry.RetryConfig;
+import io.lettuce.core.ClientOptions;
+import io.lettuce.core.RedisCommandTimeoutException;
+import io.lettuce.core.RedisException;
+import io.lettuce.core.RedisURI;
+import io.lettuce.core.TimeoutOptions;
+import io.lettuce.core.cluster.ClusterClientOptions;
+import io.lettuce.core.cluster.ClusterTopologyRefreshOptions;
+import io.lettuce.core.cluster.RedisClusterClient;
 import io.lettuce.core.cluster.api.StatefulRedisClusterConnection;
+import io.lettuce.core.cluster.pubsub.StatefulRedisClusterPubSubConnection;
+import io.lettuce.core.codec.ByteArrayCodec;
+import io.lettuce.core.resource.ClientResources;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
 import java.util.function.Consumer;
 import java.util.function.Function;
 import org.reactivestreams.Publisher;
+import org.whispersystems.textsecuregcm.configuration.CircuitBreakerConfiguration;
+import org.whispersystems.textsecuregcm.configuration.RedisClusterConfiguration;
+import org.whispersystems.textsecuregcm.configuration.RetryConfiguration;
+import org.whispersystems.textsecuregcm.util.CircuitBreakerUtil;
+import reactor.core.publisher.Flux;
+import reactor.core.scheduler.Schedulers;
 
-public interface FaultTolerantRedisCluster {
+/**
+ * A fault-tolerant access manager for a Redis cluster. Each shard in the cluster has a dedicated circuit breaker.
+ *
+ * @see LettuceShardCircuitBreaker
+ */
+public class FaultTolerantRedisCluster {
 
-  void shutdown();
+  private final String name;
 
-  String getName();
+  private final RedisClusterClient clusterClient;
 
-  void useCluster(Consumer<StatefulRedisClusterConnection<String, String>> consumer);
+  private final StatefulRedisClusterConnection<String, String> stringConnection;
+  private final StatefulRedisClusterConnection<byte[], byte[]> binaryConnection;
 
-  <T> T withCluster(Function<StatefulRedisClusterConnection<String, String>, T> function);
+  private final List<StatefulRedisClusterPubSubConnection<?, ?>> pubSubConnections = new ArrayList<>();
 
-  void useBinaryCluster(Consumer<StatefulRedisClusterConnection<byte[], byte[]>> consumer);
+  private final Retry retry;
+  private final Retry topologyChangedEventRetry;
 
-  <T> T withBinaryCluster(Function<StatefulRedisClusterConnection<byte[], byte[]>, T> function);
 
-  <T> Publisher<T> withBinaryClusterReactive(
-      Function<StatefulRedisClusterConnection<byte[], byte[]>, Publisher<T>> function);
+  public FaultTolerantRedisCluster(final String name, final RedisClusterConfiguration clusterConfiguration,
+      final ClientResources.Builder clientResourcesBuilder) {
 
-  <K, V> void useConnection(StatefulRedisClusterConnection<K, V> connection,
-      Consumer<StatefulRedisClusterConnection<K, V>> consumer);
+    this(name, clientResourcesBuilder,
+        Collections.singleton(RedisUriUtil.createRedisUriWithTimeout(clusterConfiguration.getConfigurationUri(),
+            clusterConfiguration.getTimeout())),
+        clusterConfiguration.getTimeout(),
+        clusterConfiguration.getCircuitBreakerConfiguration(),
+        clusterConfiguration.getRetryConfiguration());
 
-  <T, K, V> T withConnection(StatefulRedisClusterConnection<K, V> connection,
-      Function<StatefulRedisClusterConnection<K, V>, T> function);
+  }
 
-  <T, K, V> Publisher<T> withConnectionReactive(StatefulRedisClusterConnection<K, V> connection,
-      Function<StatefulRedisClusterConnection<K, V>, Publisher<T>> function);
+  FaultTolerantRedisCluster(String name, final ClientResources.Builder clientResourcesBuilder,
+      Iterable<RedisURI> redisUris, Duration commandTimeout, CircuitBreakerConfiguration circuitBreakerConfig,
+      RetryConfiguration retryConfiguration) {
+
+    this.name = name;
+
+    this.clusterClient = RedisClusterClient.create(
+        clientResourcesBuilder.nettyCustomizer(
+                new LettuceShardCircuitBreaker(name, circuitBreakerConfig.toCircuitBreakerConfig())).
+            build(),
+        redisUris);
+    this.clusterClient.setOptions(ClusterClientOptions.builder()
+        .disconnectedBehavior(ClientOptions.DisconnectedBehavior.REJECT_COMMANDS)
+        .validateClusterNodeMembership(false)
+        .topologyRefreshOptions(ClusterTopologyRefreshOptions.builder()
+            .enableAllAdaptiveRefreshTriggers()
+            .build())
+        // for asynchronous commands
+        .timeoutOptions(TimeoutOptions.builder()
+            .fixedTimeout(commandTimeout)
+            .build())
+        .publishOnScheduler(true)
+        .build());
+
+    this.stringConnection = clusterClient.connect();
+    this.binaryConnection = clusterClient.connect(ByteArrayCodec.INSTANCE);
+
+    this.retry = Retry.of(name + "-retry", retryConfiguration.toRetryConfigBuilder()
+        .retryOnException(exception -> exception instanceof RedisCommandTimeoutException).build());
+    final RetryConfig topologyChangedEventRetryConfig = RetryConfig.custom()
+        .maxAttempts(Integer.MAX_VALUE)
+        .intervalFunction(
+            IntervalFunction.ofExponentialRandomBackoff(Duration.ofSeconds(1), 1.5, Duration.ofSeconds(30)))
+        .build();
+
+    this.topologyChangedEventRetry = Retry.of(name + "-topologyChangedRetry", topologyChangedEventRetryConfig);
+
+    CircuitBreakerUtil.registerMetrics(retry, FaultTolerantRedisCluster.class);
+  }
+
+  public void shutdown() {
+    stringConnection.close();
+    binaryConnection.close();
+
+    for (final StatefulRedisClusterPubSubConnection<?, ?> pubSubConnection : pubSubConnections) {
+      pubSubConnection.close();
+    }
+
+    clusterClient.shutdown();
+  }
+
+  public String getName() {
+    return name;
+  }
+
+  public void useCluster(final Consumer<StatefulRedisClusterConnection<String, String>> consumer) {
+    useConnection(stringConnection, consumer);
+  }
+
+  public <T> T withCluster(final Function<StatefulRedisClusterConnection<String, String>, T> function) {
+    return withConnection(stringConnection, function);
+  }
+
+  public void useBinaryCluster(final Consumer<StatefulRedisClusterConnection<byte[], byte[]>> consumer) {
+    useConnection(binaryConnection, consumer);
+  }
+
+  public <T> T withBinaryCluster(final Function<StatefulRedisClusterConnection<byte[], byte[]>, T> function) {
+    return withConnection(binaryConnection, function);
+  }
+
+  public <T> Publisher<T> withBinaryClusterReactive(
+      final Function<StatefulRedisClusterConnection<byte[], byte[]>, Publisher<T>> function) {
+    return withConnectionReactive(binaryConnection, function);
+  }
+
+  public <K, V> void useConnection(final StatefulRedisClusterConnection<K, V> connection,
+      final Consumer<StatefulRedisClusterConnection<K, V>> consumer) {
+    try {
+      retry.executeRunnable(() -> consumer.accept(connection));
+    } catch (final Throwable t) {
+      if (t instanceof RedisException) {
+        throw (RedisException) t;
+      } else {
+        throw new RedisException(t);
+      }
+    }
+  }
+
+  public <T, K, V> T withConnection(final StatefulRedisClusterConnection<K, V> connection,
+      final Function<StatefulRedisClusterConnection<K, V>, T> function) {
+    try {
+      return retry.executeCallable(() -> function.apply(connection));
+    } catch (final Throwable t) {
+      if (t instanceof RedisException) {
+        throw (RedisException) t;
+      } else {
+        throw new RedisException(t);
+      }
+    }
+  }
+
+  public <T, K, V> Publisher<T> withConnectionReactive(final StatefulRedisClusterConnection<K, V> connection,
+      final Function<StatefulRedisClusterConnection<K, V>, Publisher<T>> function) {
+
+    return Flux.from(function.apply(connection))
+        .transformDeferred(RetryOperator.of(retry));
+  }
+
+  public FaultTolerantPubSubConnection<String, String> createPubSubConnection() {
+    final StatefulRedisClusterPubSubConnection<String, String> pubSubConnection = clusterClient.connectPubSub();
+    pubSubConnections.add(pubSubConnection);
+
+    return new FaultTolerantPubSubConnection<>(name, pubSubConnection, retry, topologyChangedEventRetry,
+        Schedulers.newSingle(name + "-redisPubSubEvents", true));
+  }
 
-  FaultTolerantPubSubConnection<String, String> createPubSubConnection();
 }

service/src/main/java/org/whispersystems/textsecuregcm/redis/ShardFaultTolerantPubSubConnection.java

@@ -1,97 +0,0 @@
-/*
- * Copyright 2013-2020 Signal Messenger, LLC
- * SPDX-License-Identifier: AGPL-3.0-only
- */
-
-package org.whispersystems.textsecuregcm.redis;
-
-import static org.whispersystems.textsecuregcm.metrics.MetricsUtil.name;
-
-import io.github.resilience4j.retry.Retry;
-import io.lettuce.core.RedisException;
-import io.lettuce.core.cluster.event.ClusterTopologyChangedEvent;
-import io.lettuce.core.cluster.pubsub.StatefulRedisClusterPubSubConnection;
-import io.micrometer.core.instrument.Metrics;
-import io.micrometer.core.instrument.Timer;
-import java.util.function.Consumer;
-import java.util.function.Function;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import reactor.core.scheduler.Scheduler;
-
-public class ShardFaultTolerantPubSubConnection<K, V> implements FaultTolerantPubSubConnection<K, V> {
-
-  private static final Logger logger = LoggerFactory.getLogger(ShardFaultTolerantPubSubConnection.class);
-
-
-  private final String name;
-  private final StatefulRedisClusterPubSubConnection<K, V> pubSubConnection;
-
-  private final Retry retry;
-  private final Retry resubscribeRetry;
-  private final Scheduler topologyChangedEventScheduler;
-
-  private final Timer executeTimer;
-
-  public ShardFaultTolerantPubSubConnection(final String name,
-      final StatefulRedisClusterPubSubConnection<K, V> pubSubConnection,
-      final Retry retry, final Retry resubscribeRetry, final Scheduler topologyChangedEventScheduler) {
-    this.name = name;
-    this.pubSubConnection = pubSubConnection;
-    this.retry = retry;
-    this.resubscribeRetry = resubscribeRetry;
-    this.topologyChangedEventScheduler = topologyChangedEventScheduler;
-
-    this.pubSubConnection.setNodeMessagePropagation(true);
-
-    this.executeTimer = Metrics.timer(name(getClass(), "execute"), "clusterName", name + "-pubsub");
-  }
-
-  @Override
-  public void usePubSubConnection(final Consumer<StatefulRedisClusterPubSubConnection<K, V>> consumer) {
-    try {
-      retry.executeRunnable(() -> executeTimer.record(() -> consumer.accept(pubSubConnection)));
-    } catch (final Throwable t) {
-      if (t instanceof RedisException) {
-        throw (RedisException) t;
-      } else {
-        throw new RedisException(t);
-      }
-    }
-  }
-
-  @Override
-  public <T> T withPubSubConnection(final Function<StatefulRedisClusterPubSubConnection<K, V>, T> function) {
-    try {
-      return retry.executeCallable(() -> executeTimer.record(() -> function.apply(pubSubConnection)));
-    } catch (final Throwable t) {
-      if (t instanceof RedisException) {
-        throw (RedisException) t;
-      } else {
-        throw new RedisException(t);
-      }
-    }
-  }
-
-
-  @Override
-  public void subscribeToClusterTopologyChangedEvents(final Runnable eventHandler) {
-
-    usePubSubConnection(connection -> connection.getResources().eventBus().get()
-        .filter(event -> event instanceof ClusterTopologyChangedEvent)
-        .subscribeOn(topologyChangedEventScheduler)
-        .subscribe(event -> {
-          logger.info("Got topology change event for {}, resubscribing all keyspace notifications", name);
-
-          resubscribeRetry.executeRunnable(() -> {
-            try {
-              eventHandler.run();
-            } catch (final RuntimeException e) {
-              logger.warn("Resubscribe for {} failed", name, e);
-              throw e;
-            }
-          });
-        }));
-  }
-
-}

service/src/main/java/org/whispersystems/textsecuregcm/redis/ShardFaultTolerantRedisCluster.java

@@ -1,197 +0,0 @@
-/*
- * Copyright 2024 Signal Messenger, LLC
- * SPDX-License-Identifier: AGPL-3.0-only
- */
-
-package org.whispersystems.textsecuregcm.redis;
-
-import io.github.resilience4j.core.IntervalFunction;
-import io.github.resilience4j.reactor.retry.RetryOperator;
-import io.github.resilience4j.retry.Retry;
-import io.github.resilience4j.retry.RetryConfig;
-import io.lettuce.core.ClientOptions;
-import io.lettuce.core.RedisCommandTimeoutException;
-import io.lettuce.core.RedisException;
-import io.lettuce.core.RedisURI;
-import io.lettuce.core.TimeoutOptions;
-import io.lettuce.core.cluster.ClusterClientOptions;
-import io.lettuce.core.cluster.ClusterTopologyRefreshOptions;
-import io.lettuce.core.cluster.RedisClusterClient;
-import io.lettuce.core.cluster.api.StatefulRedisClusterConnection;
-import io.lettuce.core.cluster.pubsub.StatefulRedisClusterPubSubConnection;
-import io.lettuce.core.codec.ByteArrayCodec;
-import io.lettuce.core.resource.ClientResources;
-import java.time.Duration;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.function.Consumer;
-import java.util.function.Function;
-import org.reactivestreams.Publisher;
-import org.whispersystems.textsecuregcm.configuration.CircuitBreakerConfiguration;
-import org.whispersystems.textsecuregcm.configuration.RedisClusterConfiguration;
-import org.whispersystems.textsecuregcm.configuration.RetryConfiguration;
-import org.whispersystems.textsecuregcm.util.CircuitBreakerUtil;
-import reactor.core.publisher.Flux;
-import reactor.core.scheduler.Schedulers;
-
-/**
- * A fault-tolerant access manager for a Redis cluster. Each shard in the cluster has a dedicated circuit breaker.
- *
- * @see LettuceShardCircuitBreaker
- */
-public class ShardFaultTolerantRedisCluster implements FaultTolerantRedisCluster {
-
-  private final String name;
-
-  private final RedisClusterClient clusterClient;
-
-  private final StatefulRedisClusterConnection<String, String> stringConnection;
-  private final StatefulRedisClusterConnection<byte[], byte[]> binaryConnection;
-
-  private final List<StatefulRedisClusterPubSubConnection<?, ?>> pubSubConnections = new ArrayList<>();
-
-  private final Retry retry;
-  private final Retry topologyChangedEventRetry;
-
-
-  public ShardFaultTolerantRedisCluster(final String name, final RedisClusterConfiguration clusterConfiguration,
-      final ClientResources.Builder clientResourcesBuilder) {
-
-    this(name, clientResourcesBuilder,
-            Collections.singleton(RedisUriUtil.createRedisUriWithTimeout(clusterConfiguration.getConfigurationUri(),
-                clusterConfiguration.getTimeout())),
-        clusterConfiguration.getTimeout(),
-        clusterConfiguration.getCircuitBreakerConfiguration(),
-        clusterConfiguration.getRetryConfiguration());
-
-  }
-
-  ShardFaultTolerantRedisCluster(String name, final ClientResources.Builder clientResourcesBuilder,
-      Iterable<RedisURI> redisUris, Duration commandTimeout, CircuitBreakerConfiguration circuitBreakerConfig,
-      RetryConfiguration retryConfiguration) {
-
-    this.name = name;
-
-    this.clusterClient = RedisClusterClient.create(
-        clientResourcesBuilder.nettyCustomizer(
-                new LettuceShardCircuitBreaker(name, circuitBreakerConfig.toCircuitBreakerConfig())).
-            build(),
-        redisUris);
-    this.clusterClient.setOptions(ClusterClientOptions.builder()
-        .disconnectedBehavior(ClientOptions.DisconnectedBehavior.REJECT_COMMANDS)
-        .validateClusterNodeMembership(false)
-        .topologyRefreshOptions(ClusterTopologyRefreshOptions.builder()
-            .enableAllAdaptiveRefreshTriggers()
-            .build())
-        // for asynchronous commands
-        .timeoutOptions(TimeoutOptions.builder()
-            .fixedTimeout(commandTimeout)
-            .build())
-        .publishOnScheduler(true)
-        .build());
-
-    this.stringConnection = clusterClient.connect();
-    this.binaryConnection = clusterClient.connect(ByteArrayCodec.INSTANCE);
-
-    this.retry = Retry.of(name + "-retry", retryConfiguration.toRetryConfigBuilder()
-        .retryOnException(exception -> exception instanceof RedisCommandTimeoutException).build());
-    final RetryConfig topologyChangedEventRetryConfig = RetryConfig.custom()
-        .maxAttempts(Integer.MAX_VALUE)
-        .intervalFunction(
-            IntervalFunction.ofExponentialRandomBackoff(Duration.ofSeconds(1), 1.5, Duration.ofSeconds(30)))
-        .build();
-
-    this.topologyChangedEventRetry = Retry.of(name + "-topologyChangedRetry", topologyChangedEventRetryConfig);
-
-    CircuitBreakerUtil.registerMetrics(retry, ShardFaultTolerantRedisCluster.class);
-  }
-
-  @Override
-  public void shutdown() {
-    stringConnection.close();
-    binaryConnection.close();
-
-    for (final StatefulRedisClusterPubSubConnection<?, ?> pubSubConnection : pubSubConnections) {
-      pubSubConnection.close();
-    }
-
-    clusterClient.shutdown();
-  }
-
-  public String getName() {
-    return name;
-  }
-
-  @Override
-  public void useCluster(final Consumer<StatefulRedisClusterConnection<String, String>> consumer) {
-    useConnection(stringConnection, consumer);
-  }
-
-  @Override
-  public <T> T withCluster(final Function<StatefulRedisClusterConnection<String, String>, T> function) {
-    return withConnection(stringConnection, function);
-  }
-
-  @Override
-  public void useBinaryCluster(final Consumer<StatefulRedisClusterConnection<byte[], byte[]>> consumer) {
-    useConnection(binaryConnection, consumer);
-  }
-
-  @Override
-  public <T> T withBinaryCluster(final Function<StatefulRedisClusterConnection<byte[], byte[]>, T> function) {
-    return withConnection(binaryConnection, function);
-  }
-
-  @Override
-  public <T> Publisher<T> withBinaryClusterReactive(
-      final Function<StatefulRedisClusterConnection<byte[], byte[]>, Publisher<T>> function) {
-    return withConnectionReactive(binaryConnection, function);
-  }
-
-  @Override
-  public <K, V> void useConnection(final StatefulRedisClusterConnection<K, V> connection,
-      final Consumer<StatefulRedisClusterConnection<K, V>> consumer) {
-    try {
-      retry.executeRunnable(() -> consumer.accept(connection));
-    } catch (final Throwable t) {
-      if (t instanceof RedisException) {
-        throw (RedisException) t;
-      } else {
-        throw new RedisException(t);
-      }
-    }
-  }
-
-  @Override
-  public <T, K, V> T withConnection(final StatefulRedisClusterConnection<K, V> connection,
-      final Function<StatefulRedisClusterConnection<K, V>, T> function) {
-    try {
-      return retry.executeCallable(() -> function.apply(connection));
-    } catch (final Throwable t) {
-      if (t instanceof RedisException) {
-        throw (RedisException) t;
-      } else {
-        throw new RedisException(t);
-      }
-    }
-  }
-
-  @Override
-  public <T, K, V> Publisher<T> withConnectionReactive(final StatefulRedisClusterConnection<K, V> connection,
-      final Function<StatefulRedisClusterConnection<K, V>, Publisher<T>> function) {
-
-    return Flux.from(function.apply(connection))
-        .transformDeferred(RetryOperator.of(retry));
-  }
-
-  @Override
-  public FaultTolerantPubSubConnection<String, String> createPubSubConnection() {
-    final StatefulRedisClusterPubSubConnection<String, String> pubSubConnection = clusterClient.connectPubSub();
-    pubSubConnections.add(pubSubConnection);
-
-    return new ShardFaultTolerantPubSubConnection<>(name, pubSubConnection, retry, topologyChangedEventRetry,
-        Schedulers.newSingle(name + "-redisPubSubEvents", true));
-  }
-
-}

service/src/main/java/org/whispersystems/textsecuregcm/workers/AssignUsernameCommand.java

@@ -29,9 +29,7 @@ import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicConfigurati
 import org.whispersystems.textsecuregcm.controllers.SecureStorageController;
 import org.whispersystems.textsecuregcm.controllers.SecureValueRecovery2Controller;
 import org.whispersystems.textsecuregcm.push.ClientPresenceManager;
-import org.whispersystems.textsecuregcm.redis.ClusterFaultTolerantRedisCluster;
 import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster;
-import org.whispersystems.textsecuregcm.redis.ShardFaultTolerantRedisCluster;
 import org.whispersystems.textsecuregcm.securestorage.SecureStorageClient;
 import org.whispersystems.textsecuregcm.securevaluerecovery.SecureValueRecovery2Client;
 import org.whispersystems.textsecuregcm.storage.Account;
@@ -108,9 +106,8 @@ public class AssignUsernameCommand extends EnvironmentCommand<WhisperServerConfi
     dynamicConfigurationManager.start();
 
     final ClientResources.Builder redisClientResourcesBuilder = ClientResources.builder();
-    final ClientResources redisClusterClientResources = redisClientResourcesBuilder.build();
 
-    FaultTolerantRedisCluster cacheCluster = new ShardFaultTolerantRedisCluster("main_cache_cluster",
+    FaultTolerantRedisCluster cacheCluster = new FaultTolerantRedisCluster("main_cache_cluster",
         configuration.getCacheClusterConfiguration(), redisClientResourcesBuilder);
 
     Scheduler messageDeliveryScheduler = Schedulers.fromExecutorService(
@@ -176,14 +173,14 @@ public class AssignUsernameCommand extends EnvironmentCommand<WhisperServerConfi
         configuration.getDynamoDbTables().getMessages().getTableName(),
         configuration.getDynamoDbTables().getMessages().getExpiration(),
         messageDeletionExecutor);
-    FaultTolerantRedisCluster messageInsertCacheCluster = new ClusterFaultTolerantRedisCluster("message_insert_cluster",
-        configuration.getMessageCacheConfiguration().getRedisClusterConfiguration(), redisClusterClientResources);
-    FaultTolerantRedisCluster messageReadDeleteCluster = new ClusterFaultTolerantRedisCluster(
+    FaultTolerantRedisCluster messageInsertCacheCluster = new FaultTolerantRedisCluster("message_insert_cluster",
+        configuration.getMessageCacheConfiguration().getRedisClusterConfiguration(), redisClientResourcesBuilder);
+    FaultTolerantRedisCluster messageReadDeleteCluster = new FaultTolerantRedisCluster(
         "message_read_delete_cluster",
-        configuration.getMessageCacheConfiguration().getRedisClusterConfiguration(), redisClusterClientResources);
-    FaultTolerantRedisCluster clientPresenceCluster = new ShardFaultTolerantRedisCluster("client_presence",
+        configuration.getMessageCacheConfiguration().getRedisClusterConfiguration(), redisClientResourcesBuilder);
+    FaultTolerantRedisCluster clientPresenceCluster = new FaultTolerantRedisCluster("client_presence",
         configuration.getClientPresenceClusterConfiguration(), redisClientResourcesBuilder);
-    FaultTolerantRedisCluster rateLimitersCluster = new ShardFaultTolerantRedisCluster("rate_limiters",
+    FaultTolerantRedisCluster rateLimitersCluster = new FaultTolerantRedisCluster("rate_limiters",
         configuration.getRateLimitersCluster(), redisClientResourcesBuilder);
     SecureValueRecovery2Client secureValueRecovery2Client = new SecureValueRecovery2Client(
         secureValueRecoveryCredentialsGenerator, secureValueRecoveryExecutor, secureValueRecoveryServiceRetryExecutor,

service/src/main/java/org/whispersystems/textsecuregcm/workers/CommandDependencies.java

@@ -31,9 +31,7 @@ import org.whispersystems.textsecuregcm.controllers.SecureValueRecovery2Controll
 import org.whispersystems.textsecuregcm.limits.RateLimiters;
 import org.whispersystems.textsecuregcm.metrics.MetricsUtil;
 import org.whispersystems.textsecuregcm.push.ClientPresenceManager;
-import org.whispersystems.textsecuregcm.redis.ClusterFaultTolerantRedisCluster;
 import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster;
-import org.whispersystems.textsecuregcm.redis.ShardFaultTolerantRedisCluster;
 import org.whispersystems.textsecuregcm.securestorage.SecureStorageClient;
 import org.whispersystems.textsecuregcm.securevaluerecovery.SecureValueRecovery2Client;
 import org.whispersystems.textsecuregcm.storage.AccountLockManager;
@@ -93,9 +91,8 @@ record CommandDependencies(
     MetricsUtil.configureRegistries(configuration, environment, dynamicConfigurationManager);
 
     final ClientResources.Builder redisClientResourcesBuilder = ClientResources.builder();
-    ClientResources redisClusterClientResources = redisClientResourcesBuilder.build();
 
-    FaultTolerantRedisCluster cacheCluster = new ShardFaultTolerantRedisCluster("main_cache",
+    FaultTolerantRedisCluster cacheCluster = new FaultTolerantRedisCluster("main_cache",
         configuration.getCacheClusterConfiguration(), redisClientResourcesBuilder);
 
     ScheduledExecutorService recurringJobExecutor = environment.lifecycle()
@@ -166,14 +163,14 @@ record CommandDependencies(
         configuration.getDynamoDbTables().getMessages().getTableName(),
         configuration.getDynamoDbTables().getMessages().getExpiration(),
         messageDeletionExecutor);
-    FaultTolerantRedisCluster messageInsertCacheCluster = new ClusterFaultTolerantRedisCluster("message_insert_cluster",
-        configuration.getMessageCacheConfiguration().getRedisClusterConfiguration(), redisClusterClientResources);
-    FaultTolerantRedisCluster messageReadDeleteCluster = new ClusterFaultTolerantRedisCluster(
+    FaultTolerantRedisCluster messageInsertCacheCluster = new FaultTolerantRedisCluster("message_insert_cluster",
+        configuration.getMessageCacheConfiguration().getRedisClusterConfiguration(), redisClientResourcesBuilder);
+    FaultTolerantRedisCluster messageReadDeleteCluster = new FaultTolerantRedisCluster(
         "message_read_delete_cluster",
-        configuration.getMessageCacheConfiguration().getRedisClusterConfiguration(), redisClusterClientResources);
-    FaultTolerantRedisCluster clientPresenceCluster = new ShardFaultTolerantRedisCluster("client_presence",
+        configuration.getMessageCacheConfiguration().getRedisClusterConfiguration(), redisClientResourcesBuilder);
+    FaultTolerantRedisCluster clientPresenceCluster = new FaultTolerantRedisCluster("client_presence",
         configuration.getClientPresenceClusterConfiguration(), redisClientResourcesBuilder);
-    FaultTolerantRedisCluster rateLimitersCluster = new ShardFaultTolerantRedisCluster("rate_limiters",
+    FaultTolerantRedisCluster rateLimitersCluster = new FaultTolerantRedisCluster("rate_limiters",
         configuration.getRateLimitersCluster(), redisClientResourcesBuilder);
     SecureValueRecovery2Client secureValueRecovery2Client = new SecureValueRecovery2Client(
         secureValueRecoveryCredentialsGenerator, secureValueRecoveryServiceExecutor,

service/src/main/java/org/whispersystems/textsecuregcm/workers/ScheduledApnPushNotificationSenderServiceCommand.java

@@ -20,7 +20,6 @@ import org.whispersystems.textsecuregcm.metrics.MetricsUtil;
 import org.whispersystems.textsecuregcm.push.APNSender;
 import org.whispersystems.textsecuregcm.push.ApnPushNotificationScheduler;
 import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster;
-import org.whispersystems.textsecuregcm.redis.ShardFaultTolerantRedisCluster;
 import org.whispersystems.textsecuregcm.util.logging.UncaughtExceptionHandler;
 
 public class ScheduledApnPushNotificationSenderServiceCommand extends ServerCommand<WhisperServerConfiguration> {
@@ -64,7 +63,7 @@ public class ScheduledApnPushNotificationSenderServiceCommand extends ServerComm
           });
     }
 
-    final FaultTolerantRedisCluster pushSchedulerCluster = new ShardFaultTolerantRedisCluster("push_scheduler",
+      final FaultTolerantRedisCluster pushSchedulerCluster = new FaultTolerantRedisCluster("push_scheduler",
         configuration.getPushSchedulerCluster(), deps.redisClusterClientResourcesBuilder());
 
     final ExecutorService apnSenderExecutor = environment.lifecycle().executorService(name(getClass(), "apnSender-%d"))