mirror of
https://github.com/signalapp/Signal-Server
synced 2026-04-21 04:58:06 +01:00
Coalesce all Redis clusters to per-shard circuit breakers
This commit is contained in:
Chris Eager
Chris Eager
@@ -1,106 +0,0 @@
|
||||
/*
|
||||
* Copyright 2013-2020 Signal Messenger, LLC
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
package org.whispersystems.textsecuregcm.redis;
|
||||
|
||||
import static org.whispersystems.textsecuregcm.metrics.MetricsUtil.name;
|
||||
|
||||
import io.github.resilience4j.circuitbreaker.CircuitBreaker;
|
||||
import io.github.resilience4j.retry.Retry;
|
||||
import io.lettuce.core.RedisException;
|
||||
import io.lettuce.core.cluster.event.ClusterTopologyChangedEvent;
|
||||
import io.lettuce.core.cluster.pubsub.StatefulRedisClusterPubSubConnection;
|
||||
import io.micrometer.core.instrument.Metrics;
|
||||
import io.micrometer.core.instrument.Tags;
|
||||
import io.micrometer.core.instrument.Timer;
|
||||
import java.util.function.Consumer;
|
||||
import java.util.function.Function;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.whispersystems.textsecuregcm.util.CircuitBreakerUtil;
|
||||
import reactor.core.scheduler.Scheduler;
|
||||
|
||||
public class ClusterFaultTolerantPubSubConnection<K, V> implements FaultTolerantPubSubConnection<K, V> {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(ClusterFaultTolerantPubSubConnection.class);
|
||||
|
||||
|
||||
private final String name;
|
||||
private final StatefulRedisClusterPubSubConnection<K, V> pubSubConnection;
|
||||
|
||||
private final CircuitBreaker circuitBreaker;
|
||||
private final Retry retry;
|
||||
private final Retry resubscribeRetry;
|
||||
private final Scheduler topologyChangedEventScheduler;
|
||||
|
||||
private final Timer executeTimer;
|
||||
|
||||
public ClusterFaultTolerantPubSubConnection(final String name,
|
||||
final StatefulRedisClusterPubSubConnection<K, V> pubSubConnection, final CircuitBreaker circuitBreaker,
|
||||
final Retry retry, final Retry resubscribeRetry, final Scheduler topologyChangedEventScheduler) {
|
||||
this.name = name;
|
||||
this.pubSubConnection = pubSubConnection;
|
||||
this.circuitBreaker = circuitBreaker;
|
||||
this.retry = retry;
|
||||
this.resubscribeRetry = resubscribeRetry;
|
||||
this.topologyChangedEventScheduler = topologyChangedEventScheduler;
|
||||
|
||||
this.pubSubConnection.setNodeMessagePropagation(true);
|
||||
|
||||
this.executeTimer = Metrics.timer(name(getClass(), "execute"), "clusterName", name + "-pubsub");
|
||||
|
||||
CircuitBreakerUtil.registerMetrics(circuitBreaker, ClusterFaultTolerantPubSubConnection.class, Tags.empty());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void usePubSubConnection(final Consumer<StatefulRedisClusterPubSubConnection<K, V>> consumer) {
|
||||
try {
|
||||
circuitBreaker.executeCheckedRunnable(
|
||||
() -> retry.executeRunnable(() -> executeTimer.record(() -> consumer.accept(pubSubConnection))));
|
||||
} catch (final Throwable t) {
|
||||
if (t instanceof RedisException) {
|
||||
throw (RedisException) t;
|
||||
} else {
|
||||
throw new RedisException(t);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public <T> T withPubSubConnection(final Function<StatefulRedisClusterPubSubConnection<K, V>, T> function) {
|
||||
try {
|
||||
return circuitBreaker.executeCheckedSupplier(
|
||||
() -> retry.executeCallable(() -> executeTimer.record(() -> function.apply(pubSubConnection))));
|
||||
} catch (final Throwable t) {
|
||||
if (t instanceof RedisException) {
|
||||
throw (RedisException) t;
|
||||
} else {
|
||||
throw new RedisException(t);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public void subscribeToClusterTopologyChangedEvents(final Runnable eventHandler) {
|
||||
|
||||
usePubSubConnection(connection -> connection.getResources().eventBus().get()
|
||||
.filter(event -> event instanceof ClusterTopologyChangedEvent)
|
||||
.subscribeOn(topologyChangedEventScheduler)
|
||||
.subscribe(event -> {
|
||||
logger.info("Got topology change event for {}, resubscribing all keyspace notifications", name);
|
||||
|
||||
resubscribeRetry.executeRunnable(() -> {
|
||||
try {
|
||||
eventHandler.run();
|
||||
} catch (final RuntimeException e) {
|
||||
logger.warn("Resubscribe for {} failed", name, e);
|
||||
throw e;
|
||||
}
|
||||
});
|
||||
}));
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,196 +0,0 @@
|
||||
/*
|
||||
* Copyright 2013-2020 Signal Messenger, LLC
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
package org.whispersystems.textsecuregcm.redis;
|
||||
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import io.github.resilience4j.circuitbreaker.CircuitBreaker;
|
||||
import io.github.resilience4j.core.IntervalFunction;
|
||||
import io.github.resilience4j.reactor.circuitbreaker.operator.CircuitBreakerOperator;
|
||||
import io.github.resilience4j.reactor.retry.RetryOperator;
|
||||
import io.github.resilience4j.retry.Retry;
|
||||
import io.github.resilience4j.retry.RetryConfig;
|
||||
import io.lettuce.core.ClientOptions.DisconnectedBehavior;
|
||||
import io.lettuce.core.RedisCommandTimeoutException;
|
||||
import io.lettuce.core.RedisException;
|
||||
import io.lettuce.core.TimeoutOptions;
|
||||
import io.lettuce.core.cluster.ClusterClientOptions;
|
||||
import io.lettuce.core.cluster.ClusterTopologyRefreshOptions;
|
||||
import io.lettuce.core.cluster.RedisClusterClient;
|
||||
import io.lettuce.core.cluster.api.StatefulRedisClusterConnection;
|
||||
import io.lettuce.core.cluster.pubsub.StatefulRedisClusterPubSubConnection;
|
||||
import io.lettuce.core.codec.ByteArrayCodec;
|
||||
import io.lettuce.core.resource.ClientResources;
|
||||
import java.time.Duration;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.function.Consumer;
|
||||
import java.util.function.Function;
|
||||
import io.micrometer.core.instrument.Tags;
|
||||
import org.reactivestreams.Publisher;
|
||||
import org.whispersystems.textsecuregcm.configuration.CircuitBreakerConfiguration;
|
||||
import org.whispersystems.textsecuregcm.configuration.RedisClusterConfiguration;
|
||||
import org.whispersystems.textsecuregcm.configuration.RetryConfiguration;
|
||||
import org.whispersystems.textsecuregcm.util.CircuitBreakerUtil;
|
||||
import reactor.core.publisher.Flux;
|
||||
import reactor.core.scheduler.Schedulers;
|
||||
|
||||
/**
|
||||
* A fault-tolerant access manager for a Redis cluster. A single circuit breaker protects all cluster
|
||||
* calls.
|
||||
*/
|
||||
public class ClusterFaultTolerantRedisCluster implements FaultTolerantRedisCluster {
|
||||
|
||||
private final String name;
|
||||
|
||||
private final RedisClusterClient clusterClient;
|
||||
|
||||
private final StatefulRedisClusterConnection<String, String> stringConnection;
|
||||
private final StatefulRedisClusterConnection<byte[], byte[]> binaryConnection;
|
||||
|
||||
private final List<StatefulRedisClusterPubSubConnection<?, ?>> pubSubConnections = new ArrayList<>();
|
||||
|
||||
private final CircuitBreaker circuitBreaker;
|
||||
private final Retry retry;
|
||||
private final Retry topologyChangedEventRetry;
|
||||
|
||||
public ClusterFaultTolerantRedisCluster(final String name, final RedisClusterConfiguration clusterConfiguration,
|
||||
final ClientResources clientResources) {
|
||||
this(name,
|
||||
RedisClusterClient.create(clientResources,
|
||||
RedisUriUtil.createRedisUriWithTimeout(clusterConfiguration.getConfigurationUri(),
|
||||
clusterConfiguration.getTimeout())),
|
||||
clusterConfiguration.getTimeout(),
|
||||
clusterConfiguration.getCircuitBreakerConfiguration(),
|
||||
clusterConfiguration.getRetryConfiguration());
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
ClusterFaultTolerantRedisCluster(final String name, final RedisClusterClient clusterClient,
|
||||
final Duration commandTimeout,
|
||||
final CircuitBreakerConfiguration circuitBreakerConfiguration, final RetryConfiguration retryConfiguration) {
|
||||
this.name = name;
|
||||
|
||||
this.clusterClient = clusterClient;
|
||||
this.clusterClient.setOptions(ClusterClientOptions.builder()
|
||||
.disconnectedBehavior(DisconnectedBehavior.REJECT_COMMANDS)
|
||||
.validateClusterNodeMembership(false)
|
||||
.topologyRefreshOptions(ClusterTopologyRefreshOptions.builder()
|
||||
.enableAllAdaptiveRefreshTriggers()
|
||||
.build())
|
||||
// for asynchronous commands
|
||||
.timeoutOptions(TimeoutOptions.builder()
|
||||
.fixedTimeout(commandTimeout)
|
||||
.build())
|
||||
.publishOnScheduler(true)
|
||||
.build());
|
||||
|
||||
this.stringConnection = clusterClient.connect();
|
||||
this.binaryConnection = clusterClient.connect(ByteArrayCodec.INSTANCE);
|
||||
|
||||
this.circuitBreaker = CircuitBreaker.of(name + "-breaker", circuitBreakerConfiguration.toCircuitBreakerConfig());
|
||||
this.retry = Retry.of(name + "-retry", retryConfiguration.toRetryConfigBuilder()
|
||||
.retryOnException(exception -> exception instanceof RedisCommandTimeoutException).build());
|
||||
final RetryConfig topologyChangedEventRetryConfig = RetryConfig.custom()
|
||||
.maxAttempts(Integer.MAX_VALUE)
|
||||
.intervalFunction(
|
||||
IntervalFunction.ofExponentialRandomBackoff(Duration.ofSeconds(1), 1.5, Duration.ofSeconds(30)))
|
||||
.build();
|
||||
|
||||
this.topologyChangedEventRetry = Retry.of(name + "-topologyChangedRetry", topologyChangedEventRetryConfig);
|
||||
|
||||
CircuitBreakerUtil.registerMetrics(circuitBreaker, FaultTolerantRedisCluster.class, Tags.empty());
|
||||
CircuitBreakerUtil.registerMetrics(retry, FaultTolerantRedisCluster.class);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void shutdown() {
|
||||
stringConnection.close();
|
||||
binaryConnection.close();
|
||||
|
||||
for (final StatefulRedisClusterPubSubConnection<?, ?> pubSubConnection : pubSubConnections) {
|
||||
pubSubConnection.close();
|
||||
}
|
||||
|
||||
clusterClient.shutdown();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void useCluster(final Consumer<StatefulRedisClusterConnection<String, String>> consumer) {
|
||||
useConnection(stringConnection, consumer);
|
||||
}
|
||||
|
||||
@Override
|
||||
public <T> T withCluster(final Function<StatefulRedisClusterConnection<String, String>, T> function) {
|
||||
return withConnection(stringConnection, function);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void useBinaryCluster(final Consumer<StatefulRedisClusterConnection<byte[], byte[]>> consumer) {
|
||||
useConnection(binaryConnection, consumer);
|
||||
}
|
||||
|
||||
@Override
|
||||
public <T> T withBinaryCluster(final Function<StatefulRedisClusterConnection<byte[], byte[]>, T> function) {
|
||||
return withConnection(binaryConnection, function);
|
||||
}
|
||||
|
||||
@Override
|
||||
public <T> Publisher<T> withBinaryClusterReactive(
|
||||
final Function<StatefulRedisClusterConnection<byte[], byte[]>, Publisher<T>> function) {
|
||||
return withConnectionReactive(binaryConnection, function);
|
||||
}
|
||||
|
||||
@Override
|
||||
public <K, V> void useConnection(final StatefulRedisClusterConnection<K, V> connection,
|
||||
final Consumer<StatefulRedisClusterConnection<K, V>> consumer) {
|
||||
try {
|
||||
circuitBreaker.executeCheckedRunnable(() -> retry.executeRunnable(() -> consumer.accept(connection)));
|
||||
} catch (final Throwable t) {
|
||||
if (t instanceof RedisException) {
|
||||
throw (RedisException) t;
|
||||
} else {
|
||||
throw new RedisException(t);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public <T, K, V> T withConnection(final StatefulRedisClusterConnection<K, V> connection,
|
||||
final Function<StatefulRedisClusterConnection<K, V>, T> function) {
|
||||
try {
|
||||
return circuitBreaker.executeCheckedSupplier(() -> retry.executeCallable(() -> function.apply(connection)));
|
||||
} catch (final Throwable t) {
|
||||
if (t instanceof RedisException) {
|
||||
throw (RedisException) t;
|
||||
} else {
|
||||
throw new RedisException(t);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public <T, K, V> Publisher<T> withConnectionReactive(final StatefulRedisClusterConnection<K, V> connection,
|
||||
final Function<StatefulRedisClusterConnection<K, V>, Publisher<T>> function) {
|
||||
|
||||
return Flux.from(function.apply(connection))
|
||||
.transformDeferred(RetryOperator.of(retry))
|
||||
.transformDeferred(CircuitBreakerOperator.of(circuitBreaker));
|
||||
}
|
||||
|
||||
public FaultTolerantPubSubConnection<String, String> createPubSubConnection() {
|
||||
final StatefulRedisClusterPubSubConnection<String, String> pubSubConnection = clusterClient.connectPubSub();
|
||||
pubSubConnections.add(pubSubConnection);
|
||||
|
||||
return new ClusterFaultTolerantPubSubConnection<>(name, pubSubConnection, circuitBreaker, retry,
|
||||
topologyChangedEventRetry,
|
||||
Schedulers.newSingle(name + "-redisPubSubEvents", true));
|
||||
}
|
||||
}
|
||||
@@ -5,15 +5,90 @@
|
||||
|
||||
package org.whispersystems.textsecuregcm.redis;
|
||||
|
||||
import static org.whispersystems.textsecuregcm.metrics.MetricsUtil.name;
|
||||
|
||||
import io.github.resilience4j.retry.Retry;
|
||||
import io.lettuce.core.RedisException;
|
||||
import io.lettuce.core.cluster.event.ClusterTopologyChangedEvent;
|
||||
import io.lettuce.core.cluster.pubsub.StatefulRedisClusterPubSubConnection;
|
||||
import io.micrometer.core.instrument.Metrics;
|
||||
import io.micrometer.core.instrument.Timer;
|
||||
import java.util.function.Consumer;
|
||||
import java.util.function.Function;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import reactor.core.scheduler.Scheduler;
|
||||
|
||||
public interface FaultTolerantPubSubConnection<K, V> {
|
||||
public class FaultTolerantPubSubConnection<K, V> {
|
||||
|
||||
void usePubSubConnection(Consumer<StatefulRedisClusterPubSubConnection<K, V>> consumer);
|
||||
private static final Logger logger = LoggerFactory.getLogger(FaultTolerantPubSubConnection.class);
|
||||
|
||||
<T> T withPubSubConnection(Function<StatefulRedisClusterPubSubConnection<K, V>, T> function);
|
||||
|
||||
void subscribeToClusterTopologyChangedEvents(Runnable eventHandler);
|
||||
private final String name;
|
||||
private final StatefulRedisClusterPubSubConnection<K, V> pubSubConnection;
|
||||
|
||||
private final Retry retry;
|
||||
private final Retry resubscribeRetry;
|
||||
private final Scheduler topologyChangedEventScheduler;
|
||||
|
||||
private final Timer executeTimer;
|
||||
|
||||
public FaultTolerantPubSubConnection(final String name,
|
||||
final StatefulRedisClusterPubSubConnection<K, V> pubSubConnection,
|
||||
final Retry retry, final Retry resubscribeRetry, final Scheduler topologyChangedEventScheduler) {
|
||||
this.name = name;
|
||||
this.pubSubConnection = pubSubConnection;
|
||||
this.retry = retry;
|
||||
this.resubscribeRetry = resubscribeRetry;
|
||||
this.topologyChangedEventScheduler = topologyChangedEventScheduler;
|
||||
|
||||
this.pubSubConnection.setNodeMessagePropagation(true);
|
||||
|
||||
this.executeTimer = Metrics.timer(name(getClass(), "execute"), "clusterName", name + "-pubsub");
|
||||
}
|
||||
|
||||
public void usePubSubConnection(final Consumer<StatefulRedisClusterPubSubConnection<K, V>> consumer) {
|
||||
try {
|
||||
retry.executeRunnable(() -> executeTimer.record(() -> consumer.accept(pubSubConnection)));
|
||||
} catch (final Throwable t) {
|
||||
if (t instanceof RedisException) {
|
||||
throw (RedisException) t;
|
||||
} else {
|
||||
throw new RedisException(t);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public <T> T withPubSubConnection(final Function<StatefulRedisClusterPubSubConnection<K, V>, T> function) {
|
||||
try {
|
||||
return retry.executeCallable(() -> executeTimer.record(() -> function.apply(pubSubConnection)));
|
||||
} catch (final Throwable t) {
|
||||
if (t instanceof RedisException) {
|
||||
throw (RedisException) t;
|
||||
} else {
|
||||
throw new RedisException(t);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public void subscribeToClusterTopologyChangedEvents(final Runnable eventHandler) {
|
||||
|
||||
usePubSubConnection(connection -> connection.getResources().eventBus().get()
|
||||
.filter(event -> event instanceof ClusterTopologyChangedEvent)
|
||||
.subscribeOn(topologyChangedEventScheduler)
|
||||
.subscribe(event -> {
|
||||
logger.info("Got topology change event for {}, resubscribing all keyspace notifications", name);
|
||||
|
||||
resubscribeRetry.executeRunnable(() -> {
|
||||
try {
|
||||
eventHandler.run();
|
||||
} catch (final RuntimeException e) {
|
||||
logger.warn("Resubscribe for {} failed", name, e);
|
||||
throw e;
|
||||
}
|
||||
});
|
||||
}));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -5,36 +5,183 @@
|
||||
|
||||
package org.whispersystems.textsecuregcm.redis;
|
||||
|
||||
import io.github.resilience4j.core.IntervalFunction;
|
||||
import io.github.resilience4j.reactor.retry.RetryOperator;
|
||||
import io.github.resilience4j.retry.Retry;
|
||||
import io.github.resilience4j.retry.RetryConfig;
|
||||
import io.lettuce.core.ClientOptions;
|
||||
import io.lettuce.core.RedisCommandTimeoutException;
|
||||
import io.lettuce.core.RedisException;
|
||||
import io.lettuce.core.RedisURI;
|
||||
import io.lettuce.core.TimeoutOptions;
|
||||
import io.lettuce.core.cluster.ClusterClientOptions;
|
||||
import io.lettuce.core.cluster.ClusterTopologyRefreshOptions;
|
||||
import io.lettuce.core.cluster.RedisClusterClient;
|
||||
import io.lettuce.core.cluster.api.StatefulRedisClusterConnection;
|
||||
import io.lettuce.core.cluster.pubsub.StatefulRedisClusterPubSubConnection;
|
||||
import io.lettuce.core.codec.ByteArrayCodec;
|
||||
import io.lettuce.core.resource.ClientResources;
|
||||
import java.time.Duration;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.function.Consumer;
|
||||
import java.util.function.Function;
|
||||
import org.reactivestreams.Publisher;
|
||||
import org.whispersystems.textsecuregcm.configuration.CircuitBreakerConfiguration;
|
||||
import org.whispersystems.textsecuregcm.configuration.RedisClusterConfiguration;
|
||||
import org.whispersystems.textsecuregcm.configuration.RetryConfiguration;
|
||||
import org.whispersystems.textsecuregcm.util.CircuitBreakerUtil;
|
||||
import reactor.core.publisher.Flux;
|
||||
import reactor.core.scheduler.Schedulers;
|
||||
|
||||
public interface FaultTolerantRedisCluster {
|
||||
/**
|
||||
* A fault-tolerant access manager for a Redis cluster. Each shard in the cluster has a dedicated circuit breaker.
|
||||
*
|
||||
* @see LettuceShardCircuitBreaker
|
||||
*/
|
||||
public class FaultTolerantRedisCluster {
|
||||
|
||||
void shutdown();
|
||||
private final String name;
|
||||
|
||||
String getName();
|
||||
private final RedisClusterClient clusterClient;
|
||||
|
||||
void useCluster(Consumer<StatefulRedisClusterConnection<String, String>> consumer);
|
||||
private final StatefulRedisClusterConnection<String, String> stringConnection;
|
||||
private final StatefulRedisClusterConnection<byte[], byte[]> binaryConnection;
|
||||
|
||||
<T> T withCluster(Function<StatefulRedisClusterConnection<String, String>, T> function);
|
||||
private final List<StatefulRedisClusterPubSubConnection<?, ?>> pubSubConnections = new ArrayList<>();
|
||||
|
||||
void useBinaryCluster(Consumer<StatefulRedisClusterConnection<byte[], byte[]>> consumer);
|
||||
private final Retry retry;
|
||||
private final Retry topologyChangedEventRetry;
|
||||
|
||||
<T> T withBinaryCluster(Function<StatefulRedisClusterConnection<byte[], byte[]>, T> function);
|
||||
|
||||
<T> Publisher<T> withBinaryClusterReactive(
|
||||
Function<StatefulRedisClusterConnection<byte[], byte[]>, Publisher<T>> function);
|
||||
public FaultTolerantRedisCluster(final String name, final RedisClusterConfiguration clusterConfiguration,
|
||||
final ClientResources.Builder clientResourcesBuilder) {
|
||||
|
||||
<K, V> void useConnection(StatefulRedisClusterConnection<K, V> connection,
|
||||
Consumer<StatefulRedisClusterConnection<K, V>> consumer);
|
||||
this(name, clientResourcesBuilder,
|
||||
Collections.singleton(RedisUriUtil.createRedisUriWithTimeout(clusterConfiguration.getConfigurationUri(),
|
||||
clusterConfiguration.getTimeout())),
|
||||
clusterConfiguration.getTimeout(),
|
||||
clusterConfiguration.getCircuitBreakerConfiguration(),
|
||||
clusterConfiguration.getRetryConfiguration());
|
||||
|
||||
<T, K, V> T withConnection(StatefulRedisClusterConnection<K, V> connection,
|
||||
Function<StatefulRedisClusterConnection<K, V>, T> function);
|
||||
}
|
||||
|
||||
<T, K, V> Publisher<T> withConnectionReactive(StatefulRedisClusterConnection<K, V> connection,
|
||||
Function<StatefulRedisClusterConnection<K, V>, Publisher<T>> function);
|
||||
FaultTolerantRedisCluster(String name, final ClientResources.Builder clientResourcesBuilder,
|
||||
Iterable<RedisURI> redisUris, Duration commandTimeout, CircuitBreakerConfiguration circuitBreakerConfig,
|
||||
RetryConfiguration retryConfiguration) {
|
||||
|
||||
this.name = name;
|
||||
|
||||
this.clusterClient = RedisClusterClient.create(
|
||||
clientResourcesBuilder.nettyCustomizer(
|
||||
new LettuceShardCircuitBreaker(name, circuitBreakerConfig.toCircuitBreakerConfig())).
|
||||
build(),
|
||||
redisUris);
|
||||
this.clusterClient.setOptions(ClusterClientOptions.builder()
|
||||
.disconnectedBehavior(ClientOptions.DisconnectedBehavior.REJECT_COMMANDS)
|
||||
.validateClusterNodeMembership(false)
|
||||
.topologyRefreshOptions(ClusterTopologyRefreshOptions.builder()
|
||||
.enableAllAdaptiveRefreshTriggers()
|
||||
.build())
|
||||
// for asynchronous commands
|
||||
.timeoutOptions(TimeoutOptions.builder()
|
||||
.fixedTimeout(commandTimeout)
|
||||
.build())
|
||||
.publishOnScheduler(true)
|
||||
.build());
|
||||
|
||||
this.stringConnection = clusterClient.connect();
|
||||
this.binaryConnection = clusterClient.connect(ByteArrayCodec.INSTANCE);
|
||||
|
||||
this.retry = Retry.of(name + "-retry", retryConfiguration.toRetryConfigBuilder()
|
||||
.retryOnException(exception -> exception instanceof RedisCommandTimeoutException).build());
|
||||
final RetryConfig topologyChangedEventRetryConfig = RetryConfig.custom()
|
||||
.maxAttempts(Integer.MAX_VALUE)
|
||||
.intervalFunction(
|
||||
IntervalFunction.ofExponentialRandomBackoff(Duration.ofSeconds(1), 1.5, Duration.ofSeconds(30)))
|
||||
.build();
|
||||
|
||||
this.topologyChangedEventRetry = Retry.of(name + "-topologyChangedRetry", topologyChangedEventRetryConfig);
|
||||
|
||||
CircuitBreakerUtil.registerMetrics(retry, FaultTolerantRedisCluster.class);
|
||||
}
|
||||
|
||||
public void shutdown() {
|
||||
stringConnection.close();
|
||||
binaryConnection.close();
|
||||
|
||||
for (final StatefulRedisClusterPubSubConnection<?, ?> pubSubConnection : pubSubConnections) {
|
||||
pubSubConnection.close();
|
||||
}
|
||||
|
||||
clusterClient.shutdown();
|
||||
}
|
||||
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
public void useCluster(final Consumer<StatefulRedisClusterConnection<String, String>> consumer) {
|
||||
useConnection(stringConnection, consumer);
|
||||
}
|
||||
|
||||
public <T> T withCluster(final Function<StatefulRedisClusterConnection<String, String>, T> function) {
|
||||
return withConnection(stringConnection, function);
|
||||
}
|
||||
|
||||
public void useBinaryCluster(final Consumer<StatefulRedisClusterConnection<byte[], byte[]>> consumer) {
|
||||
useConnection(binaryConnection, consumer);
|
||||
}
|
||||
|
||||
public <T> T withBinaryCluster(final Function<StatefulRedisClusterConnection<byte[], byte[]>, T> function) {
|
||||
return withConnection(binaryConnection, function);
|
||||
}
|
||||
|
||||
public <T> Publisher<T> withBinaryClusterReactive(
|
||||
final Function<StatefulRedisClusterConnection<byte[], byte[]>, Publisher<T>> function) {
|
||||
return withConnectionReactive(binaryConnection, function);
|
||||
}
|
||||
|
||||
public <K, V> void useConnection(final StatefulRedisClusterConnection<K, V> connection,
|
||||
final Consumer<StatefulRedisClusterConnection<K, V>> consumer) {
|
||||
try {
|
||||
retry.executeRunnable(() -> consumer.accept(connection));
|
||||
} catch (final Throwable t) {
|
||||
if (t instanceof RedisException) {
|
||||
throw (RedisException) t;
|
||||
} else {
|
||||
throw new RedisException(t);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public <T, K, V> T withConnection(final StatefulRedisClusterConnection<K, V> connection,
|
||||
final Function<StatefulRedisClusterConnection<K, V>, T> function) {
|
||||
try {
|
||||
return retry.executeCallable(() -> function.apply(connection));
|
||||
} catch (final Throwable t) {
|
||||
if (t instanceof RedisException) {
|
||||
throw (RedisException) t;
|
||||
} else {
|
||||
throw new RedisException(t);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public <T, K, V> Publisher<T> withConnectionReactive(final StatefulRedisClusterConnection<K, V> connection,
|
||||
final Function<StatefulRedisClusterConnection<K, V>, Publisher<T>> function) {
|
||||
|
||||
return Flux.from(function.apply(connection))
|
||||
.transformDeferred(RetryOperator.of(retry));
|
||||
}
|
||||
|
||||
public FaultTolerantPubSubConnection<String, String> createPubSubConnection() {
|
||||
final StatefulRedisClusterPubSubConnection<String, String> pubSubConnection = clusterClient.connectPubSub();
|
||||
pubSubConnections.add(pubSubConnection);
|
||||
|
||||
return new FaultTolerantPubSubConnection<>(name, pubSubConnection, retry, topologyChangedEventRetry,
|
||||
Schedulers.newSingle(name + "-redisPubSubEvents", true));
|
||||
}
|
||||
|
||||
FaultTolerantPubSubConnection<String, String> createPubSubConnection();
|
||||
}
|
||||
|
||||
@@ -1,97 +0,0 @@
|
||||
/*
|
||||
* Copyright 2013-2020 Signal Messenger, LLC
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
package org.whispersystems.textsecuregcm.redis;
|
||||
|
||||
import static org.whispersystems.textsecuregcm.metrics.MetricsUtil.name;
|
||||
|
||||
import io.github.resilience4j.retry.Retry;
|
||||
import io.lettuce.core.RedisException;
|
||||
import io.lettuce.core.cluster.event.ClusterTopologyChangedEvent;
|
||||
import io.lettuce.core.cluster.pubsub.StatefulRedisClusterPubSubConnection;
|
||||
import io.micrometer.core.instrument.Metrics;
|
||||
import io.micrometer.core.instrument.Timer;
|
||||
import java.util.function.Consumer;
|
||||
import java.util.function.Function;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import reactor.core.scheduler.Scheduler;
|
||||
|
||||
public class ShardFaultTolerantPubSubConnection<K, V> implements FaultTolerantPubSubConnection<K, V> {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(ShardFaultTolerantPubSubConnection.class);
|
||||
|
||||
|
||||
private final String name;
|
||||
private final StatefulRedisClusterPubSubConnection<K, V> pubSubConnection;
|
||||
|
||||
private final Retry retry;
|
||||
private final Retry resubscribeRetry;
|
||||
private final Scheduler topologyChangedEventScheduler;
|
||||
|
||||
private final Timer executeTimer;
|
||||
|
||||
public ShardFaultTolerantPubSubConnection(final String name,
|
||||
final StatefulRedisClusterPubSubConnection<K, V> pubSubConnection,
|
||||
final Retry retry, final Retry resubscribeRetry, final Scheduler topologyChangedEventScheduler) {
|
||||
this.name = name;
|
||||
this.pubSubConnection = pubSubConnection;
|
||||
this.retry = retry;
|
||||
this.resubscribeRetry = resubscribeRetry;
|
||||
this.topologyChangedEventScheduler = topologyChangedEventScheduler;
|
||||
|
||||
this.pubSubConnection.setNodeMessagePropagation(true);
|
||||
|
||||
this.executeTimer = Metrics.timer(name(getClass(), "execute"), "clusterName", name + "-pubsub");
|
||||
}
|
||||
|
||||
@Override
|
||||
public void usePubSubConnection(final Consumer<StatefulRedisClusterPubSubConnection<K, V>> consumer) {
|
||||
try {
|
||||
retry.executeRunnable(() -> executeTimer.record(() -> consumer.accept(pubSubConnection)));
|
||||
} catch (final Throwable t) {
|
||||
if (t instanceof RedisException) {
|
||||
throw (RedisException) t;
|
||||
} else {
|
||||
throw new RedisException(t);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public <T> T withPubSubConnection(final Function<StatefulRedisClusterPubSubConnection<K, V>, T> function) {
|
||||
try {
|
||||
return retry.executeCallable(() -> executeTimer.record(() -> function.apply(pubSubConnection)));
|
||||
} catch (final Throwable t) {
|
||||
if (t instanceof RedisException) {
|
||||
throw (RedisException) t;
|
||||
} else {
|
||||
throw new RedisException(t);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public void subscribeToClusterTopologyChangedEvents(final Runnable eventHandler) {
|
||||
|
||||
usePubSubConnection(connection -> connection.getResources().eventBus().get()
|
||||
.filter(event -> event instanceof ClusterTopologyChangedEvent)
|
||||
.subscribeOn(topologyChangedEventScheduler)
|
||||
.subscribe(event -> {
|
||||
logger.info("Got topology change event for {}, resubscribing all keyspace notifications", name);
|
||||
|
||||
resubscribeRetry.executeRunnable(() -> {
|
||||
try {
|
||||
eventHandler.run();
|
||||
} catch (final RuntimeException e) {
|
||||
logger.warn("Resubscribe for {} failed", name, e);
|
||||
throw e;
|
||||
}
|
||||
});
|
||||
}));
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,197 +0,0 @@
|
||||
/*
|
||||
* Copyright 2024 Signal Messenger, LLC
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
package org.whispersystems.textsecuregcm.redis;
|
||||
|
||||
import io.github.resilience4j.core.IntervalFunction;
|
||||
import io.github.resilience4j.reactor.retry.RetryOperator;
|
||||
import io.github.resilience4j.retry.Retry;
|
||||
import io.github.resilience4j.retry.RetryConfig;
|
||||
import io.lettuce.core.ClientOptions;
|
||||
import io.lettuce.core.RedisCommandTimeoutException;
|
||||
import io.lettuce.core.RedisException;
|
||||
import io.lettuce.core.RedisURI;
|
||||
import io.lettuce.core.TimeoutOptions;
|
||||
import io.lettuce.core.cluster.ClusterClientOptions;
|
||||
import io.lettuce.core.cluster.ClusterTopologyRefreshOptions;
|
||||
import io.lettuce.core.cluster.RedisClusterClient;
|
||||
import io.lettuce.core.cluster.api.StatefulRedisClusterConnection;
|
||||
import io.lettuce.core.cluster.pubsub.StatefulRedisClusterPubSubConnection;
|
||||
import io.lettuce.core.codec.ByteArrayCodec;
|
||||
import io.lettuce.core.resource.ClientResources;
|
||||
import java.time.Duration;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.function.Consumer;
|
||||
import java.util.function.Function;
|
||||
import org.reactivestreams.Publisher;
|
||||
import org.whispersystems.textsecuregcm.configuration.CircuitBreakerConfiguration;
|
||||
import org.whispersystems.textsecuregcm.configuration.RedisClusterConfiguration;
|
||||
import org.whispersystems.textsecuregcm.configuration.RetryConfiguration;
|
||||
import org.whispersystems.textsecuregcm.util.CircuitBreakerUtil;
|
||||
import reactor.core.publisher.Flux;
|
||||
import reactor.core.scheduler.Schedulers;
|
||||
|
||||
/**
|
||||
* A fault-tolerant access manager for a Redis cluster. Each shard in the cluster has a dedicated circuit breaker.
|
||||
*
|
||||
* @see LettuceShardCircuitBreaker
|
||||
*/
|
||||
public class ShardFaultTolerantRedisCluster implements FaultTolerantRedisCluster {
|
||||
|
||||
private final String name;
|
||||
|
||||
private final RedisClusterClient clusterClient;
|
||||
|
||||
private final StatefulRedisClusterConnection<String, String> stringConnection;
|
||||
private final StatefulRedisClusterConnection<byte[], byte[]> binaryConnection;
|
||||
|
||||
private final List<StatefulRedisClusterPubSubConnection<?, ?>> pubSubConnections = new ArrayList<>();
|
||||
|
||||
private final Retry retry;
|
||||
private final Retry topologyChangedEventRetry;
|
||||
|
||||
|
||||
public ShardFaultTolerantRedisCluster(final String name, final RedisClusterConfiguration clusterConfiguration,
|
||||
final ClientResources.Builder clientResourcesBuilder) {
|
||||
|
||||
this(name, clientResourcesBuilder,
|
||||
Collections.singleton(RedisUriUtil.createRedisUriWithTimeout(clusterConfiguration.getConfigurationUri(),
|
||||
clusterConfiguration.getTimeout())),
|
||||
clusterConfiguration.getTimeout(),
|
||||
clusterConfiguration.getCircuitBreakerConfiguration(),
|
||||
clusterConfiguration.getRetryConfiguration());
|
||||
|
||||
}
|
||||
|
||||
ShardFaultTolerantRedisCluster(String name, final ClientResources.Builder clientResourcesBuilder,
|
||||
Iterable<RedisURI> redisUris, Duration commandTimeout, CircuitBreakerConfiguration circuitBreakerConfig,
|
||||
RetryConfiguration retryConfiguration) {
|
||||
|
||||
this.name = name;
|
||||
|
||||
this.clusterClient = RedisClusterClient.create(
|
||||
clientResourcesBuilder.nettyCustomizer(
|
||||
new LettuceShardCircuitBreaker(name, circuitBreakerConfig.toCircuitBreakerConfig())).
|
||||
build(),
|
||||
redisUris);
|
||||
this.clusterClient.setOptions(ClusterClientOptions.builder()
|
||||
.disconnectedBehavior(ClientOptions.DisconnectedBehavior.REJECT_COMMANDS)
|
||||
.validateClusterNodeMembership(false)
|
||||
.topologyRefreshOptions(ClusterTopologyRefreshOptions.builder()
|
||||
.enableAllAdaptiveRefreshTriggers()
|
||||
.build())
|
||||
// for asynchronous commands
|
||||
.timeoutOptions(TimeoutOptions.builder()
|
||||
.fixedTimeout(commandTimeout)
|
||||
.build())
|
||||
.publishOnScheduler(true)
|
||||
.build());
|
||||
|
||||
this.stringConnection = clusterClient.connect();
|
||||
this.binaryConnection = clusterClient.connect(ByteArrayCodec.INSTANCE);
|
||||
|
||||
this.retry = Retry.of(name + "-retry", retryConfiguration.toRetryConfigBuilder()
|
||||
.retryOnException(exception -> exception instanceof RedisCommandTimeoutException).build());
|
||||
final RetryConfig topologyChangedEventRetryConfig = RetryConfig.custom()
|
||||
.maxAttempts(Integer.MAX_VALUE)
|
||||
.intervalFunction(
|
||||
IntervalFunction.ofExponentialRandomBackoff(Duration.ofSeconds(1), 1.5, Duration.ofSeconds(30)))
|
||||
.build();
|
||||
|
||||
this.topologyChangedEventRetry = Retry.of(name + "-topologyChangedRetry", topologyChangedEventRetryConfig);
|
||||
|
||||
CircuitBreakerUtil.registerMetrics(retry, ShardFaultTolerantRedisCluster.class);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void shutdown() {
|
||||
stringConnection.close();
|
||||
binaryConnection.close();
|
||||
|
||||
for (final StatefulRedisClusterPubSubConnection<?, ?> pubSubConnection : pubSubConnections) {
|
||||
pubSubConnection.close();
|
||||
}
|
||||
|
||||
clusterClient.shutdown();
|
||||
}
|
||||
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void useCluster(final Consumer<StatefulRedisClusterConnection<String, String>> consumer) {
|
||||
useConnection(stringConnection, consumer);
|
||||
}
|
||||
|
||||
@Override
|
||||
public <T> T withCluster(final Function<StatefulRedisClusterConnection<String, String>, T> function) {
|
||||
return withConnection(stringConnection, function);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void useBinaryCluster(final Consumer<StatefulRedisClusterConnection<byte[], byte[]>> consumer) {
|
||||
useConnection(binaryConnection, consumer);
|
||||
}
|
||||
|
||||
@Override
|
||||
public <T> T withBinaryCluster(final Function<StatefulRedisClusterConnection<byte[], byte[]>, T> function) {
|
||||
return withConnection(binaryConnection, function);
|
||||
}
|
||||
|
||||
@Override
|
||||
public <T> Publisher<T> withBinaryClusterReactive(
|
||||
final Function<StatefulRedisClusterConnection<byte[], byte[]>, Publisher<T>> function) {
|
||||
return withConnectionReactive(binaryConnection, function);
|
||||
}
|
||||
|
||||
@Override
|
||||
public <K, V> void useConnection(final StatefulRedisClusterConnection<K, V> connection,
|
||||
final Consumer<StatefulRedisClusterConnection<K, V>> consumer) {
|
||||
try {
|
||||
retry.executeRunnable(() -> consumer.accept(connection));
|
||||
} catch (final Throwable t) {
|
||||
if (t instanceof RedisException) {
|
||||
throw (RedisException) t;
|
||||
} else {
|
||||
throw new RedisException(t);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public <T, K, V> T withConnection(final StatefulRedisClusterConnection<K, V> connection,
|
||||
final Function<StatefulRedisClusterConnection<K, V>, T> function) {
|
||||
try {
|
||||
return retry.executeCallable(() -> function.apply(connection));
|
||||
} catch (final Throwable t) {
|
||||
if (t instanceof RedisException) {
|
||||
throw (RedisException) t;
|
||||
} else {
|
||||
throw new RedisException(t);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public <T, K, V> Publisher<T> withConnectionReactive(final StatefulRedisClusterConnection<K, V> connection,
|
||||
final Function<StatefulRedisClusterConnection<K, V>, Publisher<T>> function) {
|
||||
|
||||
return Flux.from(function.apply(connection))
|
||||
.transformDeferred(RetryOperator.of(retry));
|
||||
}
|
||||
|
||||
@Override
|
||||
public FaultTolerantPubSubConnection<String, String> createPubSubConnection() {
|
||||
final StatefulRedisClusterPubSubConnection<String, String> pubSubConnection = clusterClient.connectPubSub();
|
||||
pubSubConnections.add(pubSubConnection);
|
||||
|
||||
return new ShardFaultTolerantPubSubConnection<>(name, pubSubConnection, retry, topologyChangedEventRetry,
|
||||
Schedulers.newSingle(name + "-redisPubSubEvents", true));
|
||||
}
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user