Use central registries for Retry and CircuitBreaker instances

This commit is contained in:
Jon Chambers
2025-08-27 11:33:42 -04:00
committed by GitHub
parent a8c6fa93e0
commit f616612104
33 changed files with 326 additions and 349 deletions

View File

@@ -9,6 +9,7 @@ import io.dropwizard.core.Configuration;
import jakarta.validation.Valid;
import jakarta.validation.constraints.NotNull;
import java.time.Duration;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
@@ -22,6 +23,7 @@ import org.whispersystems.textsecuregcm.configuration.BadgesConfiguration;
import org.whispersystems.textsecuregcm.configuration.BraintreeConfiguration;
import org.whispersystems.textsecuregcm.configuration.Cdn3StorageManagerConfiguration;
import org.whispersystems.textsecuregcm.configuration.CdnConfiguration;
import org.whispersystems.textsecuregcm.configuration.CircuitBreakerConfiguration;
import org.whispersystems.textsecuregcm.configuration.ClientReleaseConfiguration;
import org.whispersystems.textsecuregcm.configuration.DatadogConfiguration;
import org.whispersystems.textsecuregcm.configuration.DefaultAwsCredentialsFactory;
@@ -51,6 +53,7 @@ import org.whispersystems.textsecuregcm.configuration.PaymentsServiceConfigurati
import org.whispersystems.textsecuregcm.configuration.RegistrationServiceClientFactory;
import org.whispersystems.textsecuregcm.configuration.RemoteConfigConfiguration;
import org.whispersystems.textsecuregcm.configuration.ReportMessageConfiguration;
import org.whispersystems.textsecuregcm.configuration.RetryConfiguration;
import org.whispersystems.textsecuregcm.configuration.S3ObjectMonitorFactory;
import org.whispersystems.textsecuregcm.configuration.SecureStorageServiceConfiguration;
import org.whispersystems.textsecuregcm.configuration.SecureValueRecoveryConfiguration;
@@ -340,6 +343,12 @@ public class WhisperServerConfiguration extends Configuration {
private IdlePrimaryDeviceReminderConfiguration idlePrimaryDeviceReminder =
new IdlePrimaryDeviceReminderConfiguration(Duration.ofDays(30));
@JsonProperty
private Map<String, CircuitBreakerConfiguration> circuitBreakers = Collections.emptyMap();
@JsonProperty
private Map<String, RetryConfiguration> retries = Collections.emptyMap();
public TlsKeyStoreConfiguration getTlsKeyStoreConfiguration() {
return tlsKeyStore;
}
@@ -562,4 +571,12 @@ public class WhisperServerConfiguration extends Configuration {
public IdlePrimaryDeviceReminderConfiguration idlePrimaryDeviceReminderConfiguration() {
return idlePrimaryDeviceReminder;
}
public Map<String, CircuitBreakerConfiguration> getCircuitBreakerConfigurations() {
return circuitBreakers;
}
public Map<String, RetryConfiguration> getRetryConfigurations() {
return retries;
}
}

View File

@@ -257,6 +257,7 @@ import org.whispersystems.textsecuregcm.subscriptions.BraintreeManager;
import org.whispersystems.textsecuregcm.subscriptions.GooglePlayBillingManager;
import org.whispersystems.textsecuregcm.subscriptions.StripeManager;
import org.whispersystems.textsecuregcm.util.BufferingInterceptor;
import org.whispersystems.textsecuregcm.util.CircuitBreakerUtil;
import org.whispersystems.textsecuregcm.util.ManagedAwsCrt;
import org.whispersystems.textsecuregcm.util.ManagedExecutors;
import org.whispersystems.textsecuregcm.util.SystemMapper;
@@ -363,6 +364,12 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
UncaughtExceptionHandler.register();
config.getCircuitBreakerConfigurations().forEach((name, configuration) ->
CircuitBreakerUtil.getCircuitBreakerRegistry().addConfiguration(name, configuration.toCircuitBreakerConfig()));
config.getRetryConfigurations().forEach((name, configuration) ->
CircuitBreakerUtil.getRetryRegistry().addConfiguration(name, configuration.toRetryConfigBuilder().build()));
ScheduledExecutorService dynamicConfigurationExecutor = ScheduledExecutorServiceBuilder.of(environment, "dynamicConfiguration")
.threads(1).build();
@@ -706,9 +713,9 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
config.getTurnConfiguration().cloudflare().urlsWithIps(),
config.getTurnConfiguration().cloudflare().hostname(),
config.getTurnConfiguration().cloudflare().numHttpClients(),
config.getTurnConfiguration().cloudflare().circuitBreaker(),
config.getTurnConfiguration().cloudflare().circuitBreakerConfigurationName(),
cloudflareTurnHttpExecutor,
config.getTurnConfiguration().cloudflare().retry(),
config.getTurnConfiguration().cloudflare().retryConfigurationName(),
cloudflareTurnRetryExecutor,
cloudflareDnsResolver
);
@@ -741,7 +748,7 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
config.getBraintree().environment(),
config.getBraintree().supportedCurrenciesByPaymentMethod(), config.getBraintree().merchantAccounts(),
config.getBraintree().graphqlUrl(), currencyManager, config.getBraintree().pubSubPublisher().build(),
config.getBraintree().circuitBreaker(), subscriptionProcessorExecutor,
config.getBraintree().circuitBreakerConfigurationName(), subscriptionProcessorExecutor,
subscriptionProcessorRetryExecutor);
GooglePlayBillingManager googlePlayBillingManager = new GooglePlayBillingManager(
new ByteArrayInputStream(config.getGooglePlayBilling().credentialsJson().value().getBytes(StandardCharsets.UTF_8)),
@@ -755,7 +762,7 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
config.getAppleAppStore().encodedKey().value(), config.getAppleAppStore().subscriptionGroupId(),
config.getAppleAppStore().productIdToLevel(),
config.getAppleAppStore().appleRootCerts(),
config.getAppleAppStore().retry(), appleAppStoreExecutor, appleAppStoreRetryExecutor);
config.getAppleAppStore().retryConfigurationName(), appleAppStoreExecutor, appleAppStoreRetryExecutor);
environment.lifecycle().manage(apnSender);
environment.lifecycle().manage(pushNotificationScheduler);

View File

@@ -19,10 +19,9 @@ import java.util.List;
import java.util.concurrent.CompletionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.ScheduledExecutorService;
import javax.annotation.Nullable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.whispersystems.textsecuregcm.configuration.CircuitBreakerConfiguration;
import org.whispersystems.textsecuregcm.configuration.RetryConfiguration;
import org.whispersystems.textsecuregcm.http.FaultTolerantHttpClient;
import org.whispersystems.textsecuregcm.util.ExceptionUtils;
import org.whispersystems.textsecuregcm.util.SystemMapper;
@@ -60,18 +59,15 @@ public class CloudflareTurnCredentialsManager {
final List<String> cloudflareTurnUrlsWithIps,
final String cloudflareTurnHostname,
final int cloudflareTurnNumHttpClients,
final CircuitBreakerConfiguration circuitBreaker,
@Nullable final String circuitBreakerConfigurationName,
final ExecutorService executor,
final RetryConfiguration retry,
@Nullable final String retryConfigurationName,
final ScheduledExecutorService retryExecutor,
final DnsNameResolver dnsNameResolver) {
this.cloudflareTurnClient = FaultTolerantHttpClient.newBuilder()
.withName("cloudflare-turn")
.withCircuitBreaker(circuitBreaker)
.withExecutor(executor)
.withRetry(retry)
.withRetryExecutor(retryExecutor)
this.cloudflareTurnClient = FaultTolerantHttpClient.newBuilder("cloudflare-turn", executor)
.withCircuitBreaker(circuitBreakerConfigurationName)
.withRetry(retryConfigurationName, retryExecutor)
.withNumClients(cloudflareTurnNumHttpClients)
.build();
this.cloudflareTurnUrls = cloudflareTurnUrls;

View File

@@ -67,12 +67,9 @@ public class Cdn3RemoteStorageManager implements RemoteStorageManager {
this.clientSecret = configuration.clientSecret().value();
// Client used for calls to storage-manager
this.storageManagerHttpClient = FaultTolerantHttpClient.newBuilder()
.withName("cdn3-storage-manager")
.withCircuitBreaker(configuration.circuitBreaker())
.withExecutor(httpExecutor)
.withRetryExecutor(retryExecutor)
.withRetry(configuration.retry())
this.storageManagerHttpClient = FaultTolerantHttpClient.newBuilder("cdn3-storage-manager", httpExecutor)
.withCircuitBreaker(configuration.circuitBreakerConfigurationName())
.withRetry(configuration.retryConfigurationName(), retryExecutor)
.withConnectTimeout(Duration.ofSeconds(10))
.withVersion(HttpClient.Version.HTTP_2)
.withNumClients(configuration.numHttpClients())

View File

@@ -12,6 +12,7 @@ import jakarta.validation.constraints.NotNull;
import java.util.List;
import java.util.Map;
import org.whispersystems.textsecuregcm.configuration.secrets.SecretString;
import javax.annotation.Nullable;
/**
* @param env The ios environment to use, typically SANDBOX or PRODUCTION
@@ -26,6 +27,8 @@ import org.whispersystems.textsecuregcm.configuration.secrets.SecretString;
* subscription levels
* @param appleRootCerts Apple root certificates to verify signed API responses, encoded as base64 strings:
* https://www.apple.com/certificateauthority/
* @param retryConfigurationName The name of the retry configuration to use in the App Store client; if `null`, uses the
* global default configuration.
*/
public record AppleAppStoreConfiguration(
@NotNull Environment env,
@@ -37,11 +40,5 @@ public record AppleAppStoreConfiguration(
@NotBlank String subscriptionGroupId,
@NotNull Map<String, Long> productIdToLevel,
@NotNull List<@NotBlank String> appleRootCerts,
@NotNull @Valid RetryConfiguration retry) {
public AppleAppStoreConfiguration {
if (retry == null) {
retry = new RetryConfiguration();
}
}
@Nullable String retryConfigurationName) {
}

View File

@@ -11,6 +11,7 @@ import jakarta.validation.constraints.NotEmpty;
import jakarta.validation.constraints.NotNull;
import java.util.Map;
import java.util.Set;
import javax.annotation.Nullable;
import org.whispersystems.textsecuregcm.configuration.secrets.SecretString;
import org.whispersystems.textsecuregcm.subscriptions.PaymentMethod;
@@ -22,7 +23,8 @@ import org.whispersystems.textsecuregcm.subscriptions.PaymentMethod;
* @param supportedCurrenciesByPaymentMethod the set of supported currencies
* @param graphqlUrl the Braintree GraphQL URl to use (this must match the environment)
* @param merchantAccounts merchant account within the merchant for processing individual currencies
* @param circuitBreaker configuration for the circuit breaker used by the GraphQL HTTP client
* @param circuitBreakerConfigurationName the name of the circuit breaker configuration for the breaker used by the
* GraphQL HTTP client; if `null`, uses the global default configuration
*/
public record BraintreeConfiguration(@NotBlank String merchantId,
@NotBlank String publicKey,
@@ -31,15 +33,6 @@ public record BraintreeConfiguration(@NotBlank String merchantId,
@Valid @NotEmpty Map<PaymentMethod, Set<@NotBlank String>> supportedCurrenciesByPaymentMethod,
@NotBlank String graphqlUrl,
@NotEmpty Map<String, String> merchantAccounts,
@NotNull @Valid CircuitBreakerConfiguration circuitBreaker,
@Nullable String circuitBreakerConfigurationName,
@Valid @NotNull PubSubPublisherFactory pubSubPublisher) {
public BraintreeConfiguration {
if (circuitBreaker == null) {
// Its a little counter-intuitive, but this compact constructor allows a default value
// to be used when one isnt specified (e.g. in YAML), allowing the field to still be
// validated as @NotNull
circuitBreaker = new CircuitBreakerConfiguration();
}
}
}

View File

@@ -5,6 +5,7 @@ import jakarta.validation.constraints.NotNull;
import java.util.Collections;
import java.util.Map;
import org.whispersystems.textsecuregcm.configuration.secrets.SecretString;
import javax.annotation.Nullable;
/**
* Configuration for the cdn3 storage manager
@@ -16,8 +17,10 @@ import org.whispersystems.textsecuregcm.configuration.secrets.SecretString;
* storage-manager when copying to determine how to read a source object. Current schemes are
* 'gcs' and 'r2'
* @param numHttpClients The number http clients to use with the storage-manager to support request striping
* @param circuitBreaker A circuit breaker configuration for the storage-manager http client
* @param retry A retry configuration for the storage-manager http client
* @param circuitBreakerConfigurationName The name of a circuit breaker configuration for the storage-manager http
* client; if `null`, uses the global default configuration
* @param retryConfigurationName The name of a retry configuration for the storage-manager http client; if
* `null`, uses the global default configuration
*/
public record Cdn3StorageManagerConfiguration(
@NotNull String baseUri,
@@ -25,8 +28,8 @@ public record Cdn3StorageManagerConfiguration(
@NotNull SecretString clientSecret,
@NotNull Map<Integer, String> sourceSchemes,
@NotNull Integer numHttpClients,
@NotNull @Valid CircuitBreakerConfiguration circuitBreaker,
@NotNull @Valid RetryConfiguration retry) {
@Nullable String circuitBreakerConfigurationName,
@Nullable String retryConfigurationName) {
public Cdn3StorageManagerConfiguration {
if (numHttpClients == null) {
@@ -35,11 +38,5 @@ public record Cdn3StorageManagerConfiguration(
if (sourceSchemes == null) {
sourceSchemes = Collections.emptyMap();
}
if (circuitBreaker == null) {
circuitBreaker = new CircuitBreakerConfiguration();
}
if (retry == null) {
retry = new RetryConfiguration();
}
}
}

View File

@@ -1,29 +0,0 @@
package org.whispersystems.textsecuregcm.configuration;
import com.fasterxml.jackson.annotation.JsonProperty;
import jakarta.validation.Valid;
import jakarta.validation.constraints.NotNull;
/**
* Configuration used to interact with a cdn via HTTP
*/
public class ClientCdnConfiguration {
@JsonProperty
@NotNull
@Valid
CircuitBreakerConfiguration circuitBreaker = new CircuitBreakerConfiguration();
@JsonProperty
@NotNull
@Valid
RetryConfiguration retry = new RetryConfiguration();
public CircuitBreakerConfiguration getCircuitBreaker() {
return circuitBreaker;
}
public RetryConfiguration getRetry() {
return retry;
}
}

View File

@@ -15,6 +15,7 @@ import java.time.Duration;
import java.util.List;
import jakarta.validation.constraints.Positive;
import org.whispersystems.textsecuregcm.configuration.secrets.SecretString;
import javax.annotation.Nullable;
/**
* Configuration properties for Cloudflare TURN integration.
@@ -27,8 +28,10 @@ import org.whispersystems.textsecuregcm.configuration.secrets.SecretString;
* @param urlsWithIps a collection of {@link String#format(String, Object...)} patterns to be populated with resolved IP
* addresses for {@link #hostname} in responses to clients; each pattern must include a single
* {@code %s} placeholder for the IP address
* @param circuitBreaker a circuit breaker for requests to Cloudflare
* @param retry a retry policy for requests to Cloudflare
* @param circuitBreakerConfigurationName the name of a circuit breaker configuration for requests to Cloudflare; if
* `null`, uses the global default configuration
* @param retryConfigurationName the name of a retry policy for requests to Cloudflare; if `null`, uses the global
* default configuration
* @param hostname the hostname to resolve to IP addresses for use with {@link #urlsWithIps}; also transmitted to
* clients for use as an SNI when connecting to pre-resolved hosts
* @param numHttpClients the number of parallel HTTP clients to use to communicate with Cloudflare
@@ -39,24 +42,11 @@ public record CloudflareTurnConfiguration(@NotNull SecretString apiToken,
@NotNull Duration clientCredentialTtl,
@NotNull @NotEmpty @Valid List<@NotBlank String> urls,
@NotNull @NotEmpty @Valid List<@NotBlank String> urlsWithIps,
@NotNull @Valid CircuitBreakerConfiguration circuitBreaker,
@NotNull @Valid RetryConfiguration retry,
@Nullable String circuitBreakerConfigurationName,
@Nullable String retryConfigurationName,
@NotBlank String hostname,
@Positive int numHttpClients) {
public CloudflareTurnConfiguration {
if (circuitBreaker == null) {
// Its a little counter-intuitive, but this compact constructor allows a default value
// to be used when one isnt specified (e.g. in YAML), allowing the field to still be
// validated as @NotNull
circuitBreaker = new CircuitBreakerConfiguration();
}
if (retry == null) {
retry = new RetryConfiguration();
}
}
@AssertTrue
@Schema(hidden = true)
public boolean isClientTtlShorterThanRequestedTtl() {

View File

@@ -9,10 +9,10 @@ import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonTypeName;
import com.google.common.annotations.VisibleForTesting;
import io.lettuce.core.resource.ClientResources;
import jakarta.validation.Valid;
import jakarta.validation.constraints.NotEmpty;
import jakarta.validation.constraints.NotNull;
import java.time.Duration;
import javax.annotation.Nullable;
import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisClusterClient;
@JsonTypeName("default")
@@ -27,9 +27,8 @@ public class RedisClusterConfiguration implements FaultTolerantRedisClusterFacto
private Duration timeout = Duration.ofSeconds(1);
@JsonProperty
@NotNull
@Valid
private CircuitBreakerConfiguration circuitBreaker = new CircuitBreakerConfiguration();
@Nullable
private String circuitBreakerConfigurationName;
@VisibleForTesting
void setConfigurationUri(final String configurationUri) {
@@ -44,8 +43,8 @@ public class RedisClusterConfiguration implements FaultTolerantRedisClusterFacto
return timeout;
}
public CircuitBreakerConfiguration getCircuitBreakerConfiguration() {
return circuitBreaker;
@Nullable public String getCircuitBreakerConfigurationName() {
return circuitBreakerConfigurationName;
}
@Override

View File

@@ -9,10 +9,10 @@ import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonTypeName;
import com.google.common.annotations.VisibleForTesting;
import io.lettuce.core.resource.ClientResources;
import jakarta.validation.Valid;
import jakarta.validation.constraints.NotEmpty;
import jakarta.validation.constraints.NotNull;
import java.time.Duration;
import javax.annotation.Nullable;
import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisClient;
@JsonTypeName("default")
@@ -27,9 +27,8 @@ public class RedisConfiguration implements FaultTolerantRedisClientFactory {
private Duration timeout = Duration.ofSeconds(1);
@JsonProperty
@NotNull
@Valid
private CircuitBreakerConfiguration circuitBreaker = new CircuitBreakerConfiguration();
@Nullable
private String circuitBreakerConfigurationName;
public String getUri() {
return uri;
@@ -44,8 +43,8 @@ public class RedisConfiguration implements FaultTolerantRedisClientFactory {
return timeout;
}
public @NotNull @Valid CircuitBreakerConfiguration getCircuitBreakerConfiguration() {
return circuitBreaker;
@Nullable public String getCircuitBreakerConfigurationName() {
return circuitBreakerConfigurationName;
}
@Override

View File

@@ -5,24 +5,16 @@
package org.whispersystems.textsecuregcm.configuration;
import jakarta.validation.Valid;
import jakarta.validation.constraints.NotBlank;
import jakarta.validation.constraints.NotEmpty;
import jakarta.validation.constraints.NotNull;
import java.util.List;
import javax.annotation.Nullable;
import org.whispersystems.textsecuregcm.configuration.secrets.SecretBytes;
public record SecureStorageServiceConfiguration(@NotNull SecretBytes userAuthenticationTokenSharedSecret,
@NotBlank String uri,
@NotEmpty List<@NotBlank String> storageCaCertificates,
@Valid CircuitBreakerConfiguration circuitBreaker,
@Valid RetryConfiguration retry) {
public SecureStorageServiceConfiguration {
if (circuitBreaker == null) {
circuitBreaker = new CircuitBreakerConfiguration();
}
if (retry == null) {
retry = new RetryConfiguration();
}
}
@Nullable String circuitBreakerConfigurationName,
@Nullable String retryConfigurationName) {
}

View File

@@ -11,22 +11,13 @@ import jakarta.validation.constraints.NotNull;
import java.util.List;
import org.whispersystems.textsecuregcm.configuration.secrets.SecretBytes;
import org.whispersystems.textsecuregcm.util.ExactlySize;
import javax.annotation.Nullable;
public record SecureValueRecoveryConfiguration(
@NotBlank String uri,
@ExactlySize(32) SecretBytes userAuthenticationTokenSharedSecret,
@ExactlySize(32) SecretBytes userIdTokenSharedSecret,
@NotEmpty List<@NotBlank String> svrCaCertificates,
@NotNull @Valid CircuitBreakerConfiguration circuitBreaker,
@NotNull @Valid RetryConfiguration retry) {
public SecureValueRecoveryConfiguration {
if (circuitBreaker == null) {
circuitBreaker = new CircuitBreakerConfiguration();
}
if (retry == null) {
retry = new RetryConfiguration();
}
}
@Nullable String circuitBreakerConfigurationName,
@Nullable String retryConfigurationName) {
}

View File

@@ -9,6 +9,7 @@ import com.google.common.annotations.VisibleForTesting;
import io.github.resilience4j.circuitbreaker.CircuitBreaker;
import io.github.resilience4j.retry.Retry;
import io.github.resilience4j.retry.RetryConfig;
import io.micrometer.core.instrument.Tags;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
@@ -16,6 +17,8 @@ import java.security.KeyStore;
import java.security.cert.CertificateException;
import java.time.Duration;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionStage;
import java.util.concurrent.Executor;
@@ -24,10 +27,8 @@ import java.util.concurrent.ThreadLocalRandom;
import java.util.function.Predicate;
import java.util.function.Supplier;
import java.util.stream.IntStream;
import io.micrometer.core.instrument.Tags;
import javax.annotation.Nullable;
import org.glassfish.jersey.SslConfigurator;
import org.whispersystems.textsecuregcm.configuration.CircuitBreakerConfiguration;
import org.whispersystems.textsecuregcm.configuration.RetryConfiguration;
import org.whispersystems.textsecuregcm.util.CertificateUtil;
import org.whispersystems.textsecuregcm.util.CircuitBreakerUtil;
import org.whispersystems.textsecuregcm.util.ExceptionUtils;
@@ -36,43 +37,31 @@ public class FaultTolerantHttpClient {
private final List<HttpClient> httpClients;
private final Duration defaultRequestTimeout;
private final ScheduledExecutorService retryExecutor;
private final Retry retry;
@Nullable private final ScheduledExecutorService retryExecutor;
@Nullable private final Retry retry;
private final CircuitBreaker breaker;
public static final String SECURITY_PROTOCOL_TLS_1_2 = "TLSv1.2";
public static final String SECURITY_PROTOCOL_TLS_1_3 = "TLSv1.3";
public static Builder newBuilder() {
return new Builder();
public static Builder newBuilder(final String name, final Executor executor) {
return new Builder(name, executor);
}
@VisibleForTesting
FaultTolerantHttpClient(String name, List<HttpClient> httpClients, ScheduledExecutorService retryExecutor,
Duration defaultRequestTimeout, RetryConfiguration retryConfiguration,
final Predicate<Throwable> retryOnException, CircuitBreakerConfiguration circuitBreakerConfiguration) {
FaultTolerantHttpClient(final List<HttpClient> httpClients,
final Duration defaultRequestTimeout,
@Nullable final ScheduledExecutorService retryExecutor,
@Nullable final Retry retry,
final CircuitBreaker circuitBreaker) {
this.httpClients = httpClients;
this.retryExecutor = retryExecutor;
this.defaultRequestTimeout = defaultRequestTimeout;
this.breaker = CircuitBreaker.of(name + "-breaker", circuitBreakerConfiguration.toCircuitBreakerConfig());
this.retryExecutor = retryExecutor;
this.retry = retry;
this.breaker = circuitBreaker;
CircuitBreakerUtil.registerMetrics(breaker, FaultTolerantHttpClient.class, Tags.empty());
if (retryConfiguration != null) {
if (this.retryExecutor == null) {
throw new IllegalArgumentException("retryExecutor must be specified with retryConfiguration");
}
final RetryConfig.Builder<HttpResponse> retryConfig = retryConfiguration.<HttpResponse>toRetryConfigBuilder()
.retryOnResult(o -> o.statusCode() >= 500);
if (retryOnException != null) {
retryConfig.retryOnException(retryOnException);
}
this.retry = Retry.of(name + "-retry", retryConfig.build());
CircuitBreakerUtil.registerMetrics(retry, FaultTolerantHttpClient.class);
} else {
this.retry = null;
}
}
private HttpClient httpClient() {
@@ -80,28 +69,30 @@ public class FaultTolerantHttpClient {
}
public <T> CompletableFuture<HttpResponse<T>> sendAsync(HttpRequest request,
HttpResponse.BodyHandler<T> bodyHandler) {
final HttpResponse.BodyHandler<T> bodyHandler) {
if (request.timeout().isEmpty()) {
request = HttpRequest.newBuilder(request, (n, v) -> true)
request = HttpRequest.newBuilder(request, (_, _) -> true)
.timeout(defaultRequestTimeout)
.build();
}
Supplier<CompletionStage<HttpResponse<T>>> asyncRequest = sendAsync(httpClient(), request, bodyHandler);
final Supplier<CompletionStage<HttpResponse<T>>> asyncRequestSupplier =
sendAsync(httpClient(), request, bodyHandler);
if (retry != null) {
return breaker.executeCompletionStage(retryableCompletionStage(asyncRequest)).toCompletableFuture();
} else {
return breaker.executeCompletionStage(asyncRequest).toCompletableFuture();
}
}
assert retryExecutor != null;
private <T> Supplier<CompletionStage<T>> retryableCompletionStage(Supplier<CompletionStage<T>> supplier) {
return () -> retry.executeCompletionStage(retryExecutor, supplier);
return breaker.executeCompletionStage(retry.decorateCompletionStage(retryExecutor, asyncRequestSupplier))
.toCompletableFuture();
} else {
return breaker.executeCompletionStage(asyncRequestSupplier).toCompletableFuture();
}
}
private <T> Supplier<CompletionStage<HttpResponse<T>>> sendAsync(HttpClient client, HttpRequest request,
HttpResponse.BodyHandler<T> bodyHandler) {
return () -> client.sendAsync(request, bodyHandler);
}
@@ -113,21 +104,18 @@ public class FaultTolerantHttpClient {
private Duration requestTimeout = Duration.ofSeconds(60);
private int numClients = 1;
private String name;
private final String name;
private Executor executor;
private ScheduledExecutorService retryExecutor;
private KeyStore trustStore;
private String securityProtocol = SECURITY_PROTOCOL_TLS_1_2;
private RetryConfiguration retryConfiguration;
private String retryConfigurationName;
private ScheduledExecutorService retryExecutor;
private Predicate<Throwable> retryOnException;
private CircuitBreakerConfiguration circuitBreakerConfiguration;
@Nullable private String circuitBreakerConfigurationName;
private Builder() {
}
public Builder withName(String name) {
this.name = name;
return this;
private Builder(final String name, final Executor executor) {
this.name = Objects.requireNonNull(name);
this.executor = Objects.requireNonNull(executor);
}
public Builder withVersion(HttpClient.Version version) {
@@ -140,11 +128,6 @@ public class FaultTolerantHttpClient {
return this;
}
public Builder withExecutor(Executor executor) {
this.executor = executor;
return this;
}
public Builder withConnectTimeout(Duration connectTimeout) {
this.connectTimeout = connectTimeout;
return this;
@@ -155,18 +138,15 @@ public class FaultTolerantHttpClient {
return this;
}
public Builder withRetry(RetryConfiguration retryConfiguration) {
this.retryConfiguration = retryConfiguration;
return this;
}
public Builder withRetryExecutor(ScheduledExecutorService retryExecutor) {
public Builder withRetry(@Nullable final String retryConfigurationName, final ScheduledExecutorService retryExecutor) {
this.retryConfigurationName = retryConfigurationName;
this.retryExecutor = retryExecutor;
return this;
}
public Builder withCircuitBreaker(CircuitBreakerConfiguration circuitBreakerConfiguration) {
this.circuitBreakerConfiguration = circuitBreakerConfiguration;
public Builder withCircuitBreaker(@Nullable final String circuitBreakerConfigurationName) {
this.circuitBreakerConfigurationName = circuitBreakerConfigurationName;
return this;
}
@@ -206,10 +186,6 @@ public class FaultTolerantHttpClient {
}
public FaultTolerantHttpClient build() {
if (this.circuitBreakerConfiguration == null || this.name == null || this.executor == null) {
throw new IllegalArgumentException("Must specify circuit breaker config, name, and executor");
}
if (numClients > 1 && version != HttpClient.Version.HTTP_2) {
throw new IllegalArgumentException("Should not use additional HTTP clients unless using HTTP/2");
}
@@ -232,8 +208,32 @@ public class FaultTolerantHttpClient {
return builder.build();
}).toList();
return new FaultTolerantHttpClient(name, httpClients, retryExecutor, requestTimeout, retryConfiguration,
retryOnException, circuitBreakerConfiguration);
@Nullable final Retry retry;
if (retryExecutor != null) {
final RetryConfig.Builder<HttpResponse<?>> retryConfigBuilder =
RetryConfig.from(Optional.ofNullable(retryConfigurationName)
.flatMap(name -> CircuitBreakerUtil.getRetryRegistry().getConfiguration(name))
.orElseGet(() -> CircuitBreakerUtil.getRetryRegistry().getDefaultConfig()));
retryConfigBuilder.retryOnResult(response -> response.statusCode() >= 500);
if (retryOnException != null) {
retryConfigBuilder.retryOnException(retryOnException);
}
retry = CircuitBreakerUtil.getRetryRegistry().retry(name + "-retry", retryConfigBuilder.build());
} else {
retry = null;
}
final String circuitBreakerName = name + "-breaker";
final CircuitBreaker circuitBreaker = circuitBreakerConfigurationName != null
? CircuitBreakerUtil.getCircuitBreakerRegistry().circuitBreaker(circuitBreakerName, circuitBreakerConfigurationName)
: CircuitBreakerUtil.getCircuitBreakerRegistry().circuitBreaker(circuitBreakerName);
return new FaultTolerantHttpClient(httpClients, requestTimeout, retryExecutor, retry, circuitBreaker);
}
}

View File

@@ -1,5 +1,6 @@
package org.whispersystems.textsecuregcm.redis;
import com.google.common.annotations.VisibleForTesting;
import io.github.resilience4j.circuitbreaker.CircuitBreaker;
import io.lettuce.core.ClientOptions;
import io.lettuce.core.RedisClient;
@@ -15,8 +16,8 @@ import java.util.ArrayList;
import java.util.List;
import java.util.function.Consumer;
import java.util.function.Function;
import org.whispersystems.textsecuregcm.configuration.CircuitBreakerConfiguration;
import org.whispersystems.textsecuregcm.configuration.RedisConfiguration;
import org.whispersystems.textsecuregcm.util.CircuitBreakerUtil;
public class FaultTolerantRedisClient {
@@ -38,14 +39,17 @@ public class FaultTolerantRedisClient {
this(name, clientResourcesBuilder,
RedisUriUtil.createRedisUriWithTimeout(redisConfiguration.getUri(), redisConfiguration.getTimeout()),
redisConfiguration.getTimeout(),
redisConfiguration.getCircuitBreakerConfiguration());
redisConfiguration.getCircuitBreakerConfigurationName() != null
? CircuitBreakerUtil.getCircuitBreakerRegistry().circuitBreaker(name + "-breaker", redisConfiguration.getCircuitBreakerConfigurationName())
: CircuitBreakerUtil.getCircuitBreakerRegistry().circuitBreaker(name + "-breaker"));
}
@VisibleForTesting
FaultTolerantRedisClient(String name,
final ClientResources.Builder clientResourcesBuilder,
final RedisURI redisUri,
final Duration commandTimeout,
final CircuitBreakerConfiguration circuitBreakerConfiguration) {
final CircuitBreaker circuitBreaker) {
this.name = name;
@@ -73,7 +77,7 @@ public class FaultTolerantRedisClient {
this.stringConnection = redisClient.connect();
this.binaryConnection = redisClient.connect(ByteArrayCodec.INSTANCE);
this.circuitBreaker = CircuitBreaker.of(name + "-breaker", circuitBreakerConfiguration.toCircuitBreakerConfig());
this.circuitBreaker = circuitBreaker;
}
public void shutdown() {

View File

@@ -26,7 +26,7 @@ import java.util.Collections;
import java.util.List;
import java.util.function.Consumer;
import java.util.function.Function;
import org.whispersystems.textsecuregcm.configuration.CircuitBreakerConfiguration;
import javax.annotation.Nullable;
import org.whispersystems.textsecuregcm.configuration.RedisClusterConfiguration;
import reactor.core.scheduler.Schedulers;
@@ -57,7 +57,7 @@ public class FaultTolerantRedisClusterClient {
Collections.singleton(RedisUriUtil.createRedisUriWithTimeout(clusterConfiguration.getConfigurationUri(),
clusterConfiguration.getTimeout())),
clusterConfiguration.getTimeout(),
clusterConfiguration.getCircuitBreakerConfiguration());
clusterConfiguration.getCircuitBreakerConfigurationName());
}
@@ -65,7 +65,7 @@ public class FaultTolerantRedisClusterClient {
final ClientResources.Builder clientResourcesBuilder,
final Iterable<RedisURI> redisUris,
final Duration commandTimeout,
final CircuitBreakerConfiguration circuitBreakerConfig) {
@Nullable final String circuitBreakerConfigurationName) {
this.name = name;
@@ -83,7 +83,7 @@ public class FaultTolerantRedisClusterClient {
});
final LettuceShardCircuitBreaker lettuceShardCircuitBreaker = new LettuceShardCircuitBreaker(name,
circuitBreakerConfig.toCircuitBreakerConfig(), Schedulers.newSingle("topology-changed-" + name, true));
circuitBreakerConfigurationName, Schedulers.newSingle("topology-changed-" + name, true));
this.clusterClient = RedisClusterClient.create(
clientResourcesBuilder.nettyCustomizer(lettuceShardCircuitBreaker).
build(),

View File

@@ -8,7 +8,6 @@ package org.whispersystems.textsecuregcm.redis;
import com.google.common.annotations.VisibleForTesting;
import io.github.resilience4j.circuitbreaker.CallNotPermittedException;
import io.github.resilience4j.circuitbreaker.CircuitBreaker;
import io.github.resilience4j.circuitbreaker.CircuitBreakerConfig;
import io.lettuce.core.RedisNoScriptException;
import io.lettuce.core.cluster.event.ClusterTopologyChangedEvent;
import io.lettuce.core.cluster.models.partitions.RedisClusterNode;
@@ -31,6 +30,7 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;
import javax.annotation.Nullable;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -54,7 +54,8 @@ public class LettuceShardCircuitBreaker implements NettyCustomizer {
private static final Logger logger = LoggerFactory.getLogger(LettuceShardCircuitBreaker.class);
private final String clusterName;
private final CircuitBreakerConfig circuitBreakerConfig;
@Nullable
private final String circuitBreakerConfigurationName;
private final Scheduler scheduler;
// this set will be shared with all child channel breakers
private final Set<String> upstreamAddresses = ConcurrentHashMap.newKeySet();
@@ -62,10 +63,12 @@ public class LettuceShardCircuitBreaker implements NettyCustomizer {
// resources, which cannot be built without this NettyCustomizer
private EventBus eventBus;
public LettuceShardCircuitBreaker(final String clusterName, final CircuitBreakerConfig circuitBreakerConfig,
public LettuceShardCircuitBreaker(final String clusterName,
@Nullable final String circuitBreakerConfigurationName,
final Scheduler scheduler) {
this.clusterName = clusterName;
this.circuitBreakerConfig = circuitBreakerConfig;
this.circuitBreakerConfigurationName = circuitBreakerConfigurationName;
this.scheduler = scheduler;
}
@@ -110,7 +113,7 @@ public class LettuceShardCircuitBreaker implements NettyCustomizer {
}
final ChannelCircuitBreakerHandler channelCircuitBreakerHandler = new ChannelCircuitBreakerHandler(clusterName,
circuitBreakerConfig, upstreamAddresses, eventBus, scheduler);
circuitBreakerConfigurationName, upstreamAddresses, eventBus, scheduler);
final String commandHandlerName = StreamSupport.stream(channel.pipeline().spliterator(), false)
.filter(entry -> entry.getValue() instanceof CommandHandler)
@@ -127,7 +130,7 @@ public class LettuceShardCircuitBreaker implements NettyCustomizer {
private static final String CLUSTER_TAG_NAME = "cluster";
private final String clusterName;
private final CircuitBreakerConfig circuitBreakerConfig;
@Nullable private final String circuitBreakerConfigurationName;
private final AtomicBoolean registeredMetrics = new AtomicBoolean(false);
private final Set<String> upstreamAddresses;
@@ -136,11 +139,12 @@ public class LettuceShardCircuitBreaker implements NettyCustomizer {
@VisibleForTesting
CircuitBreaker breaker;
public ChannelCircuitBreakerHandler(final String name, final CircuitBreakerConfig circuitBreakerConfig,
public ChannelCircuitBreakerHandler(final String name,
@Nullable final String circuitBreakerConfigurationName,
final Set<String> upstreamAddresses,
final EventBus eventBus, final Scheduler scheduler) {
this.clusterName = name;
this.circuitBreakerConfig = circuitBreakerConfig;
this.circuitBreakerConfigurationName = circuitBreakerConfigurationName;
this.upstreamAddresses = upstreamAddresses;
eventBus.get()
@@ -183,7 +187,12 @@ public class LettuceShardCircuitBreaker implements NettyCustomizer {
// In some cases, like the default connection, the remote address includes the DNS hostname, which we want to exclude.
shardAddress = StringUtils.substringAfter(remoteAddress.toString(), "/");
breaker = CircuitBreaker.of("%s/%s-breaker".formatted(clusterName, shardAddress), circuitBreakerConfig);
final String circuitBreakerName = "%s/%s-breaker".formatted(clusterName, shardAddress);
breaker = circuitBreakerConfigurationName != null
? CircuitBreakerUtil.getCircuitBreakerRegistry().circuitBreaker(circuitBreakerName, circuitBreakerConfigurationName)
: CircuitBreakerUtil.getCircuitBreakerRegistry().circuitBreaker(circuitBreakerName);
if (upstreamAddresses.contains(shardAddress)) {
registerMetrics();

View File

@@ -43,15 +43,12 @@ public class SecureStorageClient {
throws CertificateException {
this.storageServiceCredentialsGenerator = storageServiceCredentialsGenerator;
this.deleteUri = URI.create(configuration.uri()).resolve(DELETE_PATH);
this.httpClient = FaultTolerantHttpClient.newBuilder()
.withCircuitBreaker(configuration.circuitBreaker())
.withRetry(configuration.retry())
.withRetryExecutor(retryExecutor)
this.httpClient = FaultTolerantHttpClient.newBuilder("secure-storage", executor)
.withCircuitBreaker(configuration.circuitBreakerConfigurationName())
.withRetry(configuration.retryConfigurationName(), retryExecutor)
.withVersion(HttpClient.Version.HTTP_1_1)
.withConnectTimeout(Duration.ofSeconds(10))
.withRedirect(HttpClient.Redirect.NEVER)
.withExecutor(executor)
.withName("secure-storage")
.withSecurityProtocol(FaultTolerantHttpClient.SECURITY_PROTOCOL_TLS_1_3)
.withTrustedServerCertificates(configuration.storageCaCertificates().toArray(new String[0]))
.build();

View File

@@ -53,15 +53,12 @@ public class SecureValueRecoveryClient {
this.secureValueRecoveryCredentialsGenerator = secureValueRecoveryCredentialsGenerator;
this.deleteUri = URI.create(configuration.uri()).resolve(DELETE_PATH);
this.allowedDeletionErrorStatusCodes = allowedDeletionErrorStatusCodes;
this.httpClient = FaultTolerantHttpClient.newBuilder()
.withCircuitBreaker(configuration.circuitBreaker())
.withRetry(configuration.retry())
.withRetryExecutor(retryExecutor)
this.httpClient = FaultTolerantHttpClient.newBuilder("secure-value-recovery", executor)
.withCircuitBreaker(configuration.circuitBreakerConfigurationName())
.withRetry(configuration.retryConfigurationName(), retryExecutor)
.withVersion(HttpClient.Version.HTTP_1_1)
.withConnectTimeout(Duration.ofSeconds(10))
.withRedirect(HttpClient.Redirect.NEVER)
.withExecutor(executor)
.withName("secure-value-recovery")
.withSecurityProtocol(FaultTolerantHttpClient.SECURITY_PROTOCOL_TLS_1_2)
.withTrustedServerCertificates(configuration.svrCaCertificates().toArray(new String[0]))
.build();

View File

@@ -19,17 +19,20 @@ import com.apple.itunes.storekit.verification.SignedDataVerifier;
import com.apple.itunes.storekit.verification.VerificationException;
import com.google.common.annotations.VisibleForTesting;
import io.github.resilience4j.retry.Retry;
import io.github.resilience4j.retry.RetryConfig;
import io.micrometer.core.instrument.Metrics;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.math.BigDecimal;
import java.net.http.HttpResponse;
import java.time.Instant;
import java.util.Base64;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionStage;
@@ -37,13 +40,14 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.ScheduledExecutorService;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import javax.annotation.Nullable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.whispersystems.textsecuregcm.configuration.RetryConfiguration;
import org.whispersystems.textsecuregcm.controllers.RateLimitExceededException;
import org.whispersystems.textsecuregcm.metrics.MetricsUtil;
import org.whispersystems.textsecuregcm.storage.PaymentTime;
import org.whispersystems.textsecuregcm.storage.SubscriptionException;
import org.whispersystems.textsecuregcm.util.CircuitBreakerUtil;
import org.whispersystems.textsecuregcm.util.ExceptionUtils;
/**
@@ -82,12 +86,12 @@ public class AppleAppStoreManager implements SubscriptionPaymentProcessor {
final String subscriptionGroupId,
final Map<String, Long> productIdToLevel,
final List<String> base64AppleRootCerts,
final RetryConfiguration retryConfiguration,
@Nullable final String retryConfigurationName,
final ExecutorService executor,
final ScheduledExecutorService retryExecutor) {
this(new AppStoreServerAPIClient(encodedKey, keyId, issuerId, bundleId, env),
new SignedDataVerifier(decodeRootCerts(base64AppleRootCerts), bundleId, appAppleId, env, true),
subscriptionGroupId, productIdToLevel, retryConfiguration, executor, retryExecutor);
subscriptionGroupId, productIdToLevel, retryConfigurationName, executor, retryExecutor);
}
@VisibleForTesting
@@ -96,18 +100,24 @@ public class AppleAppStoreManager implements SubscriptionPaymentProcessor {
final SignedDataVerifier signedDataVerifier,
final String subscriptionGroupId,
final Map<String, Long> productIdToLevel,
final RetryConfiguration retryConfiguration,
@Nullable final String retryConfigurationName,
final ExecutorService executor,
final ScheduledExecutorService retryExecutor) {
this.apiClient = apiClient;
this.signedDataVerifier = signedDataVerifier;
this.subscriptionGroupId = subscriptionGroupId;
this.productIdToLevel = productIdToLevel;
this.retry = Retry.of("appstore-retry", retryConfiguration
.toRetryConfigBuilder()
.retryOnException(AppleAppStoreManager::shouldRetry).build());
this.executor = Objects.requireNonNull(executor);
this.retryExecutor = Objects.requireNonNull(retryExecutor);
final RetryConfig.Builder<HttpResponse<?>> retryConfigBuilder =
RetryConfig.from(Optional.ofNullable(retryConfigurationName)
.flatMap(name -> CircuitBreakerUtil.getRetryRegistry().getConfiguration(name))
.orElseGet(() -> CircuitBreakerUtil.getRetryRegistry().getDefaultConfig()));
retryConfigBuilder.retryOnException(AppleAppStoreManager::shouldRetry);
this.retry = CircuitBreakerUtil.getRetryRegistry().retry("appstore-retry", retryConfigBuilder.build());
}
@Override

View File

@@ -42,7 +42,6 @@ import java.util.concurrent.ScheduledExecutorService;
import javax.annotation.Nullable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.whispersystems.textsecuregcm.configuration.CircuitBreakerConfiguration;
import org.whispersystems.textsecuregcm.currency.CurrencyConversionManager;
import org.whispersystems.textsecuregcm.http.FaultTolerantHttpClient;
import org.whispersystems.textsecuregcm.metrics.MetricsUtil;
@@ -81,7 +80,7 @@ public class BraintreeManager implements CustomerAwareSubscriptionPaymentProcess
final String graphqlUri,
final CurrencyConversionManager currencyConversionManager,
final PublisherInterface pubsubPublisher,
final CircuitBreakerConfiguration circuitBreakerConfiguration,
@Nullable final String circuitBreakerConfigurationName,
final Executor executor,
final ScheduledExecutorService retryExecutor) {
@@ -89,11 +88,8 @@ public class BraintreeManager implements CustomerAwareSubscriptionPaymentProcess
braintreePrivateKey),
supportedCurrenciesByPaymentMethod,
currenciesToMerchantAccounts,
new BraintreeGraphqlClient(FaultTolerantHttpClient.newBuilder()
.withName("braintree-graphql")
.withCircuitBreaker(circuitBreakerConfiguration)
.withExecutor(executor)
.withRetryExecutor(retryExecutor)
new BraintreeGraphqlClient(FaultTolerantHttpClient.newBuilder("braintree-graphql", executor)
.withCircuitBreaker(circuitBreakerConfigurationName)
// Braintree documents its internal timeout at 60 seconds, and we want to make sure we dont miss
// a response
// https://developer.paypal.com/braintree/docs/reference/general/best-practices/java#timeouts

View File

@@ -8,11 +8,15 @@ package org.whispersystems.textsecuregcm.util;
import static com.codahale.metrics.MetricRegistry.name;
import io.github.resilience4j.circuitbreaker.CircuitBreaker;
import io.github.resilience4j.circuitbreaker.CircuitBreakerRegistry;
import io.github.resilience4j.retry.Retry;
import io.github.resilience4j.retry.RetryRegistry;
import io.micrometer.core.instrument.Counter;
import io.micrometer.core.instrument.Metrics;
import io.micrometer.core.instrument.Tag;
import io.micrometer.core.instrument.Tags;
import org.whispersystems.textsecuregcm.configuration.CircuitBreakerConfiguration;
import org.whispersystems.textsecuregcm.configuration.RetryConfiguration;
public class CircuitBreakerUtil {
@@ -23,6 +27,22 @@ public class CircuitBreakerUtil {
private static final String BREAKER_NAME_TAG_NAME = "breakerName";
private static final String OUTCOME_TAG_NAME = "outcome";
private static final CircuitBreakerRegistry CIRCUIT_BREAKER_REGISTRY =
CircuitBreakerRegistry.of(new CircuitBreakerConfiguration().toCircuitBreakerConfig());
private static final RetryRegistry RETRY_REGISTRY =
RetryRegistry.of(new RetryConfiguration().toRetryConfigBuilder().build());
public static CircuitBreakerRegistry getCircuitBreakerRegistry() {
return CIRCUIT_BREAKER_REGISTRY;
}
public static RetryRegistry getRetryRegistry() {
return RETRY_REGISTRY;
}
/// @deprecated [CircuitBreakerRegistry] maintains its own set of metrics, and manually managing them is unnecessary
@Deprecated(forRemoval = true)
public static void registerMetrics(CircuitBreaker circuitBreaker, Class<?> clazz, Tags additionalTags) {
final String breakerName = clazz.getSimpleName() + "/" + circuitBreaker.getName();
@@ -57,6 +77,8 @@ public class CircuitBreakerUtil {
circuitBreaker, breaker -> breaker.getState().getOrder());
}
/// @deprecated [RetryRegistry] maintains its own set of metrics, and manually managing them is unnecessary
@Deprecated(forRemoval = true)
public static void registerMetrics(Retry retry, Class<?> clazz) {
final String retryName = clazz.getSimpleName() + "/" + retry.getName();