diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/metrics/MetricsUtil.java b/service/src/main/java/org/whispersystems/textsecuregcm/metrics/MetricsUtil.java index 5d7b3ead7..08119bee7 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/metrics/MetricsUtil.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/metrics/MetricsUtil.java @@ -22,6 +22,7 @@ import io.micrometer.core.instrument.distribution.DistributionStatisticConfig; import io.micrometer.registry.otlp.OtlpMeterRegistry; import io.micrometer.statsd.StatsdMeterRegistry; import java.time.Duration; +import java.util.Set; import org.whispersystems.textsecuregcm.WhisperServerConfiguration; import org.whispersystems.textsecuregcm.WhisperServerVersion; import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicConfiguration; @@ -32,6 +33,9 @@ public class MetricsUtil { public static final String PREFIX = "chat"; + private static Set ALLOWED_R4J_METRICS = Set.of("resilience4j.circuitbreaker.calls", "resilience4j.circuitbreaker.not.permitted.calls", "resilience4j.circuitbreaker.state"); + private static Set ALLOWED_R4J_STATE_GAUGES = Set.of("open", "closed", "half_open"); + private static volatile boolean registeredMetrics = false; /** @@ -84,6 +88,7 @@ public class MetricsUtil { config.getOpenTelemetryConfiguration(), io.micrometer.core.instrument.Clock.SYSTEM); configureMeterFilters(otlpMeterRegistry.config(), dynamicConfigurationManager); + configureCircuitBreakerMeterFilters(otlpMeterRegistry.config()); Metrics.addRegistry(otlpMeterRegistry); if (config.getOpenTelemetryConfiguration().shutdownWaitDuration().compareTo(shutdownWaitDuration) > 0) { @@ -136,6 +141,36 @@ public class MetricsUtil { && id.getName().startsWith(awsSdkMetricNamePrefix))); } + // A separate function from configureMeterFilters only so we can use it on OTLP but not statsd + static void configureCircuitBreakerMeterFilters(MeterRegistry.Config config) { + config.meterFilter( + MeterFilter.deny( + id -> id.getName().equals("resilience4j.circuitbreaker.state") && + id.getTags().stream().anyMatch( + t -> t.getKey().equals("state") && !ALLOWED_R4J_STATE_GAUGES.contains(t.getValue())))); + + config.meterFilter( + MeterFilter.deny( + id -> id.getName().equals("resilience4j.circuitbreaker.calls") && + id.getTags().stream().anyMatch( + t -> t.getKey().equals("kind") && t.getValue().equals("ignored")))); + + config.meterFilter( + MeterFilter.deny( + id -> id.getName().startsWith("resilience4j.circuitbreaker") && + !ALLOWED_R4J_METRICS.contains(id.getName()))); + + config.meterFilter(new MeterFilter() { + @Override + public DistributionStatisticConfig configure(final Meter.Id id, final DistributionStatisticConfig config) { + if (id.getName().equals("resilience4j.circuitbreaker.calls")) { + return DistributionStatisticConfig.NONE; + } + return config; + } + }); + } + public static void registerSystemResourceMetrics(final Environment environment) { new ProcessorMetrics().bindTo(Metrics.globalRegistry); new FileDescriptorMetrics().bindTo(Metrics.globalRegistry); diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/redis/LettuceShardCircuitBreaker.java b/service/src/main/java/org/whispersystems/textsecuregcm/redis/LettuceShardCircuitBreaker.java index aafefc0aa..4b6682684 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/redis/LettuceShardCircuitBreaker.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/redis/LettuceShardCircuitBreaker.java @@ -32,14 +32,6 @@ import org.whispersystems.textsecuregcm.util.ResilienceUtil; /** * Adds a circuit breaker to every Netty {@link Channel} that gets created, so that a single unhealthy shard does not * impact all cluster operations. - *

- * For metrics to be registered, users must create a synthetic {@link ClusterTopologyChangedEvent} after the - * initial connection. For example: - *

- *   clusterClient.connect();
- *   clusterClient.getResources().eventBus().publish(
- *         new ClusterTopologyChangedEvent(Collections.emptyList(), clusterClient.getPartitions().getPartitions()));
- * 
*/ public class LettuceShardCircuitBreaker implements NettyCustomizer {