Signal/Signal-Server
1
0
Fork 0
mirror of https://github.com/signalapp/Signal-Server synced 2026-04-20 09:47:58 +01:00
Code Issues Packages Projects Releases Wiki Activity

Automatically register metrics/instrument breaker/retry instances on creation

Browse Source
This commit is contained in:
Jon Chambers
2025-08-28 09:35:02 -04:00
committed by Jon Chambers
parent 0e1e248564
commit 8060d74899
1 changed files with 139 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

139
service/src/main/java/org/whispersystems/textsecuregcm/util/ResilienceUtil.java
View File

@@ -5,13 +5,27 @@
package org.whispersystems.textsecuregcm.util;
import io.github.resilience4j.circuitbreaker.CircuitBreaker;
import io.github.resilience4j.circuitbreaker.CircuitBreakerRegistry;
import io.github.resilience4j.retry.Retry;
import io.github.resilience4j.retry.RetryRegistry;
import io.lettuce.core.RedisCommandTimeoutException;
import io.micrometer.core.instrument.Counter;
import io.micrometer.core.instrument.Gauge;
import io.micrometer.core.instrument.Meter;
import io.micrometer.core.instrument.Metrics;
import io.micrometer.core.instrument.Tag;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import javax.annotation.Nullable;
import org.apache.commons.lang3.RandomStringUtils;
import org.whispersystems.textsecuregcm.configuration.CircuitBreakerConfiguration;
import org.whispersystems.textsecuregcm.configuration.RetryConfiguration;
import org.whispersystems.textsecuregcm.metrics.MetricsUtil;
public class ResilienceUtil {
@@ -21,12 +35,39 @@ public class ResilienceUtil {
private static final RetryRegistry RETRY_REGISTRY =
RetryRegistry.of(new RetryConfiguration().toRetryConfigBuilder().build());
private static final ConcurrentMap<String, Set<Meter.Id>> METER_IDS_BY_BREAKER_NAME = new ConcurrentHashMap<>();
private static final ConcurrentMap<String, Set<Meter.Id>> METER_IDS_BY_RETRY_NAME = new ConcurrentHashMap<>();
private static final String BREAKER_CALL_COUNTER_NAME = MetricsUtil.name(ResilienceUtil.class, "breaker", "call");
private static final String BREAKER_STATE_GAUGE_NAME = MetricsUtil.name(ResilienceUtil.class, "breaker", "state");
private static final String RETRY_CALL_COUNTER_NAME = MetricsUtil.name(ResilienceUtil.class, "retry", "call");
private static final String BREAKER_NAME_TAG_NAME = "breakerName";
private static final String RETRY_NAME_TAG = "retryName";
private static final String OUTCOME_TAG_NAME = "outcome";
// Include a random suffix to avoid accidental collisions
private static final String GENERAL_REDIS_CONFIGURATION_NAME =
"redis-general-" + RandomStringUtils.insecure().nextAlphanumeric(8);
static {
setGeneralRedisRetryConfiguration(new RetryConfiguration());
CIRCUIT_BREAKER_REGISTRY.getEventPublisher()
.onEntryAdded(event -> addMetrics(event.getAddedEntry()))
.onEntryRemoved(event -> removeMetrics(event.getRemovedEntry()))
.onEntryReplaced(event -> {
removeMetrics(event.getOldEntry());
addMetrics(event.getNewEntry());
});
RETRY_REGISTRY.getEventPublisher()
.onEntryAdded(event -> addMetrics(event.getAddedEntry()))
.onEntryRemoved(event -> removeMetrics(event.getRemovedEntry()))
.onEntryReplaced(event -> {
removeMetrics(event.getOldEntry());
addMetrics(event.getNewEntry());
});
}
public static CircuitBreakerRegistry getCircuitBreakerRegistry() {
@@ -50,4 +91,102 @@ public class ResilienceUtil {
public static Retry getGeneralRedisRetry(final String name) {
return RETRY_REGISTRY.retry("redis/" + name, GENERAL_REDIS_CONFIGURATION_NAME);
}
private static void addMetrics(final CircuitBreaker circuitBreaker) {
// Remove previous meters before registering new ones
final Set<Meter.Id> meterIds = new HashSet<>();
final List<Tag> additionalTags = toTags(circuitBreaker.getTags());
meterIds.add(Gauge.builder(BREAKER_STATE_GAUGE_NAME, circuitBreaker, breaker -> switch (breaker.getState()) {
case OPEN, HALF_OPEN, FORCED_OPEN -> 1;
default -> 0;
})
.tag(BREAKER_NAME_TAG_NAME, circuitBreaker.getName())
.tags(additionalTags)
.register(Metrics.globalRegistry)
.getId());
final Counter successCounter = Counter.builder(BREAKER_CALL_COUNTER_NAME)
.tag(BREAKER_NAME_TAG_NAME, circuitBreaker.getName())
.tag(OUTCOME_TAG_NAME, "success")
.tags(additionalTags)
.register(Metrics.globalRegistry);
final Counter failureCounter = Counter.builder(BREAKER_CALL_COUNTER_NAME)
.tag(BREAKER_NAME_TAG_NAME, circuitBreaker.getName())
.tag(OUTCOME_TAG_NAME, "failure")
.tags(additionalTags)
.register(Metrics.globalRegistry);
final Counter unpermittedCounter = Counter.builder(BREAKER_CALL_COUNTER_NAME)
.tag(BREAKER_NAME_TAG_NAME, circuitBreaker.getName())
.tag(OUTCOME_TAG_NAME, "unpermitted")
.tags(additionalTags)
.register(Metrics.globalRegistry);
circuitBreaker.getEventPublisher()
.onSuccess(_ -> successCounter.increment())
.onError(_ -> failureCounter.increment())
.onCallNotPermitted(_ -> unpermittedCounter.increment());
meterIds.add(successCounter.getId());
meterIds.add(failureCounter.getId());
meterIds.add(unpermittedCounter.getId());
METER_IDS_BY_BREAKER_NAME.put(circuitBreaker.getName(), meterIds);
}
private static void addMetrics(final Retry retry) {
final Set<Meter.Id> meterIds = new HashSet<>();
final List<Tag> additionalTags = toTags(retry.getTags());
final Counter successCounter = Counter.builder(RETRY_CALL_COUNTER_NAME)
.tag(RETRY_NAME_TAG, retry.getName())
.tag(OUTCOME_TAG_NAME, "success")
.tags(additionalTags)
.register(Metrics.globalRegistry);
final Counter retryCounter = Counter.builder(RETRY_CALL_COUNTER_NAME)
.tag(RETRY_NAME_TAG, retry.getName())
.tag(OUTCOME_TAG_NAME, "retry")
.tags(additionalTags)
.register(Metrics.globalRegistry);
final Counter errorCounter = Counter.builder(RETRY_CALL_COUNTER_NAME)
.tag(RETRY_NAME_TAG, retry.getName())
.tag(OUTCOME_TAG_NAME, "error")
.tags(additionalTags)
.register(Metrics.globalRegistry);
retry.getEventPublisher()
.onSuccess(_ -> successCounter.increment())
.onRetry(_ -> retryCounter.increment())
.onError(_ -> errorCounter.increment());
meterIds.add(successCounter.getId());
meterIds.add(retryCounter.getId());
meterIds.add(errorCounter.getId());
METER_IDS_BY_RETRY_NAME.put(retry.getName(), meterIds);
}
private static void removeMetrics(final CircuitBreaker circuitBreaker) {
removeMetrics(METER_IDS_BY_BREAKER_NAME.remove(circuitBreaker.getName()));
}
private static void removeMetrics(final Retry retry) {
removeMetrics(METER_IDS_BY_RETRY_NAME.remove(retry.getName()));
}
private static void removeMetrics(@Nullable final Set<Meter.Id> meterIds) {
if (meterIds != null) {
meterIds.forEach(Metrics.globalRegistry::remove);
}
}
private static List<Tag> toTags(final Map<String, String> tagMap) {
return tagMap.entrySet().stream()
.map(entry -> Tag.of(entry.getKey(), entry.getValue()))
.toList();
}
}