FaultTolerantHttpClient: used managed ScheduledExecutorService for retries

2026-04-21 17:29:38 +01:00 · 2023-06-23 10:48:38 -05:00
parent 8e48ac4ede
commit b852d6681d
13 changed files with 346 additions and 252 deletions
--- a/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java
+++ b/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java
@@ -379,6 +379,10 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
        .executorService(name(getClass(), "secureValueRecoveryService-%d")).maxThreads(1).minThreads(1).build();
    ExecutorService storageServiceExecutor = environment.lifecycle()
        .executorService(name(getClass(), "storageService-%d")).maxThreads(1).minThreads(1).build();
+    ScheduledExecutorService secureValueRecoveryServiceRetryExecutor = environment.lifecycle()
+        .scheduledExecutorService(name(getClass(), "secureValueRecoveryServiceRetry-%d")).threads(1).build();
+    ScheduledExecutorService storageServiceRetryExecutor = environment.lifecycle()
+        .scheduledExecutorService(name(getClass(), "storageServiceRetry-%d")).threads(1).build();

    Scheduler messageDeliveryScheduler = Schedulers.fromExecutorService(
        ExecutorServiceMetrics.monitor(Metrics.globalRegistry,
@@ -411,6 +415,8 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
        .maxThreads(2)
        .minThreads(2)
        .build();
+    ScheduledExecutorService subscriptionProcessorRetryExecutor = environment.lifecycle()
+        .scheduledExecutorService(name(getClass(), "subscriptionProcessorRetry-%d")).threads(1).build();

    final AdminEventLogger adminEventLogger = new GoogleCloudAdminEventLogger(
        LoggingOptions.newBuilder().setProjectId(config.getAdminEventLoggingConfiguration().projectId())
@@ -426,9 +432,11 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
        config.getStripe().idempotencyKeyGenerator().value(), config.getStripe().boostDescription(), config.getStripe()
        .supportedCurrencies());
    BraintreeManager braintreeManager = new BraintreeManager(config.getBraintree().merchantId(),
-        config.getBraintree().publicKey(), config.getBraintree().privateKey().value(), config.getBraintree().environment(),
+        config.getBraintree().publicKey(), config.getBraintree().privateKey().value(),
+        config.getBraintree().environment(),
        config.getBraintree().supportedCurrencies(), config.getBraintree().merchantAccounts(),
-        config.getBraintree().graphqlUrl(), config.getBraintree().circuitBreaker(), subscriptionProcessorExecutor);
+        config.getBraintree().graphqlUrl(), config.getBraintree().circuitBreaker(), subscriptionProcessorExecutor,
+        subscriptionProcessorRetryExecutor);

    ExternalServiceCredentialsGenerator directoryV2CredentialsGenerator = DirectoryV2Controller.credentialsGenerator(
        config.getDirectoryV2Configuration().getDirectoryV2ClientConfiguration());
@@ -461,11 +469,12 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
        config.getRegistrationServiceConfiguration().registrationCaCertificate(),
        registrationCallbackExecutor);
    SecureBackupClient secureBackupClient = new SecureBackupClient(backupCredentialsGenerator,
-        secureValueRecoveryServiceExecutor, config.getSecureBackupServiceConfiguration());
+        secureValueRecoveryServiceExecutor, secureValueRecoveryServiceRetryExecutor,
+        config.getSecureBackupServiceConfiguration());
    SecureValueRecovery2Client secureValueRecovery2Client = new SecureValueRecovery2Client(svr2CredentialsGenerator,
-        secureValueRecoveryServiceExecutor, config.getSvr2Configuration());
+        secureValueRecoveryServiceExecutor, secureValueRecoveryServiceRetryExecutor, config.getSvr2Configuration());
    SecureStorageClient secureStorageClient = new SecureStorageClient(storageCredentialsGenerator,
-        storageServiceExecutor, config.getSecureStorageServiceConfiguration());
+        storageServiceExecutor, storageServiceRetryExecutor, config.getSecureStorageServiceConfiguration());
    ClientPresenceManager clientPresenceManager = new ClientPresenceManager(clientPresenceCluster, recurringJobExecutor,
        keyspaceNotificationDispatchExecutor);
    StoredVerificationCodeManager pendingAccountsManager = new StoredVerificationCodeManager(pendingAccounts);
--- a/service/src/main/java/org/whispersystems/textsecuregcm/http/FaultTolerantHttpClient.java
+++ b/service/src/main/java/org/whispersystems/textsecuregcm/http/FaultTolerantHttpClient.java
@@ -17,7 +17,6 @@ import java.time.Duration;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CompletionStage;
 import java.util.concurrent.Executor;
-import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.function.Supplier;
 import org.glassfish.jersey.SslConfigurator;
@@ -28,10 +27,10 @@ import org.whispersystems.textsecuregcm.util.CircuitBreakerUtil;

 public class FaultTolerantHttpClient {

-  private final HttpClient               httpClient;
+  private final HttpClient httpClient;
  private final ScheduledExecutorService retryExecutor;
-  private final Retry                    retry;
-  private final CircuitBreaker           breaker;
+  private final Retry retry;
+  private final CircuitBreaker breaker;

  public static final String SECURITY_PROTOCOL_TLS_1_2 = "TLSv1.2";
  public static final String SECURITY_PROTOCOL_TLS_1_3 = "TLSv1.3";
@@ -40,15 +39,20 @@ public class FaultTolerantHttpClient {
    return new Builder();
  }

-  private FaultTolerantHttpClient(String name, HttpClient httpClient, RetryConfiguration retryConfiguration, CircuitBreakerConfiguration circuitBreakerConfiguration) {
-    this.httpClient    = httpClient;
-    this.retryExecutor = Executors.newSingleThreadScheduledExecutor();
-    this.breaker       = CircuitBreaker.of(name + "-breaker", circuitBreakerConfiguration.toCircuitBreakerConfig());
+  private FaultTolerantHttpClient(String name, HttpClient httpClient, ScheduledExecutorService retryExecutor,
+      RetryConfiguration retryConfiguration, CircuitBreakerConfiguration circuitBreakerConfiguration) {
+    this.httpClient = httpClient;
+    this.retryExecutor = retryExecutor;
+    this.breaker = CircuitBreaker.of(name + "-breaker", circuitBreakerConfiguration.toCircuitBreakerConfig());

    CircuitBreakerUtil.registerMetrics(breaker, FaultTolerantHttpClient.class);

    if (retryConfiguration != null) {
-      RetryConfig retryConfig = retryConfiguration.<HttpResponse>toRetryConfigBuilder().retryOnResult(o -> o.statusCode() >= 500).build();
+      if (this.retryExecutor == null) {
+        throw new IllegalArgumentException("retryExecutor must be specified with retryConfiguration");
+      }
+      RetryConfig retryConfig = retryConfiguration.<HttpResponse>toRetryConfigBuilder()
+          .retryOnResult(o -> o.statusCode() >= 500).build();
      this.retry = Retry.of(name + "-retry", retryConfig);
      CircuitBreakerUtil.registerMetrics(retry, FaultTolerantHttpClient.class);
    } else {
@@ -76,19 +80,20 @@ public class FaultTolerantHttpClient {

  public static class Builder {

+    private HttpClient.Version version = HttpClient.Version.HTTP_2;
+    private HttpClient.Redirect redirect = HttpClient.Redirect.NEVER;
+    private Duration connectTimeout = Duration.ofSeconds(10);

-    private HttpClient.Version  version       = HttpClient.Version.HTTP_2;
-    private HttpClient.Redirect redirect      = HttpClient.Redirect.NEVER;
-    private Duration            connectTimeout = Duration.ofSeconds(10);
-
-    private String                      name;
-    private Executor                    executor;
-    private KeyStore                    trustStore;
-    private String                      securityProtocol = SECURITY_PROTOCOL_TLS_1_2;
-    private RetryConfiguration          retryConfiguration;
+    private String name;
+    private Executor executor;
+    private ScheduledExecutorService retryExecutor;
+    private KeyStore trustStore;
+    private String securityProtocol = SECURITY_PROTOCOL_TLS_1_2;
+    private RetryConfiguration retryConfiguration;
    private CircuitBreakerConfiguration circuitBreakerConfiguration;

-    private Builder() {}
+    private Builder() {
+    }

    public Builder withName(String name) {
      this.name = name;
@@ -120,6 +125,11 @@ public class FaultTolerantHttpClient {
      return this;
    }

+    public Builder withRetryExecutor(ScheduledExecutorService retryExecutor) {
+      this.retryExecutor = retryExecutor;
+      return this;
+    }
+
    public Builder withCircuitBreaker(CircuitBreakerConfiguration circuitBreakerConfiguration) {
      this.circuitBreakerConfiguration = circuitBreakerConfiguration;
      return this;
@@ -141,10 +151,10 @@ public class FaultTolerantHttpClient {
      }

      final HttpClient.Builder builder = HttpClient.newBuilder()
-                                                   .connectTimeout(connectTimeout)
-                                                   .followRedirects(redirect)
-                                                   .version(version)
-                                                   .executor(executor);
+          .connectTimeout(connectTimeout)
+          .followRedirects(redirect)
+          .version(version)
+          .executor(executor);

      final SslConfigurator sslConfigurator = SslConfigurator.newInstance().securityProtocol(securityProtocol);

@@ -154,9 +164,9 @@ public class FaultTolerantHttpClient {

      builder.sslContext(sslConfigurator.createSSLContext());

-      return new FaultTolerantHttpClient(name, builder.build(), retryConfiguration, circuitBreakerConfiguration);
+      return new FaultTolerantHttpClient(name, builder.build(), retryExecutor, retryConfiguration,
+          circuitBreakerConfiguration);
    }
-
  }

 }
--- a/service/src/main/java/org/whispersystems/textsecuregcm/securebackup/SecureBackupClient.java
+++ b/service/src/main/java/org/whispersystems/textsecuregcm/securebackup/SecureBackupClient.java
@@ -18,6 +18,7 @@ import java.time.Duration;
 import java.util.UUID;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.Executor;
+import java.util.concurrent.ScheduledExecutorService;
 import org.whispersystems.textsecuregcm.auth.ExternalServiceCredentials;
 import org.whispersystems.textsecuregcm.auth.ExternalServiceCredentialsGenerator;
 import org.whispersystems.textsecuregcm.configuration.SecureBackupServiceConfiguration;
@@ -29,37 +30,41 @@ import org.whispersystems.textsecuregcm.util.HttpUtils;
 */
 public class SecureBackupClient {

-    private final ExternalServiceCredentialsGenerator secureBackupCredentialsGenerator;
-    private final URI                                deleteUri;
-    private final FaultTolerantHttpClient            httpClient;
+  private final ExternalServiceCredentialsGenerator secureBackupCredentialsGenerator;
+  private final URI deleteUri;
+  private final FaultTolerantHttpClient httpClient;

-    @VisibleForTesting
-    static final String DELETE_PATH = "/v1/backup";
+  @VisibleForTesting
+  static final String DELETE_PATH = "/v1/backup";

-    public SecureBackupClient(final ExternalServiceCredentialsGenerator secureBackupCredentialsGenerator, final Executor executor, final SecureBackupServiceConfiguration configuration) throws CertificateException {
-        this.secureBackupCredentialsGenerator   = secureBackupCredentialsGenerator;
-        this.deleteUri                         = URI.create(configuration.getUri()).resolve(DELETE_PATH);
-        this.httpClient                        = FaultTolerantHttpClient.newBuilder()
-                                                                        .withCircuitBreaker(configuration.getCircuitBreakerConfiguration())
-                                                                        .withRetry(configuration.getRetryConfiguration())
-                                                                        .withVersion(HttpClient.Version.HTTP_1_1)
-                                                                        .withConnectTimeout(Duration.ofSeconds(10))
-                                                                        .withRedirect(HttpClient.Redirect.NEVER)
-                                                                        .withExecutor(executor)
-                                                                        .withName("secure-backup")
-                                                                        .withSecurityProtocol(FaultTolerantHttpClient.SECURITY_PROTOCOL_TLS_1_2)
-                                                                        .withTrustedServerCertificates(configuration.getBackupCaCertificates().toArray(new String[0]))
-                                                                        .build();
-    }
+  public SecureBackupClient(final ExternalServiceCredentialsGenerator secureBackupCredentialsGenerator,
+      final Executor executor, final
+  ScheduledExecutorService retryExecutor, final SecureBackupServiceConfiguration configuration)
+      throws CertificateException {
+    this.secureBackupCredentialsGenerator = secureBackupCredentialsGenerator;
+    this.deleteUri = URI.create(configuration.getUri()).resolve(DELETE_PATH);
+    this.httpClient = FaultTolerantHttpClient.newBuilder()
+        .withCircuitBreaker(configuration.getCircuitBreakerConfiguration())
+        .withRetry(configuration.getRetryConfiguration())
+        .withRetryExecutor(retryExecutor)
+        .withVersion(HttpClient.Version.HTTP_1_1)
+        .withConnectTimeout(Duration.ofSeconds(10))
+        .withRedirect(HttpClient.Redirect.NEVER)
+        .withExecutor(executor)
+        .withName("secure-backup")
+        .withSecurityProtocol(FaultTolerantHttpClient.SECURITY_PROTOCOL_TLS_1_2)
+        .withTrustedServerCertificates(configuration.getBackupCaCertificates().toArray(new String[0]))
+        .build();
+  }

-    public CompletableFuture<Void> deleteBackups(final UUID accountUuid) {
-        final ExternalServiceCredentials credentials = secureBackupCredentialsGenerator.generateForUuid(accountUuid);
+  public CompletableFuture<Void> deleteBackups(final UUID accountUuid) {
+    final ExternalServiceCredentials credentials = secureBackupCredentialsGenerator.generateForUuid(accountUuid);

-        final HttpRequest request = HttpRequest.newBuilder()
-                .uri(deleteUri)
-                .DELETE()
-                .header(HttpHeaders.AUTHORIZATION, basicAuthHeader(credentials))
-                .build();
+    final HttpRequest request = HttpRequest.newBuilder()
+        .uri(deleteUri)
+        .DELETE()
+        .header(HttpHeaders.AUTHORIZATION, basicAuthHeader(credentials))
+        .build();

        return httpClient.sendAsync(request, HttpResponse.BodyHandlers.ofString()).thenApply(response -> {
            if (HttpUtils.isSuccessfulResponse(response.statusCode())) {
--- a/service/src/main/java/org/whispersystems/textsecuregcm/securestorage/SecureStorageClient.java
+++ b/service/src/main/java/org/whispersystems/textsecuregcm/securestorage/SecureStorageClient.java
@@ -18,6 +18,7 @@ import java.time.Duration;
 import java.util.UUID;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.Executor;
+import java.util.concurrent.ScheduledExecutorService;
 import org.whispersystems.textsecuregcm.auth.ExternalServiceCredentials;
 import org.whispersystems.textsecuregcm.auth.ExternalServiceCredentialsGenerator;
 import org.whispersystems.textsecuregcm.configuration.SecureStorageServiceConfiguration;
@@ -29,37 +30,41 @@ import org.whispersystems.textsecuregcm.util.HttpUtils;
 */
 public class SecureStorageClient {

-    private final ExternalServiceCredentialsGenerator storageServiceCredentialsGenerator;
-    private final URI                                deleteUri;
-    private final FaultTolerantHttpClient            httpClient;
+  private final ExternalServiceCredentialsGenerator storageServiceCredentialsGenerator;
+  private final URI deleteUri;
+  private final FaultTolerantHttpClient httpClient;

-    @VisibleForTesting
-    static final String DELETE_PATH = "/v1/storage";
+  @VisibleForTesting
+  static final String DELETE_PATH = "/v1/storage";

-    public SecureStorageClient(final ExternalServiceCredentialsGenerator storageServiceCredentialsGenerator, final Executor executor, final SecureStorageServiceConfiguration configuration) throws CertificateException {
-        this.storageServiceCredentialsGenerator = storageServiceCredentialsGenerator;
-        this.deleteUri                         = URI.create(configuration.uri()).resolve(DELETE_PATH);
-        this.httpClient                        = FaultTolerantHttpClient.newBuilder()
-                                                                        .withCircuitBreaker(configuration.circuitBreaker())
-                                                                        .withRetry(configuration.retry())
-                                                                        .withVersion(HttpClient.Version.HTTP_1_1)
-                                                                        .withConnectTimeout(Duration.ofSeconds(10))
-                                                                        .withRedirect(HttpClient.Redirect.NEVER)
-                                                                        .withExecutor(executor)
-                                                                        .withName("secure-storage")
-                                                                        .withSecurityProtocol(FaultTolerantHttpClient.SECURITY_PROTOCOL_TLS_1_3)
-                                                                        .withTrustedServerCertificates(configuration.storageCaCertificates().toArray(new String[0]))
-                                                                        .build();
-    }
+  public SecureStorageClient(final ExternalServiceCredentialsGenerator storageServiceCredentialsGenerator,
+      final Executor executor, final
+  ScheduledExecutorService retryExecutor, final SecureStorageServiceConfiguration configuration)
+      throws CertificateException {
+    this.storageServiceCredentialsGenerator = storageServiceCredentialsGenerator;
+    this.deleteUri = URI.create(configuration.uri()).resolve(DELETE_PATH);
+    this.httpClient = FaultTolerantHttpClient.newBuilder()
+        .withCircuitBreaker(configuration.circuitBreaker())
+        .withRetry(configuration.retry())
+        .withRetryExecutor(retryExecutor)
+        .withVersion(HttpClient.Version.HTTP_1_1)
+        .withConnectTimeout(Duration.ofSeconds(10))
+        .withRedirect(HttpClient.Redirect.NEVER)
+        .withExecutor(executor)
+        .withName("secure-storage")
+        .withSecurityProtocol(FaultTolerantHttpClient.SECURITY_PROTOCOL_TLS_1_3)
+        .withTrustedServerCertificates(configuration.storageCaCertificates().toArray(new String[0]))
+        .build();
+  }

-    public CompletableFuture<Void> deleteStoredData(final UUID accountUuid) {
-        final ExternalServiceCredentials credentials = storageServiceCredentialsGenerator.generateForUuid(accountUuid);
+  public CompletableFuture<Void> deleteStoredData(final UUID accountUuid) {
+    final ExternalServiceCredentials credentials = storageServiceCredentialsGenerator.generateForUuid(accountUuid);

-        final HttpRequest request = HttpRequest.newBuilder()
-                .uri(deleteUri)
-                .DELETE()
-                .header(HttpHeaders.AUTHORIZATION, basicAuthHeader(credentials))
-                .build();
+    final HttpRequest request = HttpRequest.newBuilder()
+        .uri(deleteUri)
+        .DELETE()
+        .header(HttpHeaders.AUTHORIZATION, basicAuthHeader(credentials))
+        .build();

        return httpClient.sendAsync(request, HttpResponse.BodyHandlers.ofString()).thenApply(response -> {
            if (HttpUtils.isSuccessfulResponse(response.statusCode())) {
--- a/service/src/main/java/org/whispersystems/textsecuregcm/securevaluerecovery/SecureValueRecovery2Client.java
+++ b/service/src/main/java/org/whispersystems/textsecuregcm/securevaluerecovery/SecureValueRecovery2Client.java
@@ -18,6 +18,7 @@ import java.time.Duration;
 import java.util.UUID;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.Executor;
+import java.util.concurrent.ScheduledExecutorService;
 import org.whispersystems.textsecuregcm.auth.ExternalServiceCredentials;
 import org.whispersystems.textsecuregcm.auth.ExternalServiceCredentialsGenerator;
 import org.whispersystems.textsecuregcm.configuration.SecureValueRecovery2Configuration;
@@ -37,13 +38,15 @@ public class SecureValueRecovery2Client {
  static final String DELETE_PATH = "/v1/delete";

  public SecureValueRecovery2Client(final ExternalServiceCredentialsGenerator secureValueRecoveryCredentialsGenerator,
-      final Executor executor, final SecureValueRecovery2Configuration configuration)
+      final Executor executor, final ScheduledExecutorService retryExecutor,
+      final SecureValueRecovery2Configuration configuration)
      throws CertificateException {
    this.secureValueRecoveryCredentialsGenerator = secureValueRecoveryCredentialsGenerator;
    this.deleteUri = URI.create(configuration.uri()).resolve(DELETE_PATH);
    this.httpClient = FaultTolerantHttpClient.newBuilder()
        .withCircuitBreaker(configuration.circuitBreaker())
        .withRetry(configuration.retry())
+        .withRetryExecutor(retryExecutor)
        .withVersion(HttpClient.Version.HTTP_1_1)
        .withConnectTimeout(Duration.ofSeconds(10))
        .withRedirect(HttpClient.Redirect.NEVER)
--- a/service/src/main/java/org/whispersystems/textsecuregcm/subscriptions/BraintreeManager.java
+++ b/service/src/main/java/org/whispersystems/textsecuregcm/subscriptions/BraintreeManager.java
@@ -32,6 +32,7 @@ import java.util.Set;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CompletionException;
 import java.util.concurrent.Executor;
+import java.util.concurrent.ScheduledExecutorService;
 import javax.annotation.Nullable;
 import javax.ws.rs.ClientErrorException;
 import javax.ws.rs.WebApplicationException;
@@ -61,7 +62,8 @@ public class BraintreeManager implements SubscriptionProcessorManager {
      final Map<String, String> currenciesToMerchantAccounts,
      final String graphqlUri,
      final CircuitBreakerConfiguration circuitBreakerConfiguration,
-      final Executor executor) {
+      final Executor executor,
+      final ScheduledExecutorService retryExecutor) {

    this(new BraintreeGateway(braintreeEnvironment, braintreeMerchantId, braintreePublicKey,
            braintreePrivateKey),
@@ -71,6 +73,7 @@ public class BraintreeManager implements SubscriptionProcessorManager {
            .withName("braintree-graphql")
            .withCircuitBreaker(circuitBreakerConfiguration)
            .withExecutor(executor)
+            .withRetryExecutor(retryExecutor)
            .build(), graphqlUri, braintreePublicKey, braintreePrivateKey),
        executor);
  }
--- a/service/src/main/java/org/whispersystems/textsecuregcm/workers/AssignUsernameCommand.java
+++ b/service/src/main/java/org/whispersystems/textsecuregcm/workers/AssignUsernameCommand.java
@@ -18,6 +18,7 @@ import java.util.List;
 import java.util.UUID;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
 import net.sourceforge.argparse4j.inf.Namespace;
 import net.sourceforge.argparse4j.inf.Subparser;
 import org.whispersystems.textsecuregcm.WhisperServerConfiguration;
@@ -110,6 +111,10 @@ public class AssignUsernameCommand extends EnvironmentCommand<WhisperServerConfi
        .executorService(name(getClass(), "secureValueRecoveryService-%d")).maxThreads(1).minThreads(1).build();
    ExecutorService storageServiceExecutor = environment.lifecycle()
        .executorService(name(getClass(), "storageService-%d")).maxThreads(1).minThreads(1).build();
+    ScheduledExecutorService secureValueRecoveryServiceRetryExecutor = environment.lifecycle()
+        .scheduledExecutorService(name(getClass(), "secureValueRecoveryServiceRetry-%d")).threads(1).build();
+    ScheduledExecutorService storageServiceRetryExecutor = environment.lifecycle()
+        .scheduledExecutorService(name(getClass(), "storageServiceRetry-%d")).threads(1).build();

    ExternalServiceCredentialsGenerator backupCredentialsGenerator = SecureBackupController.credentialsGenerator(
        configuration.getSecureBackupServiceConfiguration());
@@ -174,12 +179,14 @@ public class AssignUsernameCommand extends EnvironmentCommand<WhisperServerConfi
        configuration.getClientPresenceClusterConfiguration(), redisClusterClientResources);
    FaultTolerantRedisCluster rateLimitersCluster = new FaultTolerantRedisCluster("rate_limiters",
        configuration.getRateLimitersCluster(), redisClusterClientResources);
-    SecureBackupClient secureBackupClient = new SecureBackupClient(backupCredentialsGenerator, secureValueRecoveryExecutor,
-        configuration.getSecureBackupServiceConfiguration());
+    SecureBackupClient secureBackupClient = new SecureBackupClient(backupCredentialsGenerator,
+        secureValueRecoveryExecutor,
+        secureValueRecoveryServiceRetryExecutor, configuration.getSecureBackupServiceConfiguration());
    SecureValueRecovery2Client secureValueRecovery2Client = new SecureValueRecovery2Client(
-        secureValueRecoveryCredentialsGenerator, secureValueRecoveryExecutor, configuration.getSvr2Configuration());
+        secureValueRecoveryCredentialsGenerator, secureValueRecoveryExecutor, secureValueRecoveryServiceRetryExecutor,
+        configuration.getSvr2Configuration());
    SecureStorageClient secureStorageClient = new SecureStorageClient(storageCredentialsGenerator,
-        storageServiceExecutor, configuration.getSecureStorageServiceConfiguration());
+        storageServiceExecutor, storageServiceRetryExecutor, configuration.getSecureStorageServiceConfiguration());
    ClientPresenceManager clientPresenceManager = new ClientPresenceManager(clientPresenceCluster,
        Executors.newSingleThreadScheduledExecutor(), keyspaceNotificationDispatchExecutor);
    MessagesCache messagesCache = new MessagesCache(messageInsertCacheCluster, messageReadDeleteCluster,
--- a/service/src/main/java/org/whispersystems/textsecuregcm/workers/CommandDependencies.java
+++ b/service/src/main/java/org/whispersystems/textsecuregcm/workers/CommandDependencies.java
@@ -94,6 +94,11 @@ record CommandDependencies(
    ExecutorService storageServiceExecutor = environment.lifecycle()
        .executorService(name(name, "storageService-%d")).maxThreads(8).minThreads(8).build();

+    ScheduledExecutorService secureValueRecoveryServiceRetryExecutor = environment.lifecycle()
+        .scheduledExecutorService(name(name, "secureValueRecoveryServiceRetry-%d")).threads(1).build();
+    ScheduledExecutorService storageServiceRetryExecutor = environment.lifecycle()
+        .scheduledExecutorService(name(name, "storageServiceRetry-%d")).threads(1).build();
+
    ExternalServiceCredentialsGenerator backupCredentialsGenerator = SecureBackupController.credentialsGenerator(
        configuration.getSecureBackupServiceConfiguration());
    ExternalServiceCredentialsGenerator storageCredentialsGenerator = SecureStorageController.credentialsGenerator(
@@ -159,13 +164,14 @@ record CommandDependencies(
    FaultTolerantRedisCluster rateLimitersCluster = new FaultTolerantRedisCluster("rate_limiters",
        configuration.getRateLimitersCluster(), redisClusterClientResources);
    SecureBackupClient secureBackupClient = new SecureBackupClient(backupCredentialsGenerator,
-        secureValueRecoveryServiceExecutor,
+        secureValueRecoveryServiceExecutor, secureValueRecoveryServiceRetryExecutor,
        configuration.getSecureBackupServiceConfiguration());
    SecureValueRecovery2Client secureValueRecovery2Client = new SecureValueRecovery2Client(
        secureValueRecoveryCredentialsGenerator, secureValueRecoveryServiceExecutor,
+        secureValueRecoveryServiceRetryExecutor,
        configuration.getSvr2Configuration());
    SecureStorageClient secureStorageClient = new SecureStorageClient(storageCredentialsGenerator,
-        storageServiceExecutor, configuration.getSecureStorageServiceConfiguration());
+        storageServiceExecutor, storageServiceRetryExecutor, configuration.getSecureStorageServiceConfiguration());
    ClientPresenceManager clientPresenceManager = new ClientPresenceManager(clientPresenceCluster,
        recurringJobExecutor, keyspaceNotificationDispatchExecutor);
    MessagesCache messagesCache = new MessagesCache(messageInsertCacheCluster, messageReadDeleteCluster,