FaultTolerantHttpClient: used managed ScheduledExecutorService for retries

2026-04-22 03:18:04 +01:00 · 2023-06-23 10:48:38 -05:00
parent 8e48ac4ede
commit b852d6681d
13 changed files with 346 additions and 252 deletions
--- a/service/src/test/java/org/whispersystems/textsecuregcm/securebackup/SecureBackupClientTest.java
+++ b/service/src/test/java/org/whispersystems/textsecuregcm/securebackup/SecureBackupClientTest.java
@@ -21,6 +21,7 @@ import java.util.UUID;
 import java.util.concurrent.CompletionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
 import org.apache.commons.lang3.RandomStringUtils;
 import org.junit.jupiter.api.AfterEach;
@@ -36,6 +37,7 @@ class SecureBackupClientTest {
  private UUID accountUuid;
  private ExternalServiceCredentialsGenerator credentialsGenerator;
  private ExecutorService httpExecutor;
+  private ScheduledExecutorService retryExecutor;

  private SecureBackupClient secureStorageClient;

@@ -46,68 +48,71 @@ class SecureBackupClientTest {

  @BeforeEach
  void setUp() throws CertificateException {
-    accountUuid         = UUID.randomUUID();
+    accountUuid = UUID.randomUUID();
    credentialsGenerator = mock(ExternalServiceCredentialsGenerator.class);
-    httpExecutor        = Executors.newSingleThreadExecutor();
+    httpExecutor = Executors.newSingleThreadExecutor();
+    retryExecutor = Executors.newSingleThreadScheduledExecutor();

    final SecureBackupServiceConfiguration config = new SecureBackupServiceConfiguration();
    config.setUri("http://localhost:" + wireMock.getPort());

    // This is a randomly-generated, throwaway certificate that's not actually connected to anything
-    config.setBackupCaCertificates(List.of("""
-        -----BEGIN CERTIFICATE-----
-        MIICZDCCAc2gAwIBAgIBADANBgkqhkiG9w0BAQ0FADBPMQswCQYDVQQGEwJ1czEL
-        MAkGA1UECAwCVVMxHjAcBgNVBAoMFVNpZ25hbCBNZXNzZW5nZXIsIExMQzETMBEG
-        A1UEAwwKc2lnbmFsLm9yZzAeFw0yMDEyMjMyMjQ3NTlaFw0zMDEyMjEyMjQ3NTla
-        ME8xCzAJBgNVBAYTAnVzMQswCQYDVQQIDAJVUzEeMBwGA1UECgwVU2lnbmFsIE1l
-        c3NlbmdlciwgTExDMRMwEQYDVQQDDApzaWduYWwub3JnMIGfMA0GCSqGSIb3DQEB
-        AQUAA4GNADCBiQKBgQCfSLcZNHYqbxSsgWp4JvbPRHjQTrlsrKrgD2q7f/OY6O3Y
-        /X0QNcNSOJpliN8rmzwslfsrXHO3q1diGRw4xHogUJZ/7NQrHiP/zhN0VTDh49pD
-        ZpjXVyUbayLS/6qM5arKxBspzEFBb5v8cF6bPr76SO/rpGXiI0j6yJKX6fRiKwID
-        AQABo1AwTjAdBgNVHQ4EFgQU6Jrs/Fmj0z4dA3wvdq/WqA4P49IwHwYDVR0jBBgw
-        FoAU6Jrs/Fmj0z4dA3wvdq/WqA4P49IwDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0B
-        AQ0FAAOBgQB+5d5+NtzLILfrc9QmJdIO1YeDP64JmFwTER0kEUouRsb9UwknVWZa
-        y7MTM4NoBV1k0zb5LAk89SIDPr/maW5AsLtEomzjnEiomjoMBUdNe3YCgQReoLnr
-        R/QaUNbrCjTGYfBsjGbIzmkWPUyTec2ZdRyJ8JiVl386+6CZkxnndQ==
-        -----END CERTIFICATE-----
-        """,
-        """
-        -----BEGIN CERTIFICATE-----
-        MIIEpDCCAowCCQC43PUTWSADVjANBgkqhkiG9w0BAQsFADAUMRIwEAYDVQQDDAls
-        b2NhbGhvc3QwHhcNMjIxMDE3MjA0NTM0WhcNMjMxMDE3MjA0NTM0WjAUMRIwEAYD
-        VQQDDAlsb2NhbGhvc3QwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDV
-        x1cdEd2ffQTlTXWRiCHGcrlYf4RJnctt9sw/BuHWTLXBu5LhyJSGn5LRszO/NCXK
-        Z/cmGR7pLj366RtiwL+Qo3nhvDCK7T9xZeNIusM6XMcMK9D/DGCYPqtjQz8NXd9V
-        ajBBe6nwTDTa+oqX8Mt89foWNkg5Il/lY62u9Dr18LRZ2W9zzYi3Q9/K0CbIX6pM
-        yVlPIO5rITOR2IsbeyqsO9jufgX5lP4ZKLLBAP1b7usjC4YdvWacjQg/rK5aay1x
-        jC2HCDgo/4N30QVXzSA9nFfSe6AE/xkStK4819JqOkY5JsJCbef1P3hOOdSLEjbp
-        xq3MjOs6G6dOgteaAGs10vx7dHxDWETTIiD7BIZ9zRYgOF5bkCaIUO+JfySE1MHD
-        KBAFLoRuvmRev5Ln5R0MCHpUMSmMNgJqz+RWZV3g/gpYbuWiHgJOwL1393eK50Bg
-        W7SXQ8EjJj2yXZSH+1gPzN0DRoJZiaBoTPnCL2qUgvwFpW1PJsM5FDyUJFUoK5kK
-        HLBBSKAPt6ZlSrUe2nBgJv7EF1GK+fTU08LXgW33OpLceGPa0zTShkukQUMtUtZ8
-        GqhO12ohMzEupIu5Xurthq4VVUrzHUdj1ZZRMhAbfLU36sd03MMyL/xBqTN6dzCa
-        GDGIPGpYjAllZ5xMRt2kZdv+Kr6oo3u2nLUIsqI7KQIDAQABMA0GCSqGSIb3DQEB
-        CwUAA4ICAQCB5s43YF35ssf5YONW5iAaifGpi1o0866xfeOybtohFGvQ7V2W34i9
-        TYBCt8+0hgatMcvZ08f0vqig1i7nrvYcE1hnhL7JNkU8qm0s9ytHZt6j62nB0kd/
-        uqE2hOEQalTf/2TGPV0CCgiqLyd8lEUQvQeA38wktwUeZpVnErlzHeMR2CvV3K8R
-        u4vV6SnBcf+TAt56RKYZkPyvZj5llQPo14Glyoo8qZES7Ky1SHmM0GL+baPRBjRW
-        3KgSt98Wyu4yr9qu21JpnbAnLhBfzfSKjSeCRgFElUE1GIaFGRZ7ypA74dUKeLnb
-        /VUWrszmUhGaEjV9dpI6x6B/kSpQMtIQqBaKRY2ALUeEujS/rURi4iMDwSU+GkSH
-        cyEvZKS97OA/dWeXfLXdo4beDBRG93bI4rQnDg5+VdlBOkQSLueb8x6/VThMoC5d
-        vZiotFQHseljQAdTkNa6tBu6c4XDYPCKB3CfkMYOlCfTS7Acn5G6dxTPKBtLGBnL
-        nQfYyzuwYkN09+2PVzt6auBHr3To7uoclkxX+hxyvPIwIZ0N6b4tQR1FCAkvg29Q
-        WIOjZOKGW690ESKCKOnFjUHVO0HpuWnT81URTuY62FXsYdVc2wE4v0E04mEbqQ0P
-        lY6ZKNA81Lm3YADYtObmK1IUrOPo9BeIaPy0UM08SmN880Vunqa91Q==
-        -----END CERTIFICATE-----
-        """));
+    config.setBackupCaCertificates(
+        List.of("""
+            -----BEGIN CERTIFICATE-----
+            MIICZDCCAc2gAwIBAgIBADANBgkqhkiG9w0BAQ0FADBPMQswCQYDVQQGEwJ1czEL
+            MAkGA1UECAwCVVMxHjAcBgNVBAoMFVNpZ25hbCBNZXNzZW5nZXIsIExMQzETMBEG
+            A1UEAwwKc2lnbmFsLm9yZzAeFw0yMDEyMjMyMjQ3NTlaFw0zMDEyMjEyMjQ3NTla
+            ME8xCzAJBgNVBAYTAnVzMQswCQYDVQQIDAJVUzEeMBwGA1UECgwVU2lnbmFsIE1l
+            c3NlbmdlciwgTExDMRMwEQYDVQQDDApzaWduYWwub3JnMIGfMA0GCSqGSIb3DQEB
+            AQUAA4GNADCBiQKBgQCfSLcZNHYqbxSsgWp4JvbPRHjQTrlsrKrgD2q7f/OY6O3Y
+            /X0QNcNSOJpliN8rmzwslfsrXHO3q1diGRw4xHogUJZ/7NQrHiP/zhN0VTDh49pD
+            ZpjXVyUbayLS/6qM5arKxBspzEFBb5v8cF6bPr76SO/rpGXiI0j6yJKX6fRiKwID
+            AQABo1AwTjAdBgNVHQ4EFgQU6Jrs/Fmj0z4dA3wvdq/WqA4P49IwHwYDVR0jBBgw
+            FoAU6Jrs/Fmj0z4dA3wvdq/WqA4P49IwDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0B
+            AQ0FAAOBgQB+5d5+NtzLILfrc9QmJdIO1YeDP64JmFwTER0kEUouRsb9UwknVWZa
+            y7MTM4NoBV1k0zb5LAk89SIDPr/maW5AsLtEomzjnEiomjoMBUdNe3YCgQReoLnr
+            R/QaUNbrCjTGYfBsjGbIzmkWPUyTec2ZdRyJ8JiVl386+6CZkxnndQ==
+            -----END CERTIFICATE-----
+            """, """
+            -----BEGIN CERTIFICATE-----
+            MIIEpDCCAowCCQC43PUTWSADVjANBgkqhkiG9w0BAQsFADAUMRIwEAYDVQQDDAls
+            b2NhbGhvc3QwHhcNMjIxMDE3MjA0NTM0WhcNMjMxMDE3MjA0NTM0WjAUMRIwEAYD
+            VQQDDAlsb2NhbGhvc3QwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDV
+            x1cdEd2ffQTlTXWRiCHGcrlYf4RJnctt9sw/BuHWTLXBu5LhyJSGn5LRszO/NCXK
+            Z/cmGR7pLj366RtiwL+Qo3nhvDCK7T9xZeNIusM6XMcMK9D/DGCYPqtjQz8NXd9V
+            ajBBe6nwTDTa+oqX8Mt89foWNkg5Il/lY62u9Dr18LRZ2W9zzYi3Q9/K0CbIX6pM
+            yVlPIO5rITOR2IsbeyqsO9jufgX5lP4ZKLLBAP1b7usjC4YdvWacjQg/rK5aay1x
+            jC2HCDgo/4N30QVXzSA9nFfSe6AE/xkStK4819JqOkY5JsJCbef1P3hOOdSLEjbp
+            xq3MjOs6G6dOgteaAGs10vx7dHxDWETTIiD7BIZ9zRYgOF5bkCaIUO+JfySE1MHD
+            KBAFLoRuvmRev5Ln5R0MCHpUMSmMNgJqz+RWZV3g/gpYbuWiHgJOwL1393eK50Bg
+            W7SXQ8EjJj2yXZSH+1gPzN0DRoJZiaBoTPnCL2qUgvwFpW1PJsM5FDyUJFUoK5kK
+            HLBBSKAPt6ZlSrUe2nBgJv7EF1GK+fTU08LXgW33OpLceGPa0zTShkukQUMtUtZ8
+            GqhO12ohMzEupIu5Xurthq4VVUrzHUdj1ZZRMhAbfLU36sd03MMyL/xBqTN6dzCa
+            GDGIPGpYjAllZ5xMRt2kZdv+Kr6oo3u2nLUIsqI7KQIDAQABMA0GCSqGSIb3DQEB
+            CwUAA4ICAQCB5s43YF35ssf5YONW5iAaifGpi1o0866xfeOybtohFGvQ7V2W34i9
+            TYBCt8+0hgatMcvZ08f0vqig1i7nrvYcE1hnhL7JNkU8qm0s9ytHZt6j62nB0kd/
+            uqE2hOEQalTf/2TGPV0CCgiqLyd8lEUQvQeA38wktwUeZpVnErlzHeMR2CvV3K8R
+            u4vV6SnBcf+TAt56RKYZkPyvZj5llQPo14Glyoo8qZES7Ky1SHmM0GL+baPRBjRW
+            3KgSt98Wyu4yr9qu21JpnbAnLhBfzfSKjSeCRgFElUE1GIaFGRZ7ypA74dUKeLnb
+            /VUWrszmUhGaEjV9dpI6x6B/kSpQMtIQqBaKRY2ALUeEujS/rURi4iMDwSU+GkSH
+            cyEvZKS97OA/dWeXfLXdo4beDBRG93bI4rQnDg5+VdlBOkQSLueb8x6/VThMoC5d
+            vZiotFQHseljQAdTkNa6tBu6c4XDYPCKB3CfkMYOlCfTS7Acn5G6dxTPKBtLGBnL
+            nQfYyzuwYkN09+2PVzt6auBHr3To7uoclkxX+hxyvPIwIZ0N6b4tQR1FCAkvg29Q
+            WIOjZOKGW690ESKCKOnFjUHVO0HpuWnT81URTuY62FXsYdVc2wE4v0E04mEbqQ0P
+            lY6ZKNA81Lm3YADYtObmK1IUrOPo9BeIaPy0UM08SmN880Vunqa91Q==
+            -----END CERTIFICATE-----
+            """));

-    secureStorageClient = new SecureBackupClient(credentialsGenerator, httpExecutor, config);
+    secureStorageClient = new SecureBackupClient(credentialsGenerator, httpExecutor, retryExecutor, config);
  }

  @AfterEach
  void tearDown() throws InterruptedException {
    httpExecutor.shutdown();
    httpExecutor.awaitTermination(1, TimeUnit.SECONDS);
+    retryExecutor.shutdown();
+    retryExecutor.awaitTermination(1, TimeUnit.SECONDS);
  }

  @Test
--- a/service/src/test/java/org/whispersystems/textsecuregcm/securestorage/SecureStorageClientTest.java
+++ b/service/src/test/java/org/whispersystems/textsecuregcm/securestorage/SecureStorageClientTest.java
@@ -22,6 +22,7 @@ import java.util.UUID;
 import java.util.concurrent.CompletionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
 import org.apache.commons.lang3.RandomStringUtils;
 import org.junit.jupiter.api.AfterEach;
@@ -36,111 +37,120 @@ import org.whispersystems.textsecuregcm.configuration.SecureStorageServiceConfig

 class SecureStorageClientTest {

-    private UUID accountUuid;
-    private ExternalServiceCredentialsGenerator credentialsGenerator;
-    private ExecutorService httpExecutor;
+  private UUID accountUuid;
+  private ExternalServiceCredentialsGenerator credentialsGenerator;
+  private ExecutorService httpExecutor;
+  private ScheduledExecutorService retryExecutor;

-    private SecureStorageClient secureStorageClient;
+  private SecureStorageClient secureStorageClient;

-    @RegisterExtension
-    private final WireMockExtension wireMock = WireMockExtension.newInstance()
-        .options(wireMockConfig().dynamicPort().dynamicHttpsPort())
-        .build();
+  @RegisterExtension
+  private final WireMockExtension wireMock = WireMockExtension.newInstance()
+      .options(wireMockConfig().dynamicPort().dynamicHttpsPort())
+      .build();

-    @BeforeEach
-    void setUp() throws CertificateException {
-        accountUuid         = UUID.randomUUID();
-        credentialsGenerator = mock(ExternalServiceCredentialsGenerator.class);
-        httpExecutor        = Executors.newSingleThreadExecutor();
+  @BeforeEach
+  void setUp() throws CertificateException {
+    accountUuid = UUID.randomUUID();
+    credentialsGenerator = mock(ExternalServiceCredentialsGenerator.class);
+    httpExecutor = Executors.newSingleThreadExecutor();
+    retryExecutor = Executors.newSingleThreadScheduledExecutor();

-        final SecureStorageServiceConfiguration config = new SecureStorageServiceConfiguration(
-            randomSecretBytes(32),
-            "http://localhost:" + wireMock.getPort(),
-            List.of("""  
-                -----BEGIN CERTIFICATE-----
-                MIICZDCCAc2gAwIBAgIBADANBgkqhkiG9w0BAQ0FADBPMQswCQYDVQQGEwJ1czEL
-                MAkGA1UECAwCVVMxHjAcBgNVBAoMFVNpZ25hbCBNZXNzZW5nZXIsIExMQzETMBEG
-                A1UEAwwKc2lnbmFsLm9yZzAeFw0yMDEyMjMyMjQ3NTlaFw0zMDEyMjEyMjQ3NTla
-                ME8xCzAJBgNVBAYTAnVzMQswCQYDVQQIDAJVUzEeMBwGA1UECgwVU2lnbmFsIE1l
-                c3NlbmdlciwgTExDMRMwEQYDVQQDDApzaWduYWwub3JnMIGfMA0GCSqGSIb3DQEB
-                AQUAA4GNADCBiQKBgQCfSLcZNHYqbxSsgWp4JvbPRHjQTrlsrKrgD2q7f/OY6O3Y
-                /X0QNcNSOJpliN8rmzwslfsrXHO3q1diGRw4xHogUJZ/7NQrHiP/zhN0VTDh49pD
-                ZpjXVyUbayLS/6qM5arKxBspzEFBb5v8cF6bPr76SO/rpGXiI0j6yJKX6fRiKwID
-                AQABo1AwTjAdBgNVHQ4EFgQU6Jrs/Fmj0z4dA3wvdq/WqA4P49IwHwYDVR0jBBgw
-                FoAU6Jrs/Fmj0z4dA3wvdq/WqA4P49IwDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0B
-                AQ0FAAOBgQB+5d5+NtzLILfrc9QmJdIO1YeDP64JmFwTER0kEUouRsb9UwknVWZa
-                y7MTM4NoBV1k0zb5LAk89SIDPr/maW5AsLtEomzjnEiomjoMBUdNe3YCgQReoLnr
-                R/QaUNbrCjTGYfBsjGbIzmkWPUyTec2ZdRyJ8JiVl386+6CZkxnndQ==
-                -----END CERTIFICATE-----
-                """,
-                """
-                -----BEGIN CERTIFICATE-----
-                MIIEpDCCAowCCQC43PUTWSADVjANBgkqhkiG9w0BAQsFADAUMRIwEAYDVQQDDAls
-                b2NhbGhvc3QwHhcNMjIxMDE3MjA0NTM0WhcNMjMxMDE3MjA0NTM0WjAUMRIwEAYD
-                VQQDDAlsb2NhbGhvc3QwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDV
-                x1cdEd2ffQTlTXWRiCHGcrlYf4RJnctt9sw/BuHWTLXBu5LhyJSGn5LRszO/NCXK
-                Z/cmGR7pLj366RtiwL+Qo3nhvDCK7T9xZeNIusM6XMcMK9D/DGCYPqtjQz8NXd9V
-                ajBBe6nwTDTa+oqX8Mt89foWNkg5Il/lY62u9Dr18LRZ2W9zzYi3Q9/K0CbIX6pM
-                yVlPIO5rITOR2IsbeyqsO9jufgX5lP4ZKLLBAP1b7usjC4YdvWacjQg/rK5aay1x
-                jC2HCDgo/4N30QVXzSA9nFfSe6AE/xkStK4819JqOkY5JsJCbef1P3hOOdSLEjbp
-                xq3MjOs6G6dOgteaAGs10vx7dHxDWETTIiD7BIZ9zRYgOF5bkCaIUO+JfySE1MHD
-                KBAFLoRuvmRev5Ln5R0MCHpUMSmMNgJqz+RWZV3g/gpYbuWiHgJOwL1393eK50Bg
-                W7SXQ8EjJj2yXZSH+1gPzN0DRoJZiaBoTPnCL2qUgvwFpW1PJsM5FDyUJFUoK5kK
-                HLBBSKAPt6ZlSrUe2nBgJv7EF1GK+fTU08LXgW33OpLceGPa0zTShkukQUMtUtZ8
-                GqhO12ohMzEupIu5Xurthq4VVUrzHUdj1ZZRMhAbfLU36sd03MMyL/xBqTN6dzCa
-                GDGIPGpYjAllZ5xMRt2kZdv+Kr6oo3u2nLUIsqI7KQIDAQABMA0GCSqGSIb3DQEB
-                CwUAA4ICAQCB5s43YF35ssf5YONW5iAaifGpi1o0866xfeOybtohFGvQ7V2W34i9
-                TYBCt8+0hgatMcvZ08f0vqig1i7nrvYcE1hnhL7JNkU8qm0s9ytHZt6j62nB0kd/
-                uqE2hOEQalTf/2TGPV0CCgiqLyd8lEUQvQeA38wktwUeZpVnErlzHeMR2CvV3K8R
-                u4vV6SnBcf+TAt56RKYZkPyvZj5llQPo14Glyoo8qZES7Ky1SHmM0GL+baPRBjRW
-                3KgSt98Wyu4yr9qu21JpnbAnLhBfzfSKjSeCRgFElUE1GIaFGRZ7ypA74dUKeLnb
-                /VUWrszmUhGaEjV9dpI6x6B/kSpQMtIQqBaKRY2ALUeEujS/rURi4iMDwSU+GkSH
-                cyEvZKS97OA/dWeXfLXdo4beDBRG93bI4rQnDg5+VdlBOkQSLueb8x6/VThMoC5d
-                vZiotFQHseljQAdTkNa6tBu6c4XDYPCKB3CfkMYOlCfTS7Acn5G6dxTPKBtLGBnL
-                nQfYyzuwYkN09+2PVzt6auBHr3To7uoclkxX+hxyvPIwIZ0N6b4tQR1FCAkvg29Q
-                WIOjZOKGW690ESKCKOnFjUHVO0HpuWnT81URTuY62FXsYdVc2wE4v0E04mEbqQ0P
-                lY6ZKNA81Lm3YADYtObmK1IUrOPo9BeIaPy0UM08SmN880Vunqa91Q==
-                -----END CERTIFICATE-----
-                """),
-            new CircuitBreakerConfiguration(),
-            new RetryConfiguration());
+    final SecureStorageServiceConfiguration config = new SecureStorageServiceConfiguration(
+        randomSecretBytes(32),
+        "http://localhost:" + wireMock.getPort(),
+        List.of("""
+            -----BEGIN CERTIFICATE-----
+            MIICZDCCAc2gAwIBAgIBADANBgkqhkiG9w0BAQ0FADBPMQswCQYDVQQGEwJ1czEL
+            MAkGA1UECAwCVVMxHjAcBgNVBAoMFVNpZ25hbCBNZXNzZW5nZXIsIExMQzETMBEG
+            A1UEAwwKc2lnbmFsLm9yZzAeFw0yMDEyMjMyMjQ3NTlaFw0zMDEyMjEyMjQ3NTla
+            ME8xCzAJBgNVBAYTAnVzMQswCQYDVQQIDAJVUzEeMBwGA1UECgwVU2lnbmFsIE1l
+            c3NlbmdlciwgTExDMRMwEQYDVQQDDApzaWduYWwub3JnMIGfMA0GCSqGSIb3DQEB
+            AQUAA4GNADCBiQKBgQCfSLcZNHYqbxSsgWp4JvbPRHjQTrlsrKrgD2q7f/OY6O3Y
+            /X0QNcNSOJpliN8rmzwslfsrXHO3q1diGRw4xHogUJZ/7NQrHiP/zhN0VTDh49pD
+            ZpjXVyUbayLS/6qM5arKxBspzEFBb5v8cF6bPr76SO/rpGXiI0j6yJKX6fRiKwID
+            AQABo1AwTjAdBgNVHQ4EFgQU6Jrs/Fmj0z4dA3wvdq/WqA4P49IwHwYDVR0jBBgw
+            FoAU6Jrs/Fmj0z4dA3wvdq/WqA4P49IwDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0B
+            AQ0FAAOBgQB+5d5+NtzLILfrc9QmJdIO1YeDP64JmFwTER0kEUouRsb9UwknVWZa
+            y7MTM4NoBV1k0zb5LAk89SIDPr/maW5AsLtEomzjnEiomjoMBUdNe3YCgQReoLnr
+            R/QaUNbrCjTGYfBsjGbIzmkWPUyTec2ZdRyJ8JiVl386+6CZkxnndQ==
+            -----END CERTIFICATE-----
+            """, """
+            -----BEGIN CERTIFICATE-----
+            MIIEpDCCAowCCQC43PUTWSADVjANBgkqhkiG9w0BAQsFADAUMRIwEAYDVQQDDAls
+            b2NhbGhvc3QwHhcNMjIxMDE3MjA0NTM0WhcNMjMxMDE3MjA0NTM0WjAUMRIwEAYD
+            VQQDDAlsb2NhbGhvc3QwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDV
+            x1cdEd2ffQTlTXWRiCHGcrlYf4RJnctt9sw/BuHWTLXBu5LhyJSGn5LRszO/NCXK
+            Z/cmGR7pLj366RtiwL+Qo3nhvDCK7T9xZeNIusM6XMcMK9D/DGCYPqtjQz8NXd9V
+            ajBBe6nwTDTa+oqX8Mt89foWNkg5Il/lY62u9Dr18LRZ2W9zzYi3Q9/K0CbIX6pM
+            yVlPIO5rITOR2IsbeyqsO9jufgX5lP4ZKLLBAP1b7usjC4YdvWacjQg/rK5aay1x
+            jC2HCDgo/4N30QVXzSA9nFfSe6AE/xkStK4819JqOkY5JsJCbef1P3hOOdSLEjbp
+            xq3MjOs6G6dOgteaAGs10vx7dHxDWETTIiD7BIZ9zRYgOF5bkCaIUO+JfySE1MHD
+            KBAFLoRuvmRev5Ln5R0MCHpUMSmMNgJqz+RWZV3g/gpYbuWiHgJOwL1393eK50Bg
+            W7SXQ8EjJj2yXZSH+1gPzN0DRoJZiaBoTPnCL2qUgvwFpW1PJsM5FDyUJFUoK5kK
+            HLBBSKAPt6ZlSrUe2nBgJv7EF1GK+fTU08LXgW33OpLceGPa0zTShkukQUMtUtZ8
+            GqhO12ohMzEupIu5Xurthq4VVUrzHUdj1ZZRMhAbfLU36sd03MMyL/xBqTN6dzCa
+            GDGIPGpYjAllZ5xMRt2kZdv+Kr6oo3u2nLUIsqI7KQIDAQABMA0GCSqGSIb3DQEB
+            CwUAA4ICAQCB5s43YF35ssf5YONW5iAaifGpi1o0866xfeOybtohFGvQ7V2W34i9
+            TYBCt8+0hgatMcvZ08f0vqig1i7nrvYcE1hnhL7JNkU8qm0s9ytHZt6j62nB0kd/
+            uqE2hOEQalTf/2TGPV0CCgiqLyd8lEUQvQeA38wktwUeZpVnErlzHeMR2CvV3K8R
+            u4vV6SnBcf+TAt56RKYZkPyvZj5llQPo14Glyoo8qZES7Ky1SHmM0GL+baPRBjRW
+            3KgSt98Wyu4yr9qu21JpnbAnLhBfzfSKjSeCRgFElUE1GIaFGRZ7ypA74dUKeLnb
+            /VUWrszmUhGaEjV9dpI6x6B/kSpQMtIQqBaKRY2ALUeEujS/rURi4iMDwSU+GkSH
+            cyEvZKS97OA/dWeXfLXdo4beDBRG93bI4rQnDg5+VdlBOkQSLueb8x6/VThMoC5d
+            vZiotFQHseljQAdTkNa6tBu6c4XDYPCKB3CfkMYOlCfTS7Acn5G6dxTPKBtLGBnL
+            nQfYyzuwYkN09+2PVzt6auBHr3To7uoclkxX+hxyvPIwIZ0N6b4tQR1FCAkvg29Q
+            WIOjZOKGW690ESKCKOnFjUHVO0HpuWnT81URTuY62FXsYdVc2wE4v0E04mEbqQ0P
+            lY6ZKNA81Lm3YADYtObmK1IUrOPo9BeIaPy0UM08SmN880Vunqa91Q==
+            -----END CERTIFICATE-----
+            """),
+        new CircuitBreakerConfiguration(),
+        new RetryConfiguration());

-        secureStorageClient = new SecureStorageClient(credentialsGenerator, httpExecutor, config);
-    }
+    secureStorageClient = new SecureStorageClient(credentialsGenerator, httpExecutor, retryExecutor, config);
+  }

-    @AfterEach
-    void tearDown() throws InterruptedException {
-        httpExecutor.shutdown();
-        httpExecutor.awaitTermination(1, TimeUnit.SECONDS);
-    }
+  @AfterEach
+  void tearDown() throws InterruptedException {

-    @Test
-    void deleteStoredData() {
-        final String username = RandomStringUtils.randomAlphabetic(16);
-        final String password = RandomStringUtils.randomAlphanumeric(32);
+    httpExecutor.shutdown();
+    httpExecutor.awaitTermination(1, TimeUnit.SECONDS);
+    retryExecutor.shutdown();
+    retryExecutor.awaitTermination(1, TimeUnit.SECONDS);
+  }

-        when(credentialsGenerator.generateForUuid(accountUuid)).thenReturn(new ExternalServiceCredentials(username, password));
+  @Test
+  void deleteStoredData() {

-        wireMock.stubFor(delete(urlEqualTo(SecureStorageClient.DELETE_PATH))
-                .withBasicAuth(username, password)
-                .willReturn(aResponse().withStatus(202)));
+    final String username = RandomStringUtils.randomAlphabetic(16);
+    final String password = RandomStringUtils.randomAlphanumeric(32);

-        // We're happy as long as this doesn't throw an exception
-        secureStorageClient.deleteStoredData(accountUuid).join();
-    }
+    when(credentialsGenerator.generateForUuid(accountUuid)).thenReturn(
+        new ExternalServiceCredentials(username, password));

-    @Test
-    void deleteStoredDataFailure() {
-        final String username = RandomStringUtils.randomAlphabetic(16);
-        final String password = RandomStringUtils.randomAlphanumeric(32);
+    wireMock.stubFor(delete(urlEqualTo(SecureStorageClient.DELETE_PATH))
+        .withBasicAuth(username, password)
+        .willReturn(aResponse().withStatus(202)));

-        when(credentialsGenerator.generateForUuid(accountUuid)).thenReturn(new ExternalServiceCredentials(username, password));
+    // We're happy as long as this doesn't throw an exception
+    secureStorageClient.deleteStoredData(accountUuid).join();
+  }

-        wireMock.stubFor(delete(urlEqualTo(SecureStorageClient.DELETE_PATH))
-                .withBasicAuth(username, password)
-                .willReturn(aResponse().withStatus(400)));
+  @Test
+  void deleteStoredDataFailure() {

-        final CompletionException completionException = assertThrows(CompletionException.class, () -> secureStorageClient.deleteStoredData(accountUuid).join());
-        assertTrue(completionException.getCause() instanceof SecureStorageException);
-    }
+    final String username = RandomStringUtils.randomAlphabetic(16);
+    final String password = RandomStringUtils.randomAlphanumeric(32);
+
+    when(credentialsGenerator.generateForUuid(accountUuid)).thenReturn(
+        new ExternalServiceCredentials(username, password));
+
+    wireMock.stubFor(delete(urlEqualTo(SecureStorageClient.DELETE_PATH))
+        .withBasicAuth(username, password)
+        .willReturn(aResponse().withStatus(400)));
+
+    final CompletionException completionException = assertThrows(CompletionException.class,
+        () -> secureStorageClient.deleteStoredData(accountUuid).join());
+    assertTrue(completionException.getCause() instanceof SecureStorageException);
+  }
 }
--- a/service/src/test/java/org/whispersystems/textsecuregcm/securevaluerecovery/SecureValueRecovery2ClientTest.java
+++ b/service/src/test/java/org/whispersystems/textsecuregcm/securevaluerecovery/SecureValueRecovery2ClientTest.java
@@ -23,6 +23,7 @@ import java.util.UUID;
 import java.util.concurrent.CompletionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
 import org.apache.commons.lang3.RandomStringUtils;
 import org.junit.jupiter.api.AfterEach;
@@ -38,6 +39,7 @@ class SecureValueRecovery2ClientTest {
  private UUID accountUuid;
  private ExternalServiceCredentialsGenerator credentialsGenerator;
  private ExecutorService httpExecutor;
+  private ScheduledExecutorService retryExecutor;

  private SecureValueRecovery2Client secureValueRecovery2Client;

@@ -51,6 +53,7 @@ class SecureValueRecovery2ClientTest {
    accountUuid = UUID.randomUUID();
    credentialsGenerator = mock(ExternalServiceCredentialsGenerator.class);
    httpExecutor = Executors.newSingleThreadExecutor();
+    retryExecutor = Executors.newSingleThreadScheduledExecutor();

    final SecureValueRecovery2Configuration config = new SecureValueRecovery2Configuration(
        "http://localhost:" + wireMock.getPort(),
@@ -104,13 +107,16 @@ class SecureValueRecovery2ClientTest {
            """),
        null, null);

-    secureValueRecovery2Client = new SecureValueRecovery2Client(credentialsGenerator, httpExecutor, config);
+    secureValueRecovery2Client = new SecureValueRecovery2Client(credentialsGenerator, httpExecutor, retryExecutor,
+        config);
  }

  @AfterEach
  void tearDown() throws InterruptedException {
    httpExecutor.shutdown();
    httpExecutor.awaitTermination(1, TimeUnit.SECONDS);
+    retryExecutor.shutdown();
+    retryExecutor.awaitTermination(1, TimeUnit.SECONDS);
  }

  @Test
--- a/service/src/test/java/org/whispersystems/textsecuregcm/tests/http/FaultTolerantHttpClientTest.java
+++ b/service/src/test/java/org/whispersystems/textsecuregcm/tests/http/FaultTolerantHttpClientTest.java
@@ -20,7 +20,12 @@ import java.net.http.HttpClient;
 import java.net.http.HttpRequest;
 import java.net.http.HttpResponse;
 import java.util.concurrent.CompletionException;
+import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.RegisterExtension;
 import org.whispersystems.textsecuregcm.configuration.CircuitBreakerConfiguration;
@@ -34,21 +39,38 @@ class FaultTolerantHttpClientTest {
      .options(wireMockConfig().dynamicPort().dynamicHttpsPort())
      .build();

+  private ExecutorService httpExecutor;
+  private ScheduledExecutorService retryExecutor;
+
+  @BeforeEach
+  void setUp() {
+    httpExecutor = Executors.newSingleThreadExecutor();
+    retryExecutor = Executors.newSingleThreadScheduledExecutor();
+  }
+
+  @AfterEach
+  void tearDown() throws InterruptedException {
+    httpExecutor.shutdown();
+    httpExecutor.awaitTermination(1, TimeUnit.SECONDS);
+    retryExecutor.shutdown();
+    retryExecutor.awaitTermination(1, TimeUnit.SECONDS);
+  }
+
  @Test
  void testSimpleGet() {
    wireMock.stubFor(get(urlEqualTo("/ping"))
-                             .willReturn(aResponse()
-                                             .withHeader("Content-Type", "text/plain")
-                                             .withBody("Pong!")));
-
+        .willReturn(aResponse()
+            .withHeader("Content-Type", "text/plain")
+            .withBody("Pong!")));

    FaultTolerantHttpClient client = FaultTolerantHttpClient.newBuilder()
-                                                            .withCircuitBreaker(new CircuitBreakerConfiguration())
-                                                            .withRetry(new RetryConfiguration())
-                                                            .withExecutor(Executors.newSingleThreadExecutor())
-                                                            .withName("test")
-                                                            .withVersion(HttpClient.Version.HTTP_2)
-                                                            .build();
+        .withCircuitBreaker(new CircuitBreakerConfiguration())
+        .withRetry(new RetryConfiguration())
+        .withExecutor(httpExecutor)
+        .withRetryExecutor(retryExecutor)
+        .withName("test")
+        .withVersion(HttpClient.Version.HTTP_2)
+        .build();

    HttpRequest request = HttpRequest.newBuilder()
                                     .uri(URI.create("http://localhost:" + wireMock.getPort() + "/ping"))
@@ -72,12 +94,13 @@ class FaultTolerantHttpClientTest {
                                             .withBody("Pong!")));

    FaultTolerantHttpClient client = FaultTolerantHttpClient.newBuilder()
-                                                            .withCircuitBreaker(new CircuitBreakerConfiguration())
-                                                            .withRetry(new RetryConfiguration())
-                                                            .withExecutor(Executors.newSingleThreadExecutor())
-                                                            .withName("test")
-                                                            .withVersion(HttpClient.Version.HTTP_2)
-                                                            .build();
+        .withCircuitBreaker(new CircuitBreakerConfiguration())
+        .withRetry(new RetryConfiguration())
+        .withExecutor(httpExecutor)
+        .withRetryExecutor(retryExecutor)
+        .withName("test")
+        .withVersion(HttpClient.Version.HTTP_2)
+        .build();

    HttpRequest request = HttpRequest.newBuilder()
                                     .uri(URI.create("http://localhost:" + wireMock.getPort() + "/failure"))
@@ -104,7 +127,8 @@ class FaultTolerantHttpClientTest {
    FaultTolerantHttpClient client = FaultTolerantHttpClient.newBuilder()
        .withCircuitBreaker(circuitBreakerConfiguration)
        .withRetry(new RetryConfiguration())
-        .withExecutor(Executors.newSingleThreadExecutor())
+        .withRetryExecutor(retryExecutor)
+        .withExecutor(httpExecutor)
        .withName("test")
        .withVersion(HttpClient.Version.HTTP_2)
        .build();