Moving secret values out of the main configuration file

service/src/test/java/org/whispersystems/textsecuregcm/CheckServiceConfigurations.java

@@ -15,19 +15,27 @@ import java.util.Arrays;
  */
 public class CheckServiceConfigurations {
 
+  private static final String SECRETS_BUNDLE_FILENAME = "sample-secrets-bundle.yml";
+
   private void checkConfiguration(final File configDirectory) {
 
     final File[] configFiles = configDirectory.listFiles(f ->
         !f.isDirectory()
-            && f.getPath().endsWith(".yml"));
+            && f.getPath().endsWith(".yml")
+            && !f.getPath().endsWith(SECRETS_BUNDLE_FILENAME));
 
     if (configFiles == null || configFiles.length == 0) {
       throw new IllegalArgumentException("No .yml configuration files found at " + configDirectory.getPath());
     }
 
-    for (File configFile : configFiles) {
-      String[] args = new String[]{"check", configFile.getAbsolutePath()};
+    final File[] secretsBundle = configDirectory.listFiles(f -> !f.isDirectory() && f.getName().equals(SECRETS_BUNDLE_FILENAME));
+    if (secretsBundle == null || secretsBundle.length != 1) {
+      throw new IllegalArgumentException("No [%s] file found at %s".formatted(SECRETS_BUNDLE_FILENAME, configDirectory.getPath()));
+    }
+    System.setProperty(WhisperServerService.SECRETS_BUNDLE_FILE_NAME_PROPERTY, secretsBundle[0].getAbsolutePath());
 
+    for (final File configFile : configFiles) {
+      final String[] args = new String[]{"check", configFile.getAbsolutePath()};
       try {
         new WhisperServerService().run(args);
       } catch (final Exception e) {
@@ -38,8 +46,7 @@ public class CheckServiceConfigurations {
     }
   }
 
-  public static void main(String[] args) {
-
+  public static void main(final String[] args) {
     if (args.length != 1) {
       throw new IllegalArgumentException("Expected single argument with config directory: " + Arrays.toString(args));
     }
@@ -52,5 +59,4 @@ public class CheckServiceConfigurations {
 
     new CheckServiceConfigurations().checkConfiguration(configDirectory);
   }
-
 }

service/src/test/java/org/whispersystems/textsecuregcm/configuration/secrets/SecretsTest.java

@@ -0,0 +1,154 @@
+/*
+ * Copyright 2023 Signal Messenger, LLC
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+package org.whispersystems.textsecuregcm.configuration.secrets;
+
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import com.fasterxml.jackson.databind.JsonMappingException;
+import java.util.Base64;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import javax.validation.Validation;
+import javax.validation.Validator;
+import javax.validation.constraints.NotEmpty;
+import org.apache.commons.lang3.RandomUtils;
+import org.junit.jupiter.api.Test;
+import org.whispersystems.textsecuregcm.util.ExactlySize;
+import org.whispersystems.textsecuregcm.util.SystemMapper;
+
+public class SecretsTest {
+
+  private static final String SECRET_REF = "secret_string";
+
+  private static final String SECRET_LIST_REF = "secret_string_list";
+
+  private static final String SECRET_BYTES_REF = "secret_bytes";
+
+  private static final String SECRET_BYTES_LIST_REF = "secret_bytes_list";
+
+  public record TestData(SecretString secret,
+                         SecretBytes secretBytes,
+                         SecretStringList secretList,
+                         SecretBytesList secretBytesList) {
+  }
+
+  private static final String VALID_CONFIG_YAML = """
+      secret: secret://%s
+      secretBytes: secret://%s
+      secretList: secret://%s
+      secretBytesList: secret://%s
+      """.formatted(SECRET_REF, SECRET_BYTES_REF, SECRET_LIST_REF, SECRET_BYTES_LIST_REF);
+
+
+  @Test
+  public void testDeserialization() throws Exception {
+    final String secretString = "secret_string";
+    final byte[] secretBytes = RandomUtils.nextBytes(16);
+    final String secretBytesBase64 = Base64.getEncoder().encodeToString(secretBytes);
+    final List<String> secretStringList = List.of("secret1", "secret2", "secret3");
+    final List<byte[]> secretBytesList = List.of(RandomUtils.nextBytes(16), RandomUtils.nextBytes(16), RandomUtils.nextBytes(16));
+    final List<String> secretBytesListBase64 = secretBytesList.stream().map(Base64.getEncoder()::encodeToString).toList();
+    final Map<String, Secret<?>> storeMap = Map.of(
+        SECRET_REF, new SecretString(secretString),
+        SECRET_BYTES_REF, new SecretString(secretBytesBase64),
+        SECRET_LIST_REF, new SecretStringList(secretStringList),
+        SECRET_BYTES_LIST_REF, new SecretStringList(secretBytesListBase64)
+    );
+    SecretsModule.INSTANCE.setSecretStore(new SecretStore(storeMap));
+
+    final TestData result = SystemMapper.yamlMapper().readValue(VALID_CONFIG_YAML, TestData.class);
+    assertEquals(secretString, result.secret().value());
+    assertEquals(secretStringList, result.secretList().value());
+    assertArrayEquals(secretBytes, result.secretBytes().value());
+    for (int i = 0; i < secretBytesList.size(); i++) {
+      assertArrayEquals(secretBytesList.get(i), result.secretBytesList().value().get(i));
+    }
+  }
+
+  @Test
+  public void testValueWithoutPrefix() throws Exception {
+    final String config = """
+      secret: ref
+      """;
+    SecretsModule.INSTANCE.setSecretStore(new SecretStore(Collections.emptyMap()));
+    assertThrows(JsonMappingException.class, () -> SystemMapper.yamlMapper().readValue(config, TestData.class));
+  }
+
+  @Test
+  public void testNoSecretInTheStore() throws Exception {
+    final String config = """
+      secret: secret://missing
+      secretBytes: secret://missing
+      secretList: secret://missing
+      secretBytesList: secret://missing
+      """;
+    SecretsModule.INSTANCE.setSecretStore(new SecretStore(Collections.emptyMap()));
+    assertThrows(JsonMappingException.class, () -> SystemMapper.yamlMapper().readValue(config, TestData.class));
+  }
+
+  @Test
+  public void testSecretStoreNotSet() throws Exception {
+    assertThrows(JsonMappingException.class, () -> SystemMapper.yamlMapper().readValue(VALID_CONFIG_YAML, TestData.class));
+  }
+
+  @Test
+  public void testReadFromJson() throws Exception {
+    // checking that valid json secrets bundle is read correctly
+    final SecretStore secretStore = SecretStore.fromYamlStringSecretsBundle("""
+        secret_string: value
+        secret_string_list:
+          - value1
+          - value2
+          - value3
+        """);
+    assertEquals("value", secretStore.secretString("secret_string").value());
+    assertEquals(List.of("value1", "value2", "value3"), secretStore.secretStringList("secret_string_list").value());
+
+    // checking that secrets bundle can't have objects as values
+    assertThrows(IllegalArgumentException.class, () -> SecretStore.fromYamlStringSecretsBundle("""
+        secret_string: value
+        not_a_string_or_list:
+          k: v
+        """));
+
+    // checking that secrets bundle can't have numbers as values
+    assertThrows(IllegalArgumentException.class, () -> SecretStore.fromYamlStringSecretsBundle("""
+        secret_string: value
+        not_a_string_or_list: 42
+        """));
+  }
+
+  record NotEmptySecretStringList(@NotEmpty SecretStringList secret) {
+  }
+
+  record NotEmptySecretBytesList(@NotEmpty SecretBytesList secret) {
+  }
+
+  record ExactlySizeBytesSecret(@ExactlySize(32) SecretBytes secret) {
+  }
+
+  @Test
+  public void testValidators() throws Exception {
+    final Validator validator = Validation.buildDefaultValidatorFactory().getValidator();
+
+    // @NotEmpty SecretStringList
+    assertFalse(validator.validate(new NotEmptySecretStringList(new SecretStringList(List.of()))).isEmpty());
+    assertTrue(validator.validate(new NotEmptySecretStringList(new SecretStringList(List.of("smth")))).isEmpty());
+
+    // @NotEmpty SecretBytesList
+    assertFalse(validator.validate(new NotEmptySecretBytesList(new SecretBytesList(List.of()))).isEmpty());
+    assertTrue(validator.validate(new NotEmptySecretBytesList(new SecretBytesList(List.of(new byte[4])))).isEmpty());
+
+    // @ExactlySize SecretBytes
+    assertFalse(validator.validate(new ExactlySizeBytesSecret(new SecretBytes(new byte[16]))).isEmpty());
+    assertTrue(validator.validate(new ExactlySizeBytesSecret(new SecretBytes(new byte[32]))).isEmpty());
+  }
+}

service/src/test/java/org/whispersystems/textsecuregcm/controllers/AccountControllerTest.java

@@ -24,6 +24,7 @@ import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.verifyNoInteractions;
 import static org.mockito.Mockito.when;
+import static org.whispersystems.textsecuregcm.util.MockUtils.randomSecretBytes;
 
 import com.google.common.collect.ImmutableSet;
 import com.google.common.net.HttpHeaders;
@@ -72,7 +73,6 @@ import org.signal.libsignal.protocol.ecc.ECKeyPair;
 import org.signal.libsignal.usernames.BaseUsernameException;
 import org.whispersystems.textsecuregcm.auth.AuthenticatedAccount;
 import org.whispersystems.textsecuregcm.auth.DisabledPermittedAuthenticatedAccount;
-import org.whispersystems.textsecuregcm.auth.ExternalServiceCredentials;
 import org.whispersystems.textsecuregcm.auth.ExternalServiceCredentialsGenerator;
 import org.whispersystems.textsecuregcm.auth.RegistrationLockVerificationManager;
 import org.whispersystems.textsecuregcm.auth.SaltedTokenHash;
@@ -99,7 +99,6 @@ import org.whispersystems.textsecuregcm.entities.RegistrationLock;
 import org.whispersystems.textsecuregcm.entities.RegistrationLockFailure;
 import org.whispersystems.textsecuregcm.entities.ReserveUsernameHashRequest;
 import org.whispersystems.textsecuregcm.entities.ReserveUsernameHashResponse;
-import org.whispersystems.textsecuregcm.entities.SignedPreKey;
 import org.whispersystems.textsecuregcm.entities.UsernameHashResponse;
 import org.whispersystems.textsecuregcm.limits.RateLimitByIpFilter;
 import org.whispersystems.textsecuregcm.limits.RateLimiter;
@@ -203,13 +202,13 @@ class AccountControllerTest {
 
   private static final SecureBackupServiceConfiguration SVR1_CFG = MockUtils.buildMock(
       SecureBackupServiceConfiguration.class,
-      cfg -> when(cfg.getUserAuthenticationTokenSharedSecret()).thenReturn(new byte[32]));
+      cfg -> when(cfg.userAuthenticationTokenSharedSecret()).thenReturn(randomSecretBytes(32)));
 
   private static final SecureValueRecovery2Configuration SVR2_CFG = MockUtils.buildMock(
       SecureValueRecovery2Configuration.class,
       cfg -> {
-        when(cfg.userAuthenticationTokenSharedSecret()).thenReturn(new byte[32]);
-        when(cfg.userIdTokenSharedSecret()).thenReturn(new byte[32]);
+        when(cfg.userAuthenticationTokenSharedSecret()).thenReturn(randomSecretBytes(32));
+        when(cfg.userIdTokenSharedSecret()).thenReturn(randomSecretBytes(32));
       });
 
   private static final ExternalServiceCredentialsGenerator svr1CredentialsGenerator = SecureBackupController.credentialsGenerator(

service/src/test/java/org/whispersystems/textsecuregcm/controllers/SecureBackupControllerTest.java

@@ -5,32 +5,17 @@
 
 package org.whispersystems.textsecuregcm.controllers;
 
-import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.mockito.Mockito.mock;
+import static org.whispersystems.textsecuregcm.util.MockUtils.randomSecretBytes;
 
 import io.dropwizard.testing.junit5.DropwizardExtensionsSupport;
 import io.dropwizard.testing.junit5.ResourceExtension;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.UUID;
-import java.util.concurrent.TimeUnit;
-import javax.ws.rs.client.Entity;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
-import org.apache.commons.lang3.RandomUtils;
 import org.glassfish.jersey.test.grizzly.GrizzlyWebTestContainerFactory;
-import org.junit.jupiter.api.BeforeAll;
-import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.mockito.Mockito;
-import org.whispersystems.textsecuregcm.auth.ExternalServiceCredentials;
 import org.whispersystems.textsecuregcm.auth.ExternalServiceCredentialsGenerator;
 import org.whispersystems.textsecuregcm.configuration.SecureBackupServiceConfiguration;
-import org.whispersystems.textsecuregcm.entities.AuthCheckRequest;
-import org.whispersystems.textsecuregcm.entities.AuthCheckResponse;
-import org.whispersystems.textsecuregcm.storage.Account;
+import org.whispersystems.textsecuregcm.configuration.secrets.SecretBytes;
 import org.whispersystems.textsecuregcm.storage.AccountsManager;
 import org.whispersystems.textsecuregcm.tests.util.AuthHelper;
 import org.whispersystems.textsecuregcm.util.MockUtils;
@@ -40,11 +25,11 @@ import org.whispersystems.textsecuregcm.util.SystemMapper;
 @ExtendWith(DropwizardExtensionsSupport.class)
 class SecureBackupControllerTest extends SecureValueRecoveryControllerBaseTest {
 
-  private static final byte[] SECRET = RandomUtils.nextBytes(32);
+  private static final SecretBytes SECRET = randomSecretBytes(32);
 
   private static final SecureBackupServiceConfiguration CFG = MockUtils.buildMock(
       SecureBackupServiceConfiguration.class,
-      cfg -> Mockito.when(cfg.getUserAuthenticationTokenSharedSecret()).thenReturn(SECRET)
+      cfg -> Mockito.when(cfg.userAuthenticationTokenSharedSecret()).thenReturn(SECRET)
   );
 
   private static final MutableClock CLOCK = new MutableClock();

service/src/test/java/org/whispersystems/textsecuregcm/controllers/SecureValueRecovery2ControllerTest.java

@@ -7,10 +7,10 @@ package org.whispersystems.textsecuregcm.controllers;
 
 
 import static org.mockito.Mockito.mock;
+import static org.whispersystems.textsecuregcm.util.MockUtils.randomSecretBytes;
 
 import io.dropwizard.testing.junit5.DropwizardExtensionsSupport;
 import io.dropwizard.testing.junit5.ResourceExtension;
-import org.apache.commons.lang3.RandomUtils;
 import org.glassfish.jersey.test.grizzly.GrizzlyWebTestContainerFactory;
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.whispersystems.textsecuregcm.auth.ExternalServiceCredentialsGenerator;
@@ -26,8 +26,8 @@ public class SecureValueRecovery2ControllerTest extends SecureValueRecoveryContr
   private static final SecureValueRecovery2Configuration CFG = new SecureValueRecovery2Configuration(
       true,
       "",
-      RandomUtils.nextBytes(32),
-      RandomUtils.nextBytes(32),
+      randomSecretBytes(32),
+      randomSecretBytes(32),
       null,
       null,
       null

service/src/test/java/org/whispersystems/textsecuregcm/securestorage/SecureStorageClientTest.java

@@ -13,6 +13,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
+import static org.whispersystems.textsecuregcm.util.MockUtils.randomSecretBytes;
 
 import com.github.tomakehurst.wiremock.junit5.WireMockExtension;
 import java.security.cert.CertificateException;
@@ -53,7 +54,7 @@ class SecureStorageClientTest {
         httpExecutor        = Executors.newSingleThreadExecutor();
 
         final SecureStorageServiceConfiguration config = new SecureStorageServiceConfiguration(
-            "not_used",
+            randomSecretBytes(32),
             "http://localhost:" + wireMock.getPort(),
             List.of("""  
                 -----BEGIN CERTIFICATE-----

service/src/test/java/org/whispersystems/textsecuregcm/securevaluerecovery/SecureValueRecovery2ClientTest.java

@@ -14,6 +14,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
+import static org.whispersystems.textsecuregcm.util.MockUtils.randomSecretBytes;
 
 import com.github.tomakehurst.wiremock.junit5.WireMockExtension;
 import java.security.cert.CertificateException;
@@ -53,7 +54,8 @@ class SecureValueRecovery2ClientTest {
 
     final SecureValueRecovery2Configuration config = new SecureValueRecovery2Configuration(true,
         "http://localhost:" + wireMock.getPort(),
-        new byte[0], new byte[0],
+        randomSecretBytes(32),
+        randomSecretBytes(32),
         // This is a randomly-generated, throwaway certificate that's not actually connected to anything
         List.of("""
             -----BEGIN CERTIFICATE-----

service/src/test/java/org/whispersystems/textsecuregcm/tests/controllers/ArtControllerTest.java

@@ -8,15 +8,16 @@ package org.whispersystems.textsecuregcm.tests.controllers;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
+import static org.whispersystems.textsecuregcm.util.MockUtils.randomSecretBytes;
 
 import com.google.common.collect.ImmutableSet;
 import io.dropwizard.auth.PolymorphicAuthValueFactoryProvider;
 import io.dropwizard.testing.junit5.DropwizardExtensionsSupport;
 import io.dropwizard.testing.junit5.ResourceExtension;
+import java.time.Duration;
 import org.glassfish.jersey.test.grizzly.GrizzlyWebTestContainerFactory;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
-import org.mockito.Mockito;
 import org.whispersystems.textsecuregcm.auth.AuthenticatedAccount;
 import org.whispersystems.textsecuregcm.auth.DisabledPermittedAuthenticatedAccount;
 import org.whispersystems.textsecuregcm.auth.ExternalServiceCredentials;
@@ -26,18 +27,12 @@ import org.whispersystems.textsecuregcm.controllers.ArtController;
 import org.whispersystems.textsecuregcm.limits.RateLimiter;
 import org.whispersystems.textsecuregcm.limits.RateLimiters;
 import org.whispersystems.textsecuregcm.tests.util.AuthHelper;
-import org.whispersystems.textsecuregcm.util.MockUtils;
 import org.whispersystems.textsecuregcm.util.SystemMapper;
 
 @ExtendWith(DropwizardExtensionsSupport.class)
 class ArtControllerTest {
-
-  private static final ArtServiceConfiguration ART_SERVICE_CONFIGURATION = MockUtils.buildMock(
-      ArtServiceConfiguration.class,
-      cfg -> {
-        Mockito.when(cfg.getUserAuthenticationTokenSharedSecret()).thenReturn(new byte[32]);
-        Mockito.when(cfg.getUserAuthenticationTokenUserIdSecret()).thenReturn(new byte[32]);
-      });
+  private static final ArtServiceConfiguration ART_SERVICE_CONFIGURATION = new ArtServiceConfiguration(
+      randomSecretBytes(32), randomSecretBytes(32), Duration.ofDays(1));
   private static final ExternalServiceCredentialsGenerator artCredentialsGenerator = ArtController.credentialsGenerator(ART_SERVICE_CONFIGURATION);
   private static final RateLimiter  rateLimiter  = mock(RateLimiter.class);
   private static final RateLimiters rateLimiters = mock(RateLimiters.class);

service/src/test/java/org/whispersystems/textsecuregcm/tests/controllers/DirectoryControllerV2Test.java

@@ -8,6 +8,7 @@ package org.whispersystems.textsecuregcm.tests.controllers;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
+import static org.whispersystems.textsecuregcm.util.MockUtils.secretBytesOf;
 
 import java.time.Clock;
 import java.time.Instant;
@@ -28,7 +29,7 @@ class DirectoryControllerV2Test {
   @Test
   void testAuthToken() {
     final ExternalServiceCredentialsGenerator credentialsGenerator = DirectoryV2Controller.credentialsGenerator(
-        new DirectoryV2ClientConfiguration(new byte[]{0x1}, new byte[]{0x2}),
+        new DirectoryV2ClientConfiguration(secretBytesOf(0x01), secretBytesOf(0x02)),
         Clock.fixed(Instant.ofEpochSecond(1633738643L), ZoneId.of("Etc/UTC"))
     );
 

service/src/test/java/org/whispersystems/textsecuregcm/tests/controllers/SecureStorageControllerTest.java

@@ -7,6 +7,7 @@ package org.whispersystems.textsecuregcm.tests.controllers;
 
 import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
 import static org.mockito.Mockito.when;
+import static org.whispersystems.textsecuregcm.util.MockUtils.randomSecretBytes;
 
 import com.google.common.collect.ImmutableSet;
 import io.dropwizard.auth.PolymorphicAuthValueFactoryProvider;
@@ -31,7 +32,7 @@ class SecureStorageControllerTest {
 
   private static final SecureStorageServiceConfiguration STORAGE_CFG = MockUtils.buildMock(
       SecureStorageServiceConfiguration.class,
-      cfg -> when(cfg.decodeUserAuthenticationTokenSharedSecret()).thenReturn(new byte[32]));
+      cfg -> when(cfg.userAuthenticationTokenSharedSecret()).thenReturn(randomSecretBytes(32)));
 
   private static final ExternalServiceCredentialsGenerator STORAGE_CREDENTIAL_GENERATOR = SecureStorageController
       .credentialsGenerator(STORAGE_CFG);

service/src/test/java/org/whispersystems/textsecuregcm/util/MockUtils.java

@@ -12,7 +12,9 @@ import static org.mockito.Mockito.doThrow;
 
 import java.time.Duration;
 import java.util.Optional;
+import org.apache.commons.lang3.RandomUtils;
 import org.mockito.Mockito;
+import org.whispersystems.textsecuregcm.configuration.secrets.SecretBytes;
 import org.whispersystems.textsecuregcm.controllers.RateLimitExceededException;
 import org.whispersystems.textsecuregcm.limits.RateLimiter;
 import org.whispersystems.textsecuregcm.limits.RateLimiters;
@@ -70,4 +72,16 @@ public final class MockUtils {
       throw new RuntimeException(e);
     }
   }
+
+  public static SecretBytes randomSecretBytes(final int size) {
+    return new SecretBytes(RandomUtils.nextBytes(size));
+  }
+
+  public static SecretBytes secretBytesOf(final int... byteVals) {
+    final byte[] bytes = new byte[byteVals.length];
+    for (int i = 0; i < byteVals.length; i++) {
+      bytes[i] = (byte) byteVals[i];
+    }
+    return new SecretBytes(bytes);
+  }
 }