mirror of
https://github.com/signalapp/Signal-Server
synced 2026-04-21 10:58:06 +01:00
Use storage-manager's copy implementation
This commit is contained in:
Ravi Khadiwala
committed by
ravi-signal
ravi-signal
parent
843151859d
commit
fc097db2a0
@@ -101,6 +101,8 @@ public class BackupManagerTest {
|
||||
|
||||
final RateLimiters rateLimiters = mock(RateLimiters.class);
|
||||
when(rateLimiters.forDescriptor(RateLimiters.For.BACKUP_ATTACHMENT)).thenReturn(mediaUploadLimiter);
|
||||
|
||||
when(remoteStorageManager.cdnNumber()).thenReturn(3);
|
||||
|
||||
this.backupsDb = new BackupsDb(
|
||||
DYNAMO_DB_EXTENSION.getDynamoDbAsyncClient(),
|
||||
@@ -113,7 +115,6 @@ public class BackupManagerTest {
|
||||
tusAttachmentGenerator,
|
||||
tusCredentialGenerator,
|
||||
remoteStorageManager,
|
||||
Map.of(3, "cdn3.example.org/attachments"),
|
||||
testClock);
|
||||
}
|
||||
|
||||
@@ -352,7 +353,7 @@ public class BackupManagerTest {
|
||||
final AuthenticatedBackupUser backupUser = backupUser(TestRandomUtil.nextBytes(16), BackupLevel.MEDIA);
|
||||
when(tusCredentialGenerator.generateUpload(any()))
|
||||
.thenReturn(new BackupUploadDescriptor(3, "def", Collections.emptyMap(), ""));
|
||||
when(remoteStorageManager.copy(eq(URI.create("cdn3.example.org/attachments/abc")), eq(100), any(), any()))
|
||||
when(remoteStorageManager.copy(eq(3), eq("abc"), eq(100), any(), any()))
|
||||
.thenReturn(CompletableFuture.completedFuture(null));
|
||||
final MediaEncryptionParameters encryptionParams = new MediaEncryptionParameters(
|
||||
TestRandomUtil.nextBytes(32),
|
||||
@@ -378,7 +379,7 @@ public class BackupManagerTest {
|
||||
final AuthenticatedBackupUser backupUser = backupUser(TestRandomUtil.nextBytes(16), BackupLevel.MEDIA);
|
||||
when(tusCredentialGenerator.generateUpload(any()))
|
||||
.thenReturn(new BackupUploadDescriptor(3, "def", Collections.emptyMap(), ""));
|
||||
when(remoteStorageManager.copy(eq(URI.create("cdn3.example.org/attachments/abc")), eq(100), any(), any()))
|
||||
when(remoteStorageManager.copy(eq(3), eq("abc"), eq(100), any(), any()))
|
||||
.thenReturn(CompletableFuture.failedFuture(new SourceObjectNotFoundException()));
|
||||
|
||||
CompletableFutureTestUtil.assertFailsWithCause(SourceObjectNotFoundException.class,
|
||||
@@ -394,17 +395,6 @@ public class BackupManagerTest {
|
||||
assertThat(AttributeValues.getLong(backup, BackupsDb.ATTR_MEDIA_COUNT, -1L)).isEqualTo(0L);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void unknownSourceCdn() {
|
||||
final AuthenticatedBackupUser backupUser = backupUser(TestRandomUtil.nextBytes(16), BackupLevel.MEDIA);
|
||||
CompletableFutureTestUtil.assertFailsWithCause(SourceObjectNotFoundException.class,
|
||||
backupManager.copyToBackup(
|
||||
backupUser,
|
||||
0, "abc", 100,
|
||||
mock(MediaEncryptionParameters.class),
|
||||
"def".getBytes(StandardCharsets.UTF_8)));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void quotaEnforcementNoRecalculation() {
|
||||
final AuthenticatedBackupUser backupUser = backupUser(TestRandomUtil.nextBytes(16), BackupLevel.MEDIA);
|
||||
|
||||
@@ -1,20 +0,0 @@
|
||||
package org.whispersystems.textsecuregcm.backup;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
import org.junit.jupiter.params.ParameterizedTest;
|
||||
import org.junit.jupiter.params.provider.ValueSource;
|
||||
import org.whispersystems.textsecuregcm.util.TestRandomUtil;
|
||||
|
||||
public class BackupMediaEncrypterTest {
|
||||
|
||||
@ParameterizedTest
|
||||
@ValueSource(ints = {0, 1, 2, 15, 16, 17, 63, 64, 65, 1023, 1024, 1025})
|
||||
public void sizeCalc() {
|
||||
final MediaEncryptionParameters params = new MediaEncryptionParameters(
|
||||
TestRandomUtil.nextBytes(32),
|
||||
TestRandomUtil.nextBytes(32), TestRandomUtil.nextBytes(16));
|
||||
final BackupMediaEncrypter encrypter = new BackupMediaEncrypter(params);
|
||||
assertThat(params.outputSize(1)).isEqualTo(encrypter.outputSize(1));
|
||||
}
|
||||
}
|
||||
@@ -3,37 +3,23 @@ package org.whispersystems.textsecuregcm.backup;
|
||||
import static com.github.tomakehurst.wiremock.client.WireMock.aResponse;
|
||||
import static com.github.tomakehurst.wiremock.client.WireMock.equalTo;
|
||||
import static com.github.tomakehurst.wiremock.client.WireMock.get;
|
||||
import static com.github.tomakehurst.wiremock.client.WireMock.post;
|
||||
import static com.github.tomakehurst.wiremock.client.WireMock.postRequestedFor;
|
||||
import static com.github.tomakehurst.wiremock.client.WireMock.put;
|
||||
import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo;
|
||||
import static com.github.tomakehurst.wiremock.client.WireMock.urlPathEqualTo;
|
||||
import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
|
||||
import static org.assertj.core.api.Assertions.assertThatNoException;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.github.tomakehurst.wiremock.client.WireMock;
|
||||
import com.github.tomakehurst.wiremock.junit5.WireMockExtension;
|
||||
import io.dropwizard.testing.junit5.DropwizardExtensionsSupport;
|
||||
import java.io.IOException;
|
||||
import java.net.URI;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.security.InvalidAlgorithmParameterException;
|
||||
import java.security.InvalidKeyException;
|
||||
import java.security.NoSuchAlgorithmException;
|
||||
import java.security.cert.CertificateException;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.Executors;
|
||||
import javax.crypto.BadPaddingException;
|
||||
import javax.crypto.Cipher;
|
||||
import javax.crypto.IllegalBlockSizeException;
|
||||
import javax.crypto.Mac;
|
||||
import javax.crypto.NoSuchPaddingException;
|
||||
import javax.crypto.spec.IvParameterSpec;
|
||||
import javax.crypto.spec.SecretKeySpec;
|
||||
import javax.ws.rs.core.HttpHeaders;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
@@ -60,150 +46,83 @@ public class Cdn3RemoteStorageManagerTest {
|
||||
.options(wireMockConfig().dynamicPort())
|
||||
.build();
|
||||
|
||||
private static final String SMALL_CDN2 = "a small object from cdn2";
|
||||
private static final String SMALL_CDN3 = "a small object from cdn3";
|
||||
private static final String LARGE = "a".repeat(1024 * 1024 * 5);
|
||||
|
||||
private RemoteStorageManager remoteStorageManager;
|
||||
|
||||
@BeforeEach
|
||||
public void init() throws CertificateException {
|
||||
public void init() {
|
||||
remoteStorageManager = new Cdn3RemoteStorageManager(
|
||||
Executors.newSingleThreadScheduledExecutor(),
|
||||
new CircuitBreakerConfiguration(),
|
||||
new RetryConfiguration(),
|
||||
Collections.emptyList(),
|
||||
new Cdn3StorageManagerConfiguration(
|
||||
wireMock.url("storage-manager/"),
|
||||
"clientId",
|
||||
new SecretString("clientSecret"),
|
||||
2));
|
||||
|
||||
wireMock.stubFor(get(urlEqualTo("/cdn2/source/small"))
|
||||
.willReturn(aResponse()
|
||||
.withHeader("Content-Length", Integer.toString(SMALL_CDN2.length()))
|
||||
.withBody(SMALL_CDN2)));
|
||||
|
||||
wireMock.stubFor(get(urlEqualTo("/cdn3/source/small"))
|
||||
.willReturn(aResponse()
|
||||
.withHeader("Content-Length", Integer.toString(SMALL_CDN3.length()))
|
||||
.withBody(SMALL_CDN3)));
|
||||
|
||||
wireMock.stubFor(get(urlEqualTo("/cdn3/source/large"))
|
||||
.willReturn(aResponse()
|
||||
.withHeader("Content-Length", Integer.toString(LARGE.length()))
|
||||
.withBody(LARGE)));
|
||||
|
||||
wireMock.stubFor(get(urlEqualTo("/cdn3/source/missing"))
|
||||
.willReturn(aResponse().withStatus(404)));
|
||||
Map.of(2, "gcs", 3, "r2"),
|
||||
2,
|
||||
new CircuitBreakerConfiguration(),
|
||||
new RetryConfiguration()));
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@ValueSource(ints = {2, 3})
|
||||
public void copySmall(final int sourceCdn)
|
||||
throws InvalidAlgorithmParameterException, InvalidKeyException, IllegalBlockSizeException, BadPaddingException {
|
||||
|
||||
final String expectedSource = switch (sourceCdn) {
|
||||
case 2 -> SMALL_CDN2;
|
||||
case 3 -> SMALL_CDN3;
|
||||
public void copy(final int sourceCdn) throws JsonProcessingException {
|
||||
final MediaEncryptionParameters encryptionParameters = new MediaEncryptionParameters(AES_KEY, HMAC_KEY, IV);
|
||||
final String scheme = switch (sourceCdn) {
|
||||
case 2 -> "gcs";
|
||||
case 3 -> "r2";
|
||||
default -> throw new AssertionError();
|
||||
};
|
||||
|
||||
final MediaEncryptionParameters encryptionParameters = new MediaEncryptionParameters(AES_KEY, HMAC_KEY, IV);
|
||||
final long expectedEncryptedLength = encryptionParameters.outputSize(expectedSource.length());
|
||||
|
||||
wireMock.stubFor(post(urlEqualTo("/cdn3/dest"))
|
||||
.withHeader("Content-Length", equalTo(Long.toString(expectedEncryptedLength)))
|
||||
.withHeader("Upload-Length", equalTo(Long.toString(expectedEncryptedLength)))
|
||||
.withHeader("Content-Type", equalTo("application/offset+octet-stream"))
|
||||
.willReturn(aResponse()
|
||||
.withStatus(201)
|
||||
.withHeader("Upload-Offset", Long.toString(expectedEncryptedLength))));
|
||||
|
||||
remoteStorageManager.copy(
|
||||
URI.create(wireMock.url("/cdn" + sourceCdn + "/source/small")),
|
||||
expectedSource.length(),
|
||||
encryptionParameters,
|
||||
new BackupUploadDescriptor(3, "test", Collections.emptyMap(), wireMock.url("/cdn3/dest")))
|
||||
.toCompletableFuture().join();
|
||||
|
||||
final byte[] destBody = wireMock.findAll(postRequestedFor(urlEqualTo("/cdn3/dest"))).getFirst().getBody();
|
||||
assertThat(new String(decrypt(destBody), StandardCharsets.UTF_8))
|
||||
.isEqualTo(expectedSource);
|
||||
final Cdn3RemoteStorageManager.Cdn3CopyRequest expectedCopyRequest = new Cdn3RemoteStorageManager.Cdn3CopyRequest(
|
||||
encryptionParameters,
|
||||
new Cdn3RemoteStorageManager.Cdn3CopyRequest.SourceDescriptor(scheme, "a/test/source"),
|
||||
100,
|
||||
"a/destination");
|
||||
wireMock.stubFor(put(urlEqualTo("/storage-manager/copy"))
|
||||
.withHeader(HttpHeaders.CONTENT_TYPE, equalTo("application/json"))
|
||||
.withRequestBody(WireMock.equalToJson(SystemMapper.jsonMapper().writeValueAsString(expectedCopyRequest)))
|
||||
.willReturn(aResponse().withStatus(204)));
|
||||
assertThatNoException().isThrownBy(() ->
|
||||
remoteStorageManager.copy(
|
||||
sourceCdn,
|
||||
"a/test/source",
|
||||
100,
|
||||
encryptionParameters,
|
||||
"a/destination")
|
||||
.toCompletableFuture().join());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void copyLarge()
|
||||
throws InvalidAlgorithmParameterException, IllegalBlockSizeException, BadPaddingException, InvalidKeyException {
|
||||
final MediaEncryptionParameters params = new MediaEncryptionParameters(AES_KEY, HMAC_KEY, IV);
|
||||
final long expectedEncryptedLength = params.outputSize(LARGE.length());
|
||||
wireMock.stubFor(post(urlEqualTo("/cdn3/dest"))
|
||||
.withHeader("Content-Length", equalTo(Long.toString(expectedEncryptedLength)))
|
||||
.withHeader("Upload-Length", equalTo(Long.toString(expectedEncryptedLength)))
|
||||
.withHeader("Content-Type", equalTo("application/offset+octet-stream"))
|
||||
.willReturn(aResponse()
|
||||
.withStatus(201)
|
||||
.withHeader("Upload-Offset", Long.toString(expectedEncryptedLength))));
|
||||
remoteStorageManager.copy(
|
||||
URI.create(wireMock.url("/cdn3/source/large")),
|
||||
LARGE.length(),
|
||||
params,
|
||||
new BackupUploadDescriptor(3, "test", Collections.emptyMap(), wireMock.url("/cdn3/dest")))
|
||||
.toCompletableFuture().join();
|
||||
|
||||
final byte[] destBody = wireMock.findAll(postRequestedFor(urlEqualTo("/cdn3/dest"))).getFirst().getBody();
|
||||
assertThat(destBody.length)
|
||||
.isEqualTo(new BackupMediaEncrypter(params).outputSize(LARGE.length()))
|
||||
.isEqualTo(params.outputSize(LARGE.length()));
|
||||
assertThat(new String(decrypt(destBody), StandardCharsets.UTF_8)).isEqualTo(LARGE);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void incorrectLength() {
|
||||
public void copyIncorrectLength() {
|
||||
wireMock.stubFor(put(urlPathEqualTo("/storage-manager/copy")).willReturn(aResponse().withStatus(409)));
|
||||
CompletableFutureTestUtil.assertFailsWithCause(InvalidLengthException.class,
|
||||
remoteStorageManager.copy(
|
||||
URI.create(wireMock.url("/cdn3/source/small")),
|
||||
SMALL_CDN3.length() - 1,
|
||||
new MediaEncryptionParameters(AES_KEY, HMAC_KEY, IV),
|
||||
new BackupUploadDescriptor(3, "test", Collections.emptyMap(), wireMock.url("/cdn3/dest")))
|
||||
.toCompletableFuture());
|
||||
2,
|
||||
"a/test/source",
|
||||
100,
|
||||
new MediaEncryptionParameters(AES_KEY, HMAC_KEY, IV),
|
||||
"a/destination").toCompletableFuture());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void sourceMissing() {
|
||||
public void copySourceMissing() {
|
||||
wireMock.stubFor(put(urlPathEqualTo("/storage-manager/copy")).willReturn(aResponse().withStatus(404)));
|
||||
CompletableFutureTestUtil.assertFailsWithCause(SourceObjectNotFoundException.class,
|
||||
remoteStorageManager.copy(
|
||||
URI.create(wireMock.url("/cdn3/source/missing")),
|
||||
1,
|
||||
new MediaEncryptionParameters(AES_KEY, HMAC_KEY, IV),
|
||||
new BackupUploadDescriptor(3, "test", Collections.emptyMap(), wireMock.url("/cdn3/dest")))
|
||||
.toCompletableFuture());
|
||||
2,
|
||||
"a/test/source",
|
||||
100,
|
||||
new MediaEncryptionParameters(AES_KEY, HMAC_KEY, IV),
|
||||
"a/destination").toCompletableFuture());
|
||||
}
|
||||
|
||||
private byte[] decrypt(final byte[] encrypted)
|
||||
throws InvalidAlgorithmParameterException, InvalidKeyException, IllegalBlockSizeException, BadPaddingException {
|
||||
|
||||
final Mac mac;
|
||||
try {
|
||||
mac = Mac.getInstance("HmacSHA256");
|
||||
} catch (NoSuchAlgorithmException e) {
|
||||
throw new AssertionError(e);
|
||||
}
|
||||
|
||||
mac.init(new SecretKeySpec(HMAC_KEY, "HmacSHA256"));
|
||||
mac.update(encrypted, 0, encrypted.length - mac.getMacLength());
|
||||
assertArrayEquals(mac.doFinal(),
|
||||
Arrays.copyOfRange(encrypted, encrypted.length - mac.getMacLength(), encrypted.length));
|
||||
assertArrayEquals(IV, Arrays.copyOf(encrypted, 16));
|
||||
|
||||
final Cipher cipher;
|
||||
try {
|
||||
cipher = Cipher.getInstance("AES/CBC/PKCS5Padding");
|
||||
} catch (NoSuchAlgorithmException | NoSuchPaddingException e) {
|
||||
throw new AssertionError(e);
|
||||
}
|
||||
cipher.init(Cipher.DECRYPT_MODE, new SecretKeySpec(AES_KEY, "AES"), new IvParameterSpec(IV));
|
||||
return cipher.doFinal(encrypted, IV.length, encrypted.length - IV.length - mac.getMacLength());
|
||||
@Test
|
||||
public void copyUnknownCdn() {
|
||||
CompletableFutureTestUtil.assertFailsWithCause(SourceObjectNotFoundException.class,
|
||||
remoteStorageManager.copy(
|
||||
0,
|
||||
"a/test/source",
|
||||
100,
|
||||
new MediaEncryptionParameters(AES_KEY, HMAC_KEY, IV),
|
||||
"a/destination").toCompletableFuture());
|
||||
}
|
||||
|
||||
@Test
|
||||
|
||||
Reference in New Issue
Block a user