Signal/Signal-Server
1
0
Fork 0
mirror of https://github.com/signalapp/Signal-Server synced 2026-04-21 21:18:04 +01:00
Code Issues Packages Projects Releases Wiki Activity

Add media deletion endpoint

Browse Source
This commit is contained in:
Ravi Khadiwala
2024-01-09 13:52:15 -06:00
committed by ravi-signal
parent e934ead85c
commit cc6cf8194f
11 changed files with 340 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

95
service/src/test/java/org/whispersystems/textsecuregcm/backup/BackupManagerTest.java
View File

@@ -8,6 +8,7 @@ package org.whispersystems.textsecuregcm.backup;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatExceptionOfType;
import static org.assertj.core.api.Assertions.assertThatNoException;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
@@ -20,12 +21,14 @@ import static org.mockito.Mockito.when;
import io.grpc.Status;
import io.grpc.StatusRuntimeException;
import java.io.IOException;
import java.net.URI;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Base64;
import java.util.Collections;
@@ -36,6 +39,7 @@ import java.util.UUID;
import java.util.concurrent.CompletableFuture;
import javax.annotation.Nullable;
import org.apache.commons.lang3.StringUtils;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.RegisterExtension;
@@ -102,7 +106,7 @@ public class BackupManagerTest {
backupManager.createMessageBackupUploadDescriptor(backupUser).join();
verify(tusCredentialGenerator, times(1))
.generateUpload(encodedBackupId, BackupManager.MESSAGE_BACKUP_NAME);
.generateUpload("%s/%s".formatted(encodedBackupId, BackupManager.MESSAGE_BACKUP_NAME));
final BackupManager.BackupInfo info = backupManager.backupInfo(backupUser).join();
assertThat(info.backupSubdir()).isEqualTo(encodedBackupId);
@@ -260,7 +264,7 @@ public class BackupManagerTest {
@Test
public void copySuccess() {
final AuthenticatedBackupUser backupUser = backupUser(TestRandomUtil.nextBytes(16), BackupTier.MEDIA);
when(tusCredentialGenerator.generateUpload(any(), any()))
when(tusCredentialGenerator.generateUpload(any()))
.thenReturn(new MessageBackupUploadDescriptor(3, "def", Collections.emptyMap(), ""));
when(remoteStorageManager.copy(eq(URI.create("cdn3.example.org/attachments/abc")), eq(100), any(), any()))
.thenReturn(CompletableFuture.completedFuture(null));
@@ -286,7 +290,7 @@ public class BackupManagerTest {
@Test
public void copyFailure() {
final AuthenticatedBackupUser backupUser = backupUser(TestRandomUtil.nextBytes(16), BackupTier.MEDIA);
when(tusCredentialGenerator.generateUpload(any(), any()))
when(tusCredentialGenerator.generateUpload(any()))
.thenReturn(new MessageBackupUploadDescriptor(3, "def", Collections.emptyMap(), ""));
when(remoteStorageManager.copy(eq(URI.create("cdn3.example.org/attachments/abc")), eq(100), any(), any()))
.thenReturn(CompletableFuture.failedFuture(new SourceObjectNotFoundException()));
@@ -409,6 +413,91 @@ public class BackupManagerTest {
}
@Test
public void delete() {
final AuthenticatedBackupUser backupUser = backupUser(TestRandomUtil.nextBytes(16), BackupTier.MEDIA);
final byte[] mediaId = TestRandomUtil.nextBytes(16);
final String backupMediaKey = "%s/%s/%s".formatted(
BackupManager.encodeBackupIdForCdn(backupUser),
BackupManager.MEDIA_DIRECTORY_NAME,
BackupManager.encodeForCdn(mediaId));
backupsDb.setMediaUsage(backupUser, new UsageInfo(100, 1000)).join();
when(remoteStorageManager.delete(backupMediaKey))
.thenReturn(CompletableFuture.completedFuture(7L));
when(remoteStorageManager.cdnNumber()).thenReturn(5);
backupManager.delete(backupUser, List.of(new BackupManager.StorageDescriptor(5, mediaId))).toCompletableFuture()
.join();
assertThat(backupsDb.getMediaUsage(backupUser).join().usageInfo())
.isEqualTo(new UsageInfo(93, 999));
}
@Test
public void deleteUnknownCdn() {
final AuthenticatedBackupUser backupUser = backupUser(TestRandomUtil.nextBytes(16), BackupTier.MEDIA);
when(remoteStorageManager.cdnNumber()).thenReturn(5);
assertThatThrownBy(() ->
backupManager.delete( backupUser, List.of(new BackupManager.StorageDescriptor(4, TestRandomUtil.nextBytes(15)))))
.isInstanceOf(StatusRuntimeException.class)
.matches(e -> ((StatusRuntimeException) e).getStatus().getCode() == Status.INVALID_ARGUMENT.getCode());
}
@Test
public void deletePartialFailure() {
final AuthenticatedBackupUser backupUser = backupUser(TestRandomUtil.nextBytes(16), BackupTier.MEDIA);
final List<BackupManager.StorageDescriptor> descriptors = new ArrayList<>();
long initialBytes = 0;
for (int i = 1; i <= 10; i++) {
final BackupManager.StorageDescriptor descriptor = new BackupManager.StorageDescriptor(5,
TestRandomUtil.nextBytes(15));
descriptors.add(descriptor);
final String backupMediaKey = "%s/%s/%s".formatted(
BackupManager.encodeBackupIdForCdn(backupUser),
BackupManager.MEDIA_DIRECTORY_NAME,
BackupManager.encodeForCdn(descriptor.key()));
initialBytes += i;
// fail 2 deletions, otherwise return the corresponding object's size as i
final CompletableFuture<Long> deleteResult =
i == 3 || i == 6
? CompletableFuture.failedFuture(new IOException("oh no"))
: CompletableFuture.completedFuture(Long.valueOf(i));
when(remoteStorageManager.delete(backupMediaKey)).thenReturn(deleteResult);
}
when(remoteStorageManager.cdnNumber()).thenReturn(5);
backupsDb.setMediaUsage(backupUser, new UsageInfo(initialBytes, 10)).join();
CompletableFutureTestUtil.assertFailsWithCause(IOException.class, backupManager.delete(backupUser, descriptors));
// 2 objects should have failed to be deleted
assertThat(backupsDb.getMediaUsage(backupUser).join().usageInfo())
.isEqualTo(new UsageInfo(9, 2));
}
@Test
public void alreadyDeleted() {
final AuthenticatedBackupUser backupUser = backupUser(TestRandomUtil.nextBytes(16), BackupTier.MEDIA);
final byte[] mediaId = TestRandomUtil.nextBytes(16);
final String backupMediaKey = "%s/%s/%s".formatted(
BackupManager.encodeBackupIdForCdn(backupUser),
BackupManager.MEDIA_DIRECTORY_NAME,
BackupManager.encodeForCdn(mediaId));
backupsDb.setMediaUsage(backupUser, new UsageInfo(100, 5)).join();
// Deletion doesn't remove anything
when(remoteStorageManager.delete(backupMediaKey)).thenReturn(CompletableFuture.completedFuture(0L));
when(remoteStorageManager.cdnNumber()).thenReturn(5);
backupManager.delete(backupUser, List.of(new BackupManager.StorageDescriptor(5, mediaId))).toCompletableFuture()
.join();
assertThat(backupsDb.getMediaUsage(backupUser).join().usageInfo())
.isEqualTo(new UsageInfo(100, 5));
}
private Map<String, AttributeValue> getBackupItem(final AuthenticatedBackupUser backupUser) {
return DYNAMO_DB_EXTENSION.getDynamoDbClient().getItem(GetItemRequest.builder()
.tableName(DynamoDbExtensionSchema.Tables.BACKUPS.tableName())

6
service/src/test/java/org/whispersystems/textsecuregcm/backup/BackupsDbTest.java
View File

@@ -50,14 +50,14 @@ public class BackupsDbTest {
backupsDb.addMessageBackup(backupUser).join();
int total = 0;
for (int i = 0; i < 5; i++) {
this.backupsDb.trackMedia(backupUser, i).join();
this.backupsDb.trackMedia(backupUser, 1, i).join();
total += i;
final BackupsDb.BackupDescription description = this.backupsDb.describeBackup(backupUser).join();
assertThat(description.mediaUsedSpace().get()).isEqualTo(total);
}
for (int i = 0; i < 5; i++) {
this.backupsDb.trackMedia(backupUser, -i).join();
this.backupsDb.trackMedia(backupUser, -1, -i).join();
total -= i;
final BackupsDb.BackupDescription description = this.backupsDb.describeBackup(backupUser).join();
assertThat(description.mediaUsedSpace().get()).isEqualTo(total);
@@ -70,7 +70,7 @@ public class BackupsDbTest {
testClock.pin(Instant.ofEpochSecond(5));
final AuthenticatedBackupUser backupUser = backupUser(TestRandomUtil.nextBytes(16), BackupTier.MEDIA);
if (mediaAlreadyExists) {
this.backupsDb.trackMedia(backupUser, 10).join();
this.backupsDb.trackMedia(backupUser, 1, 10).join();
}
backupsDb.setMediaUsage(backupUser, new UsageInfo( 113, 17)).join();
final BackupsDb.TimestampedUsageInfo info = backupsDb.getMediaUsage(backupUser).join();

2
service/src/test/java/org/whispersystems/textsecuregcm/backup/Cdn3BackupCredentialGeneratorTest.java
View File

@@ -22,7 +22,7 @@ public class Cdn3BackupCredentialGeneratorTest {
new SecretBytes(TestRandomUtil.nextBytes(32)),
"https://example.org/upload"));
final MessageBackupUploadDescriptor messageBackupUploadDescriptor = generator.generateUpload("subdir", "key");
final MessageBackupUploadDescriptor messageBackupUploadDescriptor = generator.generateUpload("subdir/key");
assertThat(messageBackupUploadDescriptor.signedUploadLocation()).isEqualTo("https://example.org/upload/backups");
assertThat(messageBackupUploadDescriptor.key()).isEqualTo("subdir/key");
assertThat(messageBackupUploadDescriptor.headers()).containsKey("Authorization");

14
service/src/test/java/org/whispersystems/textsecuregcm/backup/Cdn3RemoteStorageManagerTest.java
View File

@@ -12,6 +12,7 @@ import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.github.tomakehurst.wiremock.client.WireMock;
import com.github.tomakehurst.wiremock.junit5.WireMockExtension;
import io.dropwizard.testing.junit5.DropwizardExtensionsSupport;
import java.io.IOException;
@@ -234,6 +235,8 @@ public class Cdn3RemoteStorageManagerTest {
public void usage() throws JsonProcessingException {
wireMock.stubFor(get(urlPathEqualTo("/storage-manager/usage"))
.withQueryParam("prefix", equalTo("abc/"))
.withHeader(Cdn3RemoteStorageManager.CLIENT_ID_HEADER, equalTo("clientId"))
.withHeader(Cdn3RemoteStorageManager.CLIENT_SECRET_HEADER, equalTo("clientSecret"))
.willReturn(aResponse()
.withBody(SystemMapper.jsonMapper().writeValueAsString(new Cdn3RemoteStorageManager.UsageResponse(
17,
@@ -244,4 +247,15 @@ public class Cdn3RemoteStorageManagerTest {
assertThat(result.numObjects()).isEqualTo(17);
assertThat(result.bytesUsed()).isEqualTo(113);
}
@Test
public void delete() throws JsonProcessingException {
wireMock.stubFor(WireMock.delete(urlEqualTo("/storage-manager/backups/abc/def"))
.withHeader(Cdn3RemoteStorageManager.CLIENT_ID_HEADER, equalTo("clientId"))
.withHeader(Cdn3RemoteStorageManager.CLIENT_SECRET_HEADER, equalTo("clientSecret"))
.willReturn(aResponse()
.withBody(SystemMapper.jsonMapper().writeValueAsString(new Cdn3RemoteStorageManager.DeleteResponse(9L)))));
final long deleted = remoteStorageManager.delete("abc/def").toCompletableFuture().join();
assertThat(deleted).isEqualTo(9L);
}
}

25
service/src/test/java/org/whispersystems/textsecuregcm/controllers/ArchiveControllerTest.java
View File

@@ -478,4 +478,29 @@ public class ArchiveControllerTest {
assertThat(response.storedMediaObjects().get(0).mediaId()).isEqualTo(mediaId);
assertThat(response.cursor()).isEqualTo(returnedCursor.orElse(null));
}
@Test
public void delete() throws VerificationFailedException {
final BackupAuthCredentialPresentation presentation = backupAuthTestUtil.getPresentation(BackupTier.MEDIA,
backupKey, aci);
when(backupManager.authenticateBackupUser(any(), any()))
.thenReturn(CompletableFuture.completedFuture(
new AuthenticatedBackupUser(presentation.getBackupId(), BackupTier.MEDIA)));
final ArchiveController.DeleteMedia deleteRequest = new ArchiveController.DeleteMedia(
IntStream
.range(0, 100)
.mapToObj(i -> new ArchiveController.DeleteMedia.MediaToDelete(3, TestRandomUtil.nextBytes(15)))
.toList());
when(backupManager.delete(any(), any())).thenReturn(CompletableFuture.completedFuture(null));
final Response response = resources.getJerseyTest()
.target("v1/archives/media/delete")
.request()
.header("X-Signal-ZK-Auth", Base64.getEncoder().encodeToString(presentation.serialize()))
.header("X-Signal-ZK-Auth-Signature", "aaa")
.post(Entity.json(deleteRequest));
assertThat(response.getStatus()).isEqualTo(204);
}
}