diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/backup/BackupManager.java b/service/src/main/java/org/whispersystems/textsecuregcm/backup/BackupManager.java index e46742f91..3aeb5b68c 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/backup/BackupManager.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/backup/BackupManager.java @@ -8,6 +8,7 @@ package org.whispersystems.textsecuregcm.backup; import com.google.common.annotations.VisibleForTesting; import io.dropwizard.util.DataSize; import io.grpc.Status; +import io.grpc.StatusRuntimeException; import io.micrometer.core.instrument.DistributionSummary; import io.micrometer.core.instrument.Metrics; import io.micrometer.core.instrument.Tag; @@ -54,6 +55,7 @@ public class BackupManager { static final String MESSAGE_BACKUP_NAME = "messageBackup"; public static final long MAX_TOTAL_BACKUP_MEDIA_BYTES = DataSize.gibibytes(100).toBytes(); + public static final long MAX_MESSAGE_BACKUP_OBJECT_SIZE = DataSize.mebibytes(101).toBytes(); public static final long MAX_MEDIA_OBJECT_SIZE = DataSize.mebibytes(101).toBytes(); // If the last media usage recalculation is over MAX_QUOTA_STALENESS, force a recalculation before quota enforcement. diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/controllers/ArchiveController.java b/service/src/main/java/org/whispersystems/textsecuregcm/controllers/ArchiveController.java index 16ce7e450..7e0ff19bd 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/controllers/ArchiveController.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/controllers/ArchiveController.java @@ -516,12 +516,24 @@ public class ArchiveController { @Parameter(description = BackupAuthCredentialPresentationSignature.DESCRIPTION, schema = @Schema(implementation = String.class)) @NotNull - @HeaderParam(X_SIGNAL_ZK_AUTH_SIGNATURE) final BackupAuthCredentialPresentationSignature signature) { + @HeaderParam(X_SIGNAL_ZK_AUTH_SIGNATURE) final BackupAuthCredentialPresentationSignature signature, + + @Parameter(description = "The size of the message backup to upload in bytes") + @QueryParam("uploadLength") final Optional uploadLength) { if (account.isPresent()) { throw new BadRequestException("must not use authenticated connection for anonymous operations"); } return backupManager.authenticateBackupUser(presentation.presentation, signature.signature, userAgent) - .thenCompose(backupManager::createMessageBackupUploadDescriptor) + .thenCompose(backupUser -> { + final boolean oversize = uploadLength + .map(length -> length > BackupManager.MAX_MESSAGE_BACKUP_OBJECT_SIZE) + .orElse(false); + backupMetrics.updateMessageBackupSizeDistribution(backupUser, oversize, uploadLength); + if (oversize) { + throw new ClientErrorException("exceeded maximum uploadLength", Response.Status.REQUEST_ENTITY_TOO_LARGE); + } + return backupManager.createMessageBackupUploadDescriptor(backupUser); + }) .thenApply(result -> new UploadDescriptorResponse( result.cdn(), result.key(), diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/grpc/BackupsAnonymousGrpcService.java b/service/src/main/java/org/whispersystems/textsecuregcm/grpc/BackupsAnonymousGrpcService.java index 4666265fe..090c41e8d 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/grpc/BackupsAnonymousGrpcService.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/grpc/BackupsAnonymousGrpcService.java @@ -101,7 +101,18 @@ public class BackupsAnonymousGrpcService extends ReactorBackupsAnonymousGrpc.Bac public Mono getUploadForm(final GetUploadFormRequest request) { return authenticateBackupUserMono(request.getSignedPresentation()) .flatMap(backupUser -> switch (request.getUploadTypeCase()) { - case MESSAGES -> Mono.fromFuture(backupManager.createMessageBackupUploadDescriptor(backupUser)); + case MESSAGES -> { + final long uploadLength = request.getMessages().getUploadLength(); + final boolean oversize = uploadLength > BackupManager.MAX_MESSAGE_BACKUP_OBJECT_SIZE; + backupMetrics.updateMessageBackupSizeDistribution(backupUser, oversize, Optional.of(uploadLength)); + if (oversize) { + yield Mono.error(Status.FAILED_PRECONDITION + .withDescription("Exceeds max upload length") + .asRuntimeException()); + } + + yield Mono.fromFuture(backupManager.createMessageBackupUploadDescriptor(backupUser)); + } case MEDIA -> Mono.fromCompletionStage(backupManager.createTemporaryAttachmentUploadDescriptor(backupUser)); case UPLOADTYPE_NOT_SET -> Mono.error(Status.INVALID_ARGUMENT .withDescription("Must set upload_type") diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/metrics/BackupMetrics.java b/service/src/main/java/org/whispersystems/textsecuregcm/metrics/BackupMetrics.java index 5757b339a..88d463f95 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/metrics/BackupMetrics.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/metrics/BackupMetrics.java @@ -8,17 +8,21 @@ package org.whispersystems.textsecuregcm.metrics; import static org.whispersystems.textsecuregcm.metrics.MetricsUtil.name; import com.google.common.annotations.VisibleForTesting; +import io.micrometer.core.instrument.DistributionSummary; import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.Metrics; import io.micrometer.core.instrument.Tag; import io.micrometer.core.instrument.Tags; import org.signal.libsignal.zkgroup.backups.BackupCredentialType; +import org.whispersystems.textsecuregcm.auth.AuthenticatedBackupUser; import org.whispersystems.textsecuregcm.backup.CopyResult; +import java.util.Optional; public class BackupMetrics { private final static String COPY_MEDIA_COUNTER_NAME = name(BackupMetrics.class, "copyMedia"); private final static String GET_BACKUP_CREDENTIALS_NAME = name(BackupMetrics.class, "getBackupCredentials"); + private final static String MESSAGE_BACKUP_SIZE_NAME = name(BackupMetrics.class, "messageBackupSize"); private MeterRegistry registry; @@ -48,4 +52,19 @@ public class BackupMetrics { .increment(); } + public void updateMessageBackupSizeDistribution( + AuthenticatedBackupUser authenticatedBackupUser, + final boolean oversize, + final Optional backupLength) { + DistributionSummary.builder(MESSAGE_BACKUP_SIZE_NAME) + .publishPercentileHistogram(true) + .tags(Tags.of( + UserAgentTagUtil.getPlatformTag(authenticatedBackupUser.userAgent()), + Tag.of("tier", authenticatedBackupUser.backupLevel().name().toLowerCase()), + Tag.of("oversize", Boolean.toString(oversize)), + Tag.of("hasBackupLength", Boolean.toString(backupLength.isPresent())))) + .register(Metrics.globalRegistry) + .record(backupLength.orElse(0L)); + } + } diff --git a/service/src/main/proto/org/signal/chat/backups.proto b/service/src/main/proto/org/signal/chat/backups.proto index 5b409c637..b7507dfc0 100644 --- a/service/src/main/proto/org/signal/chat/backups.proto +++ b/service/src/main/proto/org/signal/chat/backups.proto @@ -212,6 +212,10 @@ service BackupsAnonymous { /** * Retrieve an upload form that can be used to perform a resumable upload + * + * Trying to request an upload form larger than the maximum supported upload + * size will return a PRECONDITION_FAILED error. The maximum upload size is + * subject to change. */ rpc GetUploadForm(GetUploadFormRequest) returns (GetUploadFormResponse) {} @@ -333,7 +337,9 @@ message RefreshResponse { message GetUploadFormRequest { SignedPresentation signed_presentation = 1; - message MessagesUploadType {} + message MessagesUploadType { + uint64 uploadLength = 1; + } message MediaUploadType {} oneof upload_type { /** diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/controllers/ArchiveControllerTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/controllers/ArchiveControllerTest.java index 5feb07c65..81c96c31d 100644 --- a/service/src/test/java/org/whispersystems/textsecuregcm/controllers/ArchiveControllerTest.java +++ b/service/src/test/java/org/whispersystems/textsecuregcm/controllers/ArchiveControllerTest.java @@ -574,6 +574,46 @@ public class ArchiveControllerTest { assertThat(response.getStatus()).isEqualTo(204); } + + static Stream messagesUploadForm() { + return Stream.of( + Arguments.of(Optional.empty(), true), + Arguments.of(Optional.of(BackupManager.MAX_MESSAGE_BACKUP_OBJECT_SIZE), true), + Arguments.of(Optional.of(BackupManager.MAX_MESSAGE_BACKUP_OBJECT_SIZE + 1), false) + ); + } + + @ParameterizedTest + @MethodSource + public void messagesUploadForm(Optional uploadLength, boolean expectSuccess) throws VerificationFailedException { + final BackupAuthCredentialPresentation presentation = + backupAuthTestUtil.getPresentation(BackupLevel.PAID, messagesBackupKey, aci); + when(backupManager.authenticateBackupUser(any(), any(), any())) + .thenReturn(CompletableFuture.completedFuture(backupUser(presentation.getBackupId(), BackupCredentialType.MESSAGES, BackupLevel.PAID))); + when(backupManager.createMessageBackupUploadDescriptor(any())) + .thenReturn(CompletableFuture.completedFuture( + new BackupUploadDescriptor(3, "abc", Map.of("k", "v"), "example.org"))); + + final WebTarget builder = resources.getJerseyTest().target("v1/archives/upload/form"); + final Response response = uploadLength + .map(length -> builder.queryParam("uploadLength", length)) + .orElse(builder) + .request() + .header("X-Signal-ZK-Auth", Base64.getEncoder().encodeToString(presentation.serialize())) + .header("X-Signal-ZK-Auth-Signature", "aaa") + .get(); + if (expectSuccess) { + assertThat(response.getStatus()).isEqualTo(200); + ArchiveController.UploadDescriptorResponse desc = response.readEntity(ArchiveController.UploadDescriptorResponse.class); + assertThat(desc.cdn()).isEqualTo(3); + assertThat(desc.key()).isEqualTo("abc"); + assertThat(desc.headers()).containsExactlyEntriesOf(Map.of("k", "v")); + assertThat(desc.signedUploadLocation()).isEqualTo("example.org"); + } else { + assertThat(response.getStatus()).isEqualTo(413); + } + } + @Test public void mediaUploadForm() throws VerificationFailedException { final BackupAuthCredentialPresentation presentation = diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/grpc/BackupsAnonymousGrpcServiceTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/grpc/BackupsAnonymousGrpcServiceTest.java index f925c0607..f7bfd3080 100644 --- a/service/src/test/java/org/whispersystems/textsecuregcm/grpc/BackupsAnonymousGrpcServiceTest.java +++ b/service/src/test/java/org/whispersystems/textsecuregcm/grpc/BackupsAnonymousGrpcServiceTest.java @@ -8,6 +8,7 @@ package org.whispersystems.textsecuregcm.grpc; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatExceptionOfType; import static org.assertj.core.api.Assertions.assertThatNoException; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.when; @@ -17,6 +18,7 @@ import io.grpc.Status; import io.grpc.StatusRuntimeException; import java.time.Clock; import java.util.Arrays; +import java.util.Base64; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -26,8 +28,14 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import java.util.stream.IntStream; +import java.util.stream.Stream; +import jakarta.ws.rs.client.WebTarget; +import jakarta.ws.rs.core.Response; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import org.junitpioneer.jupiter.cartesian.CartesianTest; import org.mockito.Mock; import org.signal.chat.backup.BackupsAnonymousGrpc; @@ -56,6 +64,7 @@ import org.whispersystems.textsecuregcm.backup.BackupAuthTestUtil; import org.whispersystems.textsecuregcm.backup.BackupManager; import org.whispersystems.textsecuregcm.backup.BackupUploadDescriptor; import org.whispersystems.textsecuregcm.backup.CopyResult; +import org.whispersystems.textsecuregcm.controllers.ArchiveController; import org.whispersystems.textsecuregcm.controllers.RateLimitExceededException; import org.whispersystems.textsecuregcm.metrics.BackupMetrics; import org.whispersystems.textsecuregcm.util.TestRandomUtil; @@ -294,6 +303,42 @@ class BackupsAnonymousGrpcServiceTest extends .isEqualTo(Status.RESOURCE_EXHAUSTED); } + static Stream messagesUploadForm() { + return Stream.of( + Arguments.of(Optional.empty(), true), + Arguments.of(Optional.of(BackupManager.MAX_MESSAGE_BACKUP_OBJECT_SIZE), true), + Arguments.of(Optional.of(BackupManager.MAX_MESSAGE_BACKUP_OBJECT_SIZE + 1), false) + ); + } + + @ParameterizedTest + @MethodSource + public void messagesUploadForm(Optional uploadLength, boolean expectSuccess) { + when(backupManager.createMessageBackupUploadDescriptor(any())) + .thenReturn(CompletableFuture.completedFuture( + new BackupUploadDescriptor(3, "abc", Map.of("k", "v"), "example.org"))); + final GetUploadFormRequest.MessagesUploadType.Builder builder = GetUploadFormRequest.MessagesUploadType.newBuilder(); + uploadLength.ifPresent(builder::setUploadLength); + final GetUploadFormRequest request = GetUploadFormRequest.newBuilder() + .setMessages(builder.build()) + .setSignedPresentation(signedPresentation(presentation)) + .build(); + if (expectSuccess) { + final GetUploadFormResponse uploadForm = unauthenticatedServiceStub().getUploadForm(request); + assertThat(uploadForm.getCdn()).isEqualTo(3); + assertThat(uploadForm.getKey()).isEqualTo("abc"); + assertThat(uploadForm.getHeadersMap()).containsExactlyEntriesOf(Map.of("k", "v")); + assertThat(uploadForm.getSignedUploadLocation()).isEqualTo("example.org"); + } else { + assertThatExceptionOfType(StatusRuntimeException.class) + .isThrownBy(() -> unauthenticatedServiceStub().getUploadForm(request)) + .extracting(StatusRuntimeException::getStatus) + .extracting(Status::getCode) + .isEqualTo(Status.FAILED_PRECONDITION.getCode()); + } + } + + @Test void readAuth() { when(backupManager.generateReadAuth(any(), eq(3))).thenReturn(Map.of("key", "value"));