diff --git a/service/config/sample.yml b/service/config/sample.yml index 98c7aa4bc..3880ca52a 100644 --- a/service/config/sample.yml +++ b/service/config/sample.yml @@ -231,17 +231,18 @@ messageCache: # Redis server configuration for message store cache cluster: configurationUri: redis://redis.example.com:6379/ +attachments: + maxUploadSizeInBytes: 1024 + gcpAttachments: # GCP Storage configuration domain: example.com email: user@example.cocm - maxSizeInBytes: 1024 pathPrefix: rsaSigningKey: secret://gcpAttachments.rsaSigningKey tus: uploadUri: https://example.org/upload userAuthenticationTokenSharedSecret: secret://tus.userAuthenticationTokenSharedSecret - maxSizeInBytes: 1024 apn: # Apple Push Notifications configuration sandbox: true diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerConfiguration.java b/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerConfiguration.java index 406a7f102..bfb2f8754 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerConfiguration.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerConfiguration.java @@ -18,6 +18,7 @@ import org.whispersystems.textsecuregcm.attachments.TusConfiguration; import org.whispersystems.textsecuregcm.configuration.ApnConfiguration; import org.whispersystems.textsecuregcm.configuration.AppleAppStoreConfiguration; import org.whispersystems.textsecuregcm.configuration.AppleDeviceCheckConfiguration; +import org.whispersystems.textsecuregcm.configuration.AttachmentsConfiguration; import org.whispersystems.textsecuregcm.configuration.AwsCredentialsProviderFactory; import org.whispersystems.textsecuregcm.configuration.BadgesConfiguration; import org.whispersystems.textsecuregcm.configuration.BraintreeConfiguration; @@ -122,6 +123,11 @@ public class WhisperServerConfiguration extends Configuration { @JsonProperty private DynamoDbTables dynamoDbTables; + @NotNull + @Valid + @JsonProperty + private AttachmentsConfiguration attachments; + @NotNull @Valid @JsonProperty @@ -412,6 +418,10 @@ public class WhisperServerConfiguration extends Configuration { return webSocket; } + public AttachmentsConfiguration getAttachments() { + return attachments; + } + public GcpAttachmentsConfiguration getGcpAttachmentsConfiguration() { return gcpAttachments; } @@ -610,4 +620,5 @@ public class WhisperServerConfiguration extends Configuration { public HlrLookupConfiguration getHlrLookupConfiguration() { return hlrLookup; } + } diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java b/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java index 770cbf84b..8de99a1bb 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java @@ -827,7 +827,6 @@ public class WhisperServerService extends Application ServerInterceptors.intercept(bindableService, // Note: interceptors run in the reverse order they are added; the remote deprecation filter // depends on the user-agent context so it has to come first here! @@ -997,7 +997,7 @@ public class WhisperServerService extends Application ServerInterceptors.intercept(bindableService, // Note: interceptors run in the reverse order they are added; the remote deprecation filter @@ -1100,8 +1100,8 @@ public class WhisperServerService extends Application maxSizeInBytes) { - throw new IllegalArgumentException("uploadLength " + uploadLength + " exceeds maximum " + maxSizeInBytes); - } final ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC); final CanonicalRequest canonicalRequest = canonicalRequestGenerator.createFor(key, now, uploadLength); return new Descriptor(getHeaderMap(canonicalRequest), getSignedUploadLocation(canonicalRequest)); } - @Override - public long maxUploadSizeInBytes() { - return this.maxSizeInBytes; - } - private String getSignedUploadLocation(@Nonnull CanonicalRequest canonicalRequest) { return "https://" + canonicalRequest.getDomain() + canonicalRequest.getResourcePath() + '?' + canonicalRequest.getCanonicalQuery() diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/attachments/TusAttachmentGenerator.java b/service/src/main/java/org/whispersystems/textsecuregcm/attachments/TusAttachmentGenerator.java index 467fb2bcc..b3f52ce35 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/attachments/TusAttachmentGenerator.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/attachments/TusAttachmentGenerator.java @@ -19,20 +19,14 @@ public class TusAttachmentGenerator implements AttachmentGenerator { private final JwtGenerator jwtGenerator; private final String tusUri; - private final long maxUploadSize; public TusAttachmentGenerator(final TusConfiguration cfg) { this.tusUri = cfg.uploadUri(); this.jwtGenerator = new JwtGenerator(cfg.userAuthenticationTokenSharedSecret().value(), Clock.systemUTC()); - this.maxUploadSize = cfg.maxSizeInBytes(); } @Override public Descriptor generateAttachment(final String key, final long uploadLength) { - if (uploadLength > maxUploadSize) { - throw new IllegalArgumentException("uploadLength " + uploadLength + " exceeds maximum " + maxUploadSize); - } - final String token = jwtGenerator.generateJwt(ATTACHMENTS, key, builder -> builder.withClaim(JwtGenerator.MAX_LENGTH_CLAIM_KEY, uploadLength)); final String b64Key = Base64.getEncoder().encodeToString(key.getBytes(StandardCharsets.UTF_8)); @@ -43,9 +37,4 @@ public class TusAttachmentGenerator implements AttachmentGenerator { ); return new Descriptor(headers, tusUri + "/" + ATTACHMENTS); } - - @Override - public long maxUploadSizeInBytes() { - return maxUploadSize; - } } diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/attachments/TusConfiguration.java b/service/src/main/java/org/whispersystems/textsecuregcm/attachments/TusConfiguration.java index a65ded517..e7c399209 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/attachments/TusConfiguration.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/attachments/TusConfiguration.java @@ -12,6 +12,4 @@ import org.whispersystems.textsecuregcm.util.ExactlySize; public record TusConfiguration( @ExactlySize(32) SecretBytes userAuthenticationTokenSharedSecret, - @NotEmpty String uploadUri, - @Positive long maxSizeInBytes -){} + @NotEmpty String uploadUri){} diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/backup/BackupManager.java b/service/src/main/java/org/whispersystems/textsecuregcm/backup/BackupManager.java index 1cac2ce86..d8cac5ae6 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/backup/BackupManager.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/backup/BackupManager.java @@ -155,28 +155,33 @@ public class BackupManager { * If successful, this also updates the TTL of the backup. * * @param backupUser an already ZK authenticated backup user + * @param uploadSize the maximum size of a message backup that can be uploaded with the returned form * @return the upload form * @throws BackupPermissionException if the credential does not have the correct level * @throws BackupWrongCredentialTypeException if the credential does not have the messages type */ public BackupUploadDescriptor createMessageBackupUploadDescriptor( - final AuthenticatedBackupUser backupUser) throws BackupPermissionException, BackupWrongCredentialTypeException { + final AuthenticatedBackupUser backupUser, + final long uploadSize) throws BackupPermissionException, BackupWrongCredentialTypeException { checkBackupLevel(backupUser, BackupLevel.FREE); checkBackupCredentialType(backupUser, BackupCredentialType.MESSAGES); // this could race with concurrent updates, but the only effect would be last-writer-wins on the timestamp backupsDb.addMessageBackup(backupUser).join(); - return cdn3BackupCredentialGenerator.generateUpload(cdnMessageBackupName(backupUser)); + return cdn3BackupCredentialGenerator.generateUpload(cdnMessageBackupName(backupUser), uploadSize); } - public BackupUploadDescriptor createTemporaryAttachmentUploadDescriptor(final AuthenticatedBackupUser backupUser) + public BackupUploadDescriptor createTemporaryAttachmentUploadDescriptor( + final AuthenticatedBackupUser backupUser, + final long uploadSize) throws RateLimitExceededException, BackupPermissionException, BackupWrongCredentialTypeException { checkBackupLevel(backupUser, BackupLevel.PAID); checkBackupCredentialType(backupUser, BackupCredentialType.MEDIA); rateLimiters.forDescriptor(RateLimiters.For.BACKUP_ATTACHMENT).validate(rateLimitKey(backupUser)); final String attachmentKey = AttachmentUtil.generateAttachmentKey(secureRandom); - final AttachmentGenerator.Descriptor descriptor = tusAttachmentGenerator.generateAttachment(attachmentKey, tusAttachmentGenerator.maxUploadSizeInBytes()); + final AttachmentGenerator.Descriptor descriptor = + tusAttachmentGenerator.generateAttachment(attachmentKey, uploadSize); return new BackupUploadDescriptor(3, attachmentKey, descriptor.headers(), descriptor.signedUploadLocation()); } @@ -194,9 +199,9 @@ public class BackupManager { final long maxTotalMediaSize = dynamicConfigurationManager.getConfiguration().getBackupConfiguration().maxTotalMediaSize(); - // Report that the backup is out of quota if it cannot store a max size media object + // Report that the backup is out of quota if it is within 100MiB of the limit final boolean quotaExhausted = storedBackupAttributes.bytesUsed() >= - (maxTotalMediaSize - tusAttachmentGenerator.maxUploadSizeInBytes()); + (maxTotalMediaSize - 1024 * 1024 * 100); final Tags tags = Tags.of( UserAgentTagUtil.getPlatformTag(backupUser.userAgent()), @@ -329,13 +334,14 @@ public class BackupManager { */ public CopyQuota getCopyQuota( final AuthenticatedBackupUser backupUser, - final List toCopy) + final List toCopy, + final long maximumSourceObjectSize) throws BackupWrongCredentialTypeException, BackupPermissionException, BackupInvalidArgumentException { checkBackupLevel(backupUser, BackupLevel.PAID); checkBackupCredentialType(backupUser, BackupCredentialType.MEDIA); for (CopyParameters copyParameters : toCopy) { - if (copyParameters.sourceLength() > tusAttachmentGenerator.maxUploadSizeInBytes() || copyParameters.sourceLength() < 0) { + if (copyParameters.sourceLength() > maximumSourceObjectSize || copyParameters.sourceLength() < 0) { throw new BackupInvalidArgumentException("Invalid sourceObject size"); } } @@ -729,10 +735,6 @@ public class BackupManager { .toFuture(); } - public long maxMessageBackupUploadSize() { - return tusAttachmentGenerator.maxUploadSizeInBytes(); - } - interface PresentationSignatureVerifier { Pair verifySignature(byte[] signature, ECPublicKey publicKey) throws BackupFailedZkAuthenticationException; diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/backup/Cdn3BackupCredentialGenerator.java b/service/src/main/java/org/whispersystems/textsecuregcm/backup/Cdn3BackupCredentialGenerator.java index 10c3df254..c0fb2c921 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/backup/Cdn3BackupCredentialGenerator.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/backup/Cdn3BackupCredentialGenerator.java @@ -21,21 +21,19 @@ public class Cdn3BackupCredentialGenerator { private final String tusUri; private final JwtGenerator jwtGenerator; - private final long maxUploadSize; public Cdn3BackupCredentialGenerator(final TusConfiguration cfg) { this.tusUri = cfg.uploadUri(); this.jwtGenerator = new JwtGenerator(cfg.userAuthenticationTokenSharedSecret().value(), Clock.systemUTC()); - this.maxUploadSize = cfg.maxSizeInBytes(); } - public BackupUploadDescriptor generateUpload(final String key) { + public BackupUploadDescriptor generateUpload(final String key, final long uploadSize) { if (key.isBlank()) { throw new IllegalArgumentException("Upload descriptors must have non-empty keys"); } final String token = jwtGenerator.generateJwt(CDN_PATH, key, builder -> builder - .withClaim(JwtGenerator.MAX_LENGTH_CLAIM_KEY, maxUploadSize) + .withClaim(JwtGenerator.MAX_LENGTH_CLAIM_KEY, uploadSize) .withClaim(JwtGenerator.SCOPE_CLAIM_KEY, "write")); final String b64Key = Base64.getEncoder().encodeToString(key.getBytes(StandardCharsets.UTF_8)); diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/configuration/AttachmentsConfiguration.java b/service/src/main/java/org/whispersystems/textsecuregcm/configuration/AttachmentsConfiguration.java new file mode 100644 index 000000000..f905c9abd --- /dev/null +++ b/service/src/main/java/org/whispersystems/textsecuregcm/configuration/AttachmentsConfiguration.java @@ -0,0 +1,10 @@ +/* + * Copyright 2026 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ +package org.whispersystems.textsecuregcm.configuration; + +import jakarta.validation.constraints.Positive; + +public record AttachmentsConfiguration(@Positive long maxUploadSizeInBytes) { +} diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/configuration/GcpAttachmentsConfiguration.java b/service/src/main/java/org/whispersystems/textsecuregcm/configuration/GcpAttachmentsConfiguration.java index 646eb0136..c85709a9d 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/configuration/GcpAttachmentsConfiguration.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/configuration/GcpAttachmentsConfiguration.java @@ -14,7 +14,6 @@ import org.whispersystems.textsecuregcm.configuration.secrets.SecretString; public record GcpAttachmentsConfiguration(@NotBlank String domain, @NotBlank String email, - @Min(1) int maxSizeInBytes, String pathPrefix, @NotNull SecretString rsaSigningKey) { @SuppressWarnings("unused") diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/controllers/ArchiveController.java b/service/src/main/java/org/whispersystems/textsecuregcm/controllers/ArchiveController.java index 959edd671..7d876a806 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/controllers/ArchiveController.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/controllers/ArchiveController.java @@ -102,17 +102,20 @@ public class ArchiveController { private final BackupAuthManager backupAuthManager; private final BackupManager backupManager; private final BackupMetrics backupMetrics; + private final long maxAttachmentSize; public ArchiveController( final AccountsManager accountsManager, final BackupAuthManager backupAuthManager, final BackupManager backupManager, - final BackupMetrics backupMetrics) { + final BackupMetrics backupMetrics, + final long maxAttachmentSize) { this.accountsManager = accountsManager; this.backupAuthManager = backupAuthManager; this.backupManager = backupManager; this.backupMetrics = backupMetrics; + this.maxAttachmentSize = maxAttachmentSize; } public record SetBackupIdRequest( @@ -566,8 +569,11 @@ public class ArchiveController { @Path("/upload/form") @Produces(MediaType.APPLICATION_JSON) @Operation( - summary = "Fetch message backup upload form", - description = "Retrieve an upload form that can be used to perform a resumable upload of a message backup.") + summary = "Fetch message backup upload form", description = """ + Retrieve an upload form that can be used to perform a resumable upload of a message backup. + + Uploads with the returned form will be limited to a maximum size of the provided uploadLength. + """) @ApiResponse(responseCode = "200", content = @Content(schema = @Schema(implementation = UploadDescriptorResponse.class))) @ApiResponse(responseCode = "429", description = "Rate limited.") @ApiResponse(responseCode = "413", description = "The provided uploadLength is larger than the maximum supported upload size. The maximum upload size is subject to change.") @@ -596,7 +602,7 @@ public class ArchiveController { backupManager.authenticateBackupUser(presentation.presentation, signature.signature, userAgent); final boolean oversize = uploadLength - .map(length -> length > backupManager.maxMessageBackupUploadSize()) + .map(length -> length > maxAttachmentSize) .orElse(false); backupMetrics.updateMessageBackupSizeDistribution(backupUser, oversize, uploadLength); @@ -604,7 +610,7 @@ public class ArchiveController { throw new ClientErrorException("exceeded maximum uploadLength", Response.Status.REQUEST_ENTITY_TOO_LARGE); } final BackupUploadDescriptor uploadDescriptor = - backupManager.createMessageBackupUploadDescriptor(backupUser); + backupManager.createMessageBackupUploadDescriptor(backupUser, uploadLength.orElse(maxAttachmentSize)); return new UploadDescriptorResponse( uploadDescriptor.cdn(), uploadDescriptor.key(), @@ -621,32 +627,42 @@ public class ArchiveController { Retrieve an upload form that can be used to perform a resumable upload of an attachment. After uploading, the attachment can be copied into the backup at PUT /archives/media/. - Like the account authenticated version at /attachments, the uploaded object is only temporary. + Like the account authenticated version at /attachments, the uploaded object is only temporary. Uploads with + the returned form will be limited to a maximum size of the provided uploadLength. """) @ApiResponse(responseCode = "200", content = @Content(schema = @Schema(implementation = UploadDescriptorResponse.class))) @ApiResponse(responseCode = "429", description = "Rate limited.") + @ApiResponse(responseCode = "413", description = "The provided uploadLength is larger than the maximum supported upload size. The maximum upload size is subject to change and is governed by `global.attachments.maxBytes`.") @ApiResponseZkAuth @ManagedAsync public UploadDescriptorResponse uploadTemporaryAttachment( @Auth final Optional account, @HeaderParam(HttpHeaders.USER_AGENT) final String userAgent, - @Parameter(description = BackupAuthCredentialPresentationHeader.DESCRIPTION, schema = @Schema(implementation = String.class)) @NotNull @HeaderParam(X_SIGNAL_ZK_AUTH) final ArchiveController.BackupAuthCredentialPresentationHeader presentation, @Parameter(description = BackupAuthCredentialPresentationSignature.DESCRIPTION, schema = @Schema(implementation = String.class)) @NotNull - @HeaderParam(X_SIGNAL_ZK_AUTH_SIGNATURE) final BackupAuthCredentialPresentationSignature signature) + @HeaderParam(X_SIGNAL_ZK_AUTH_SIGNATURE) final BackupAuthCredentialPresentationSignature signature, + + @Parameter(description = "The size of the temporary attachment to upload in bytes") + @QueryParam("uploadLength") final Optional uploadLength) throws RateLimitExceededException, BackupFailedZkAuthenticationException, BackupWrongCredentialTypeException, BackupPermissionException { if (account.isPresent()) { throw new BadRequestException("must not use authenticated connection for anonymous operations"); } final AuthenticatedBackupUser backupUser = backupManager.authenticateBackupUser(presentation.presentation, signature.signature, userAgent); + + final boolean oversize = uploadLength.map(length -> length > maxAttachmentSize).orElse(false); + if (oversize) { + throw new ClientErrorException("exceeded maximum uploadLength", Response.Status.REQUEST_ENTITY_TOO_LARGE); + } + final BackupUploadDescriptor uploadDescriptor = - backupManager.createTemporaryAttachmentUploadDescriptor(backupUser); + backupManager.createTemporaryAttachmentUploadDescriptor(backupUser, uploadLength.orElse(maxAttachmentSize)); return new UploadDescriptorResponse( uploadDescriptor.cdn(), uploadDescriptor.key(), @@ -741,7 +757,7 @@ public class ArchiveController { final AuthenticatedBackupUser backupUser = backupManager.authenticateBackupUser(presentation.presentation, signature.signature, userAgent); final BackupManager.CopyQuota copyQuota = - backupManager.getCopyQuota(backupUser, List.of(copyMediaRequest.toCopyParameters())); + backupManager.getCopyQuota(backupUser, List.of(copyMediaRequest.toCopyParameters()), maxAttachmentSize); final CopyResult copyResult = backupManager.copyToBackup(copyQuota).next() .blockOptional() .orElseThrow(() -> new IllegalStateException("Non empty copy request must return result")); @@ -844,7 +860,7 @@ public class ArchiveController { final Stream copyParams = copyMediaRequest.items().stream().map(CopyMediaRequest::toCopyParameters); final AuthenticatedBackupUser backupUser = backupManager.authenticateBackupUser(presentation.presentation, signature.signature, userAgent); - final BackupManager.CopyQuota copyQuota = backupManager.getCopyQuota(backupUser, copyParams.toList()); + final BackupManager.CopyQuota copyQuota = backupManager.getCopyQuota(backupUser, copyParams.toList(), maxAttachmentSize); final List copyResults = backupManager.copyToBackup(copyQuota) .doOnNext(result -> backupMetrics.updateCopyCounter(result, UserAgentTagUtil.getPlatformTag(userAgent))) .map(CopyMediaBatchResponse.Entry::fromCopyResult) diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/controllers/AttachmentControllerV4.java b/service/src/main/java/org/whispersystems/textsecuregcm/controllers/AttachmentControllerV4.java index 1233584e9..d90d3cd95 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/controllers/AttachmentControllerV4.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/controllers/AttachmentControllerV4.java @@ -45,6 +45,7 @@ public class AttachmentControllerV4 { private final ExperimentEnrollmentManager experimentEnrollmentManager; private final RateLimiter rateLimiter; + private final long maxUploadLength; private final Map attachmentGenerators; @@ -55,9 +56,11 @@ public class AttachmentControllerV4 { final RateLimiters rateLimiters, final GcsAttachmentGenerator gcsAttachmentGenerator, final TusAttachmentGenerator tusAttachmentGenerator, - final ExperimentEnrollmentManager experimentEnrollmentManager) { + final ExperimentEnrollmentManager experimentEnrollmentManager, + final long maxUploadLength) { this.rateLimiter = rateLimiters.getAttachmentLimiter(); this.experimentEnrollmentManager = experimentEnrollmentManager; + this.maxUploadLength = maxUploadLength; this.secureRandom = new SecureRandom(); this.attachmentGenerators = Map.of( 2, gcsAttachmentGenerator, @@ -88,17 +91,16 @@ public class AttachmentControllerV4 { @Parameter(description = "The size of the attachment to upload in bytes") @QueryParam("uploadLength") final @Valid Optional<@Positive Long> maybeUploadLength) throws RateLimitExceededException { + final long uploadLength = maybeUploadLength.orElse(maxUploadLength); + if (uploadLength > maxUploadLength) { + throw new ClientErrorException("exceeded maximum uploadLength", Response.Status.REQUEST_ENTITY_TOO_LARGE); + } + rateLimiter.validate(auth.accountIdentifier()); + final String key = AttachmentUtil.generateAttachmentKey(secureRandom); final boolean useCdn3 = this.experimentEnrollmentManager.isEnrolled(auth.accountIdentifier(), AttachmentUtil.CDN3_EXPERIMENT_NAME); int cdn = useCdn3 ? 3 : 2; - final AttachmentGenerator attachmentGenerator = this.attachmentGenerators.get(cdn); - final long uploadLength = maybeUploadLength.orElse(attachmentGenerator.maxUploadSizeInBytes()); - if (uploadLength > attachmentGenerator.maxUploadSizeInBytes()) { - throw new ClientErrorException("exceeded maximum uploadLength", Response.Status.REQUEST_ENTITY_TOO_LARGE); - } - - rateLimiter.validate(auth.accountIdentifier()); - final AttachmentGenerator.Descriptor descriptor = attachmentGenerator.generateAttachment(key, uploadLength); + final AttachmentGenerator.Descriptor descriptor = this.attachmentGenerators.get(cdn).generateAttachment(key, uploadLength); return new AttachmentDescriptorV3(cdn, key, descriptor.headers(), descriptor.signedUploadLocation()); } diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/grpc/AttachmentsGrpcService.java b/service/src/main/java/org/whispersystems/textsecuregcm/grpc/AttachmentsGrpcService.java index 8f86ac509..d0871f275 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/grpc/AttachmentsGrpcService.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/grpc/AttachmentsGrpcService.java @@ -27,6 +27,7 @@ public class AttachmentsGrpcService extends SimpleAttachmentsGrpc.AttachmentsImp private final ExperimentEnrollmentManager experimentEnrollmentManager; private final RateLimiter rateLimiter; + private final long maxUploadLength; private final Map attachmentGenerators; private final SecureRandom secureRandom; @@ -34,9 +35,11 @@ public class AttachmentsGrpcService extends SimpleAttachmentsGrpc.AttachmentsImp final ExperimentEnrollmentManager experimentEnrollmentManager, final RateLimiters rateLimiters, final GcsAttachmentGenerator gcsAttachmentGenerator, - final TusAttachmentGenerator tusAttachmentGenerator) { + final TusAttachmentGenerator tusAttachmentGenerator, + final long maxUploadLength) { this.experimentEnrollmentManager = experimentEnrollmentManager; this.rateLimiter = rateLimiters.getAttachmentLimiter(); + this.maxUploadLength = maxUploadLength; this.secureRandom = new SecureRandom(); this.attachmentGenerators = Map.of( 2, gcsAttachmentGenerator, @@ -45,20 +48,20 @@ public class AttachmentsGrpcService extends SimpleAttachmentsGrpc.AttachmentsImp @Override public GetUploadFormResponse getUploadForm(final GetUploadFormRequest request) throws RateLimitExceededException { - final AuthenticatedDevice auth = AuthenticationUtil.requireAuthenticatedDevice(); - final String key = AttachmentUtil.generateAttachmentKey(secureRandom); - final boolean useCdn3 = this.experimentEnrollmentManager.isEnrolled(auth.accountIdentifier(), - AttachmentUtil.CDN3_EXPERIMENT_NAME); - final int cdn = useCdn3 ? 3 : 2; - final AttachmentGenerator attachmentGenerator = this.attachmentGenerators.get(cdn); - if (request.getUploadLength() > attachmentGenerator.maxUploadSizeInBytes()) { + if (request.getUploadLength() > maxUploadLength) { return GetUploadFormResponse.newBuilder() .setExceedsMaxUploadLength(FailedPrecondition.getDefaultInstance()) .build(); } - + final AuthenticatedDevice auth = AuthenticationUtil.requireAuthenticatedDevice(); rateLimiter.validate(auth.accountIdentifier()); - final AttachmentGenerator.Descriptor descriptor = attachmentGenerator.generateAttachment(key, request.getUploadLength()); + + final String key = AttachmentUtil.generateAttachmentKey(secureRandom); + final boolean useCdn3 = this.experimentEnrollmentManager.isEnrolled(auth.accountIdentifier(), + AttachmentUtil.CDN3_EXPERIMENT_NAME); + final int cdn = useCdn3 ? 3 : 2; + final AttachmentGenerator.Descriptor descriptor = + this.attachmentGenerators.get(cdn).generateAttachment(key, request.getUploadLength()); return GetUploadFormResponse.newBuilder().setUploadForm(UploadForm.newBuilder() .setCdn(cdn) .setKey(key) diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/grpc/BackupsAnonymousGrpcService.java b/service/src/main/java/org/whispersystems/textsecuregcm/grpc/BackupsAnonymousGrpcService.java index 98e91e852..6afba8b5b 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/grpc/BackupsAnonymousGrpcService.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/grpc/BackupsAnonymousGrpcService.java @@ -32,6 +32,7 @@ import org.signal.chat.backup.SetPublicKeyRequest; import org.signal.chat.backup.SetPublicKeyResponse; import org.signal.chat.backup.SignedPresentation; import org.signal.chat.backup.SimpleBackupsAnonymousGrpc; +import org.signal.chat.common.UploadForm; import org.signal.chat.errors.FailedPrecondition; import org.signal.chat.errors.FailedZkAuthentication; import org.signal.libsignal.protocol.InvalidKeyException; @@ -60,10 +61,12 @@ public class BackupsAnonymousGrpcService extends SimpleBackupsAnonymousGrpc.Back private final BackupManager backupManager; private final BackupMetrics backupMetrics; + private final long maxAttachmentSize; - public BackupsAnonymousGrpcService(final BackupManager backupManager, final BackupMetrics backupMetrics) { + public BackupsAnonymousGrpcService(final BackupManager backupManager, final BackupMetrics backupMetrics, final long maxAttachmentSize) { this.backupManager = backupManager; this.backupMetrics = backupMetrics; + this.maxAttachmentSize = maxAttachmentSize; } @Override @@ -178,35 +181,29 @@ public class BackupsAnonymousGrpcService extends SimpleBackupsAnonymousGrpc.Back .setFailedAuthentication(FailedZkAuthentication.newBuilder().setDescription(e.getMessage()).build()) .build(); } - final GetUploadFormResponse.Builder builder = GetUploadFormResponse.newBuilder(); - switch (request.getUploadTypeCase()) { - case MESSAGES -> { - final long uploadLength = request.getMessages().getUploadLength(); - final boolean oversize = uploadLength > backupManager.maxMessageBackupUploadSize(); - backupMetrics.updateMessageBackupSizeDistribution(backupUser, oversize, Optional.of(uploadLength)); - if (oversize) { - builder.setExceedsMaxUploadLength(FailedPrecondition.getDefaultInstance()); - } else { - final BackupUploadDescriptor uploadDescriptor = backupManager.createMessageBackupUploadDescriptor(backupUser); - builder.setUploadForm(builder.getUploadFormBuilder() - .setCdn(uploadDescriptor.cdn()) - .setKey(uploadDescriptor.key()) - .setSignedUploadLocation(uploadDescriptor.signedUploadLocation()) - .putAllHeaders(uploadDescriptor.headers())).build(); - } + final long uploadLength = request.getUploadLength(); + if (uploadLength > maxAttachmentSize) { + if (request.getUploadTypeCase() == GetUploadFormRequest.UploadTypeCase.MESSAGES) { + backupMetrics.updateMessageBackupSizeDistribution(backupUser, true, Optional.of(uploadLength)); } - case MEDIA -> { - final BackupUploadDescriptor uploadDescriptor = backupManager.createTemporaryAttachmentUploadDescriptor( - backupUser); - builder.setUploadForm(builder.getUploadFormBuilder() + return GetUploadFormResponse.newBuilder().setExceedsMaxUploadLength(FailedPrecondition.getDefaultInstance()).build(); + } + + final BackupUploadDescriptor uploadDescriptor = switch (request.getUploadTypeCase()) { + case MESSAGES -> { + backupMetrics.updateMessageBackupSizeDistribution(backupUser, false, Optional.of(uploadLength)); + yield backupManager.createMessageBackupUploadDescriptor(backupUser, uploadLength); + } + case MEDIA -> backupManager.createTemporaryAttachmentUploadDescriptor(backupUser, uploadLength); + case UPLOADTYPE_NOT_SET -> throw GrpcExceptions.fieldViolation("upload_type", "Must set upload_type"); + }; + return GetUploadFormResponse.newBuilder() + .setUploadForm(UploadForm.newBuilder() .setCdn(uploadDescriptor.cdn()) .setKey(uploadDescriptor.key()) .setSignedUploadLocation(uploadDescriptor.signedUploadLocation()) - .putAllHeaders(uploadDescriptor.headers())).build(); - } - case UPLOADTYPE_NOT_SET -> throw GrpcExceptions.fieldViolation("upload_type", "Must set upload_type"); - } - return builder.build(); + .putAllHeaders(uploadDescriptor.headers())) + .build(); } @Override @@ -221,7 +218,7 @@ public class BackupsAnonymousGrpcService extends SimpleBackupsAnonymousGrpc.Back // uint32 in proto, make sure it fits in a signed int fromUnsignedExact(item.getObjectLength()), new MediaEncryptionParameters(item.getEncryptionKey().toByteArray(), item.getHmacKey().toByteArray()), - item.getMediaId().toByteArray())).toList()); + item.getMediaId().toByteArray())).toList(), maxAttachmentSize); } catch (BackupFailedZkAuthenticationException e) { return JdkFlowAdapter.publisherToFlowPublisher(Mono.just(CopyMediaResponse .newBuilder() diff --git a/service/src/main/proto/org/signal/chat/backups.proto b/service/src/main/proto/org/signal/chat/backups.proto index e06ae7645..0e0ef00b9 100644 --- a/service/src/main/proto/org/signal/chat/backups.proto +++ b/service/src/main/proto/org/signal/chat/backups.proto @@ -349,9 +349,7 @@ message RefreshResponse { message GetUploadFormRequest { SignedPresentation signed_presentation = 1; - message MessagesUploadType { - uint64 upload_length = 1; - } + message MessagesUploadType {} message MediaUploadType {} oneof upload_type { // Retrieve an upload form that can be used to perform a resumable upload of @@ -365,6 +363,10 @@ message GetUploadFormRequest { // Behaves identically to the account authenticated version at /attachments. MediaUploadType media = 3; } + + // The length of the attachment for the requested upload form. Uploads + // performed with this form will be limited to the provided length. + uint64 uploadLength = 4 [(require.range) = {min: 1}]; } message GetUploadFormResponse { oneof outcome { @@ -377,7 +379,8 @@ message GetUploadFormResponse { errors.FailedZkAuthentication failed_authentication = 2 [(tag.reason) = "failed_authentication"]; // The request size was larger than the maximum supported upload size. The - // maximum upload size is subject to change. + // maximum upload size is subject to change and is governed by + // `global.attachments.maxBytes` errors.FailedPrecondition exceeds_max_upload_length = 3 [(tag.reason) = "oversize_upload"]; } } diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/backup/BackupManagerTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/backup/BackupManagerTest.java index c92599bcc..639807840 100644 --- a/service/src/test/java/org/whispersystems/textsecuregcm/backup/BackupManagerTest.java +++ b/service/src/test/java/org/whispersystems/textsecuregcm/backup/BackupManagerTest.java @@ -152,8 +152,6 @@ public class BackupManagerTest { when(dynamicConfiguration.getBackupConfiguration()).thenReturn(backupConfiguration); when(dynamicConfigurationManager.getConfiguration()).thenReturn(dynamicConfiguration); - when(tusAttachmentGenerator.maxUploadSizeInBytes()).thenReturn(MAX_UPLOAD_SIZE); - this.backupManager = new BackupManager( backupsDb, backupAuthTestUtil.params, @@ -224,9 +222,9 @@ public class BackupManagerTest { final AuthenticatedBackupUser backupUser = backupUser(TestRandomUtil.nextBytes(16), BackupCredentialType.MESSAGES, backupLevel); - backupManager.createMessageBackupUploadDescriptor(backupUser); + backupManager.createMessageBackupUploadDescriptor(backupUser, 17); verify(tusCredentialGenerator, times(1)) - .generateUpload("%s/%s".formatted(backupUser.backupDir(), BackupManager.MESSAGE_BACKUP_NAME)); + .generateUpload("%s/%s".formatted(backupUser.backupDir(), BackupManager.MESSAGE_BACKUP_NAME), 17); final BackupManager.BackupInfo info = backupManager.backupInfo(backupUser); assertThat(info.backupSubdir()).isEqualTo(backupUser.backupDir()).isNotBlank(); @@ -247,7 +245,7 @@ public class BackupManagerTest { final AuthenticatedBackupUser backupUser = backupUser(TestRandomUtil.nextBytes(16), BackupCredentialType.MEDIA, backupLevel); assertThatExceptionOfType(BackupWrongCredentialTypeException.class) - .isThrownBy(() -> backupManager.createMessageBackupUploadDescriptor(backupUser)); + .isThrownBy(() -> backupManager.createMessageBackupUploadDescriptor(backupUser, MAX_UPLOAD_SIZE)); } @Test @@ -256,21 +254,21 @@ public class BackupManagerTest { doThrow(new RateLimitExceededException(null)) .when(mediaUploadLimiter).validate(eq(BackupManager.rateLimitKey(backupUser))); assertThatExceptionOfType(RateLimitExceededException.class) - .isThrownBy(() -> backupManager.createTemporaryAttachmentUploadDescriptor(backupUser)); + .isThrownBy(() -> backupManager.createTemporaryAttachmentUploadDescriptor(backupUser, MAX_UPLOAD_SIZE)); } @Test public void createTemporaryMediaAttachmentWrongTier() { final AuthenticatedBackupUser backupUser = backupUser(TestRandomUtil.nextBytes(16), BackupCredentialType.MEDIA, BackupLevel.FREE); assertThatExceptionOfType(BackupPermissionException.class) - .isThrownBy(() -> backupManager.createTemporaryAttachmentUploadDescriptor(backupUser)); + .isThrownBy(() -> backupManager.createTemporaryAttachmentUploadDescriptor(backupUser, MAX_UPLOAD_SIZE)); } @Test public void createTemporaryMediaAttachmentWrongCredentialType() { final AuthenticatedBackupUser backupUser = backupUser(TestRandomUtil.nextBytes(16), BackupCredentialType.MESSAGES, BackupLevel.PAID); assertThatExceptionOfType(BackupWrongCredentialTypeException.class) - .isThrownBy(() -> backupManager.createTemporaryAttachmentUploadDescriptor(backupUser)); + .isThrownBy(() -> backupManager.createTemporaryAttachmentUploadDescriptor(backupUser, MAX_UPLOAD_SIZE)); } @ParameterizedTest @@ -283,7 +281,7 @@ public class BackupManagerTest { // create backup at t=tstart testClock.pin(tstart); - backupManager.createMessageBackupUploadDescriptor(backupUser); + backupManager.createMessageBackupUploadDescriptor(backupUser, MAX_UPLOAD_SIZE); // refresh at t=tnext testClock.pin(tnext); @@ -305,11 +303,11 @@ public class BackupManagerTest { // create backup at t=tstart testClock.pin(tstart); - backupManager.createMessageBackupUploadDescriptor(backupUser); + backupManager.createMessageBackupUploadDescriptor(backupUser, MAX_UPLOAD_SIZE); // create again at t=tnext testClock.pin(tnext); - backupManager.createMessageBackupUploadDescriptor(backupUser); + backupManager.createMessageBackupUploadDescriptor(backupUser, MAX_UPLOAD_SIZE); checkExpectedExpirations( tnext.truncatedTo(ChronoUnit.DAYS), @@ -478,7 +476,7 @@ public class BackupManagerTest { .thenReturn(slow); final ArrayBlockingQueue copyResults = new ArrayBlockingQueue<>(100); final CompletableFuture future = backupManager - .copyToBackup(backupManager.getCopyQuota(backupUser, toCopy)) + .copyToBackup(backupManager.getCopyQuota(backupUser, toCopy, MAX_UPLOAD_SIZE)) .doOnNext(copyResults::add).then().toFuture(); for (int i = 0; i < slowIndex; i++) { @@ -525,7 +523,7 @@ public class BackupManagerTest { new CopyParameters(3, "missing", 200, COPY_ENCRYPTION_PARAM, TestRandomUtil.nextBytes(15)), new CopyParameters(3, "badlength", 300, COPY_ENCRYPTION_PARAM, TestRandomUtil.nextBytes(15))); - when(tusCredentialGenerator.generateUpload(any())) + when(tusCredentialGenerator.generateUpload(any(), anyLong())) .thenReturn(new BackupUploadDescriptor(3, "", Collections.emptyMap(), "")); when(remoteStorageManager.copy(eq(3), eq("success"), eq(100), any(), any())) .thenReturn(CompletableFuture.completedFuture(null)); @@ -534,7 +532,8 @@ public class BackupManagerTest { when(remoteStorageManager.copy(eq(3), eq("badlength"), eq(300), any(), any())) .thenReturn(CompletableFuture.failedFuture(new InvalidLengthException(""))); - final List results = backupManager.copyToBackup(backupManager.getCopyQuota(backupUser, toCopy)) + final List results = backupManager + .copyToBackup(backupManager.getCopyQuota(backupUser, toCopy, MAX_UPLOAD_SIZE)) .collectList().block(); assertThat(results).hasSize(3); @@ -871,7 +870,7 @@ public class BackupManagerTest { .toList(); for (int i = 0; i < backupUsers.size(); i++) { testClock.pin(days(i)); - backupManager.createMessageBackupUploadDescriptor(backupUsers.get(i)); + backupManager.createMessageBackupUploadDescriptor(backupUsers.get(i), MAX_UPLOAD_SIZE); } // set of backup-id hashes that should be expired (initially t=0) @@ -906,11 +905,11 @@ public class BackupManagerTest { // refreshed media timestamp at t=5 testClock.pin(days(5)); - backupManager.createMessageBackupUploadDescriptor(backupUser(backupId, BackupCredentialType.MESSAGES, BackupLevel.PAID)); + backupManager.createMessageBackupUploadDescriptor(backupUser(backupId, BackupCredentialType.MESSAGES, BackupLevel.PAID), MAX_UPLOAD_SIZE); // refreshed messages timestamp at t=6 testClock.pin(days(6)); - backupManager.createMessageBackupUploadDescriptor(backupUser(backupId, BackupCredentialType.MESSAGES, BackupLevel.FREE)); + backupManager.createMessageBackupUploadDescriptor(backupUser(backupId, BackupCredentialType.MESSAGES, BackupLevel.FREE), MAX_UPLOAD_SIZE); Function> getExpired = time -> backupManager .getExpiredBackups(1, Schedulers.immediate(), time) @@ -931,7 +930,7 @@ public class BackupManagerTest { @EnumSource(mode = EnumSource.Mode.INCLUDE, names = {"MEDIA", "ALL"}) public void expireBackup(ExpiredBackup.ExpirationType expirationType) throws BackupException { final AuthenticatedBackupUser backupUser = backupUser(TestRandomUtil.nextBytes(16), BackupCredentialType.MESSAGES, BackupLevel.PAID); - backupManager.createMessageBackupUploadDescriptor(backupUser); + backupManager.createMessageBackupUploadDescriptor(backupUser, MAX_UPLOAD_SIZE); final String expectedPrefixToDelete = switch (expirationType) { case ALL -> backupUser.backupDir(); @@ -975,7 +974,7 @@ public class BackupManagerTest { @Test public void deleteBackupPaginated() throws BackupException { final AuthenticatedBackupUser backupUser = backupUser(TestRandomUtil.nextBytes(16), BackupCredentialType.MESSAGES, BackupLevel.PAID); - backupManager.createMessageBackupUploadDescriptor(backupUser); + backupManager.createMessageBackupUploadDescriptor(backupUser, MAX_UPLOAD_SIZE); final ExpiredBackup expiredBackup = expiredBackup(ExpiredBackup.ExpirationType.MEDIA, backupUser); final String mediaPrefix = expiredBackup.prefixToDelete() + "/"; @@ -1033,21 +1032,24 @@ public class BackupManagerTest { } private CopyResult copyError(final AuthenticatedBackupUser backupUser, Throwable copyException) throws BackupException { - when(tusCredentialGenerator.generateUpload(any())) + when(tusCredentialGenerator.generateUpload(any(), anyLong())) .thenReturn(new BackupUploadDescriptor(3, "def", Collections.emptyMap(), "")); when(remoteStorageManager.copy(eq(3), eq(COPY_PARAM.sourceKey()), eq(COPY_PARAM.sourceLength()), any(), any())) .thenReturn(CompletableFuture.failedFuture(copyException)); - return backupManager.copyToBackup(backupManager.getCopyQuota(backupUser, List.of(COPY_PARAM))).single().block(); + return backupManager + .copyToBackup(backupManager.getCopyQuota(backupUser, List.of(COPY_PARAM), MAX_UPLOAD_SIZE)) + .single().block(); } private CopyResult copy(final AuthenticatedBackupUser backupUser) throws BackupException { - when(tusCredentialGenerator.generateUpload(any())) + when(tusCredentialGenerator.generateUpload(any(), anyLong())) .thenReturn(new BackupUploadDescriptor(3, "def", Collections.emptyMap(), "")); - when(tusCredentialGenerator.generateUpload(any())) + when(tusCredentialGenerator.generateUpload(any(), anyLong())) .thenReturn(new BackupUploadDescriptor(3, "def", Collections.emptyMap(), "")); when(remoteStorageManager.copy(eq(3), eq(COPY_PARAM.sourceKey()), eq(COPY_PARAM.sourceLength()), any(), any())) .thenReturn(CompletableFuture.completedFuture(null)); - return backupManager.copyToBackup(backupManager.getCopyQuota(backupUser, List.of(COPY_PARAM))).single().block(); + return backupManager.copyToBackup(backupManager.getCopyQuota(backupUser, List.of(COPY_PARAM), MAX_UPLOAD_SIZE)) + .single().block(); } private static ExpiredBackup expiredBackup(final ExpiredBackup.ExpirationType expirationType, diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/backup/Cdn3BackupCredentialGeneratorTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/backup/Cdn3BackupCredentialGeneratorTest.java index adf673f23..7e13c64eb 100644 --- a/service/src/test/java/org/whispersystems/textsecuregcm/backup/Cdn3BackupCredentialGeneratorTest.java +++ b/service/src/test/java/org/whispersystems/textsecuregcm/backup/Cdn3BackupCredentialGeneratorTest.java @@ -28,10 +28,9 @@ public class Cdn3BackupCredentialGeneratorTest { public void uploadGenerator() { final Cdn3BackupCredentialGenerator generator = new Cdn3BackupCredentialGenerator(new TusConfiguration( new SecretBytes(SECRET), - "https://example.org/upload", - MAX_UPLOAD_LENGTH)); + "https://example.org/upload")); - final BackupUploadDescriptor messageBackupUploadDescriptor = generator.generateUpload("subdir/key"); + final BackupUploadDescriptor messageBackupUploadDescriptor = generator.generateUpload("subdir/key", MAX_UPLOAD_LENGTH); assertThat(messageBackupUploadDescriptor.signedUploadLocation()).isEqualTo("https://example.org/upload/backups"); assertThat(messageBackupUploadDescriptor.key()).isEqualTo("subdir/key"); assertThat(messageBackupUploadDescriptor.headers()).containsKey("Authorization"); @@ -50,8 +49,7 @@ public class Cdn3BackupCredentialGeneratorTest { public void readCredential() { final Cdn3BackupCredentialGenerator generator = new Cdn3BackupCredentialGenerator(new TusConfiguration( new SecretBytes(SECRET), - "https://example.org/upload", - MAX_UPLOAD_LENGTH)); + "https://example.org/upload")); final Map headers = generator.readHeaders("subdir"); assertThat(headers).containsKey("Authorization"); diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/controllers/ArchiveControllerTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/controllers/ArchiveControllerTest.java index 2a0754d00..fc309cc36 100644 --- a/service/src/test/java/org/whispersystems/textsecuregcm/controllers/ArchiveControllerTest.java +++ b/service/src/test/java/org/whispersystems/textsecuregcm/controllers/ArchiveControllerTest.java @@ -7,6 +7,7 @@ package org.whispersystems.textsecuregcm.controllers; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; @@ -73,6 +74,7 @@ import org.whispersystems.textsecuregcm.backup.BackupManager; import org.whispersystems.textsecuregcm.backup.BackupNotFoundException; import org.whispersystems.textsecuregcm.backup.BackupPermissionException; import org.whispersystems.textsecuregcm.backup.BackupUploadDescriptor; +import org.whispersystems.textsecuregcm.backup.BackupWrongCredentialTypeException; import org.whispersystems.textsecuregcm.backup.CopyResult; import org.whispersystems.textsecuregcm.entities.RemoteAttachment; import org.whispersystems.textsecuregcm.mappers.BackupExceptionMapper; @@ -90,7 +92,7 @@ import reactor.core.publisher.Flux; @ExtendWith(DropwizardExtensionsSupport.class) public class ArchiveControllerTest { - private static final long MAX_MESSAGE_BACKUP_OBJECT_SIZE = 1000L; + private static final long MAX_ATTACHMENT_SIZE = 1000L; private static final AccountsManager accountsManager = mock(AccountsManager.class); private static final BackupAuthManager backupAuthManager = mock(BackupAuthManager.class); private static final BackupManager backupManager = mock(BackupManager.class); @@ -106,7 +108,7 @@ public class ArchiveControllerTest { .addProvider(new RateLimitExceededExceptionMapper()) .setMapper(SystemMapper.jsonMapper()) .setTestContainerFactory(new GrizzlyWebTestContainerFactory()) - .addResource(new ArchiveController(accountsManager, backupAuthManager, backupManager, new BackupMetrics())) + .addResource(new ArchiveController(accountsManager, backupAuthManager, backupManager, new BackupMetrics(), MAX_ATTACHMENT_SIZE)) .build(); private final UUID aci = UUID.randomUUID(); @@ -118,8 +120,6 @@ public class ArchiveControllerTest { reset(backupAuthManager); reset(backupManager); - when(backupManager.maxMessageBackupUploadSize()).thenReturn(MAX_MESSAGE_BACKUP_OBJECT_SIZE); - when(accountsManager.getByAccountIdentifier(AuthHelper.VALID_UUID)) .thenReturn(Optional.of(AuthHelper.VALID_ACCOUNT)); } @@ -615,8 +615,8 @@ public class ArchiveControllerTest { static Stream messagesUploadForm() { return Stream.of( Arguments.of(Optional.empty(), true), - Arguments.of(Optional.of(MAX_MESSAGE_BACKUP_OBJECT_SIZE), true), - Arguments.of(Optional.of(MAX_MESSAGE_BACKUP_OBJECT_SIZE + 1), false) + Arguments.of(Optional.of(MAX_ATTACHMENT_SIZE), true), + Arguments.of(Optional.of(MAX_ATTACHMENT_SIZE + 1), false) ); } @@ -627,7 +627,7 @@ public class ArchiveControllerTest { backupAuthTestUtil.getPresentation(BackupLevel.PAID, messagesBackupKey, aci); when(backupManager.authenticateBackupUser(any(), any(), any())) .thenReturn(backupUser(presentation.getBackupId(), BackupCredentialType.MESSAGES, BackupLevel.PAID)); - when(backupManager.createMessageBackupUploadDescriptor(any())) + when(backupManager.createMessageBackupUploadDescriptor(any(), anyLong())) .thenReturn(new BackupUploadDescriptor(3, "abc", Map.of("k", "v"), "example.org")); final WebTarget builder = resources.getJerseyTest().target("v1/archives/upload/form"); @@ -641,36 +641,59 @@ public class ArchiveControllerTest { if (expectSuccess) { assertThat(response.getStatus()).isEqualTo(200); ArchiveController.UploadDescriptorResponse desc = response.readEntity(ArchiveController.UploadDescriptorResponse.class); - assertThat(desc.cdn()).isEqualTo(3); - assertThat(desc.key()).isEqualTo("abc"); - assertThat(desc.headers()).containsExactlyEntriesOf(Map.of("k", "v")); - assertThat(desc.signedUploadLocation()).isEqualTo("example.org"); + assertThat(desc) + .isEqualTo(new ArchiveController.UploadDescriptorResponse(3, "abc", Map.of("k", "v"), "example.org")); + verify(backupManager).createMessageBackupUploadDescriptor(any(), eq(uploadLength.orElse(MAX_ATTACHMENT_SIZE))); + } else { + assertThat(response.getStatus()).isEqualTo(413); + } + } + + static Stream mediaUploadForm() { + return Stream.of( + Arguments.of(Optional.empty(), true), + Arguments.of(Optional.of(MAX_ATTACHMENT_SIZE), true), + Arguments.of(Optional.of(MAX_ATTACHMENT_SIZE + 1), false) + ); + } + + @ParameterizedTest + @MethodSource + public void mediaUploadForm(Optional uploadLength, boolean expectSuccess) throws VerificationFailedException, BackupException, RateLimitExceededException { + final BackupAuthCredentialPresentation presentation = + backupAuthTestUtil.getPresentation(BackupLevel.PAID, messagesBackupKey, aci); + when(backupManager.authenticateBackupUser(any(), any(), any())) + .thenReturn(backupUser(presentation.getBackupId(), BackupCredentialType.MESSAGES, BackupLevel.PAID)); + when(backupManager.createTemporaryAttachmentUploadDescriptor(any(), anyLong())) + .thenReturn(new BackupUploadDescriptor(3, "abc", Map.of("k", "v"), "example.org")); + final WebTarget builder = resources.getJerseyTest().target("v1/archives/media/upload/form"); + final Response response = uploadLength + .map(length -> builder.queryParam("uploadLength", length)) + .orElse(builder) + .request() + .header("X-Signal-ZK-Auth", Base64.getEncoder().encodeToString(presentation.serialize())) + .header("X-Signal-ZK-Auth-Signature", "aaa") + .get(); + if (expectSuccess) { + assertThat(response.getStatus()).isEqualTo(200); + final ArchiveController.UploadDescriptorResponse desc = + response.readEntity(ArchiveController.UploadDescriptorResponse.class); + assertThat(desc) + .isEqualTo(new ArchiveController.UploadDescriptorResponse(3, "abc", Map.of("k", "v"), "example.org")); + verify(backupManager).createTemporaryAttachmentUploadDescriptor(any(), eq(uploadLength.orElse(MAX_ATTACHMENT_SIZE))); } else { assertThat(response.getStatus()).isEqualTo(413); } } @Test - public void mediaUploadForm() throws VerificationFailedException, BackupException, RateLimitExceededException { + public void rateLimitMediaUploadForm() + throws BackupWrongCredentialTypeException, RateLimitExceededException, BackupPermissionException, VerificationFailedException, BackupFailedZkAuthenticationException { final BackupAuthCredentialPresentation presentation = backupAuthTestUtil.getPresentation(BackupLevel.PAID, messagesBackupKey, aci); when(backupManager.authenticateBackupUser(any(), any(), any())) .thenReturn(backupUser(presentation.getBackupId(), BackupCredentialType.MESSAGES, BackupLevel.PAID)); - when(backupManager.createTemporaryAttachmentUploadDescriptor(any())) - .thenReturn(new BackupUploadDescriptor(3, "abc", Map.of("k", "v"), "example.org")); - final ArchiveController.UploadDescriptorResponse desc = resources.getJerseyTest() - .target("v1/archives/media/upload/form") - .request() - .header("X-Signal-ZK-Auth", Base64.getEncoder().encodeToString(presentation.serialize())) - .header("X-Signal-ZK-Auth-Signature", "aaa") - .get(ArchiveController.UploadDescriptorResponse.class); - assertThat(desc.cdn()).isEqualTo(3); - assertThat(desc.key()).isEqualTo("abc"); - assertThat(desc.headers()).containsExactlyEntriesOf(Map.of("k", "v")); - assertThat(desc.signedUploadLocation()).isEqualTo("example.org"); - - // rate limit - when(backupManager.createTemporaryAttachmentUploadDescriptor(any())).thenThrow(new RateLimitExceededException(null)); + when(backupManager.createTemporaryAttachmentUploadDescriptor(any(), anyLong())).thenThrow(new RateLimitExceededException(null)); final Response response = resources.getJerseyTest() .target("v1/archives/media/upload/form") .request() diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/controllers/AttachmentControllerV4Test.java b/service/src/test/java/org/whispersystems/textsecuregcm/controllers/AttachmentControllerV4Test.java index a81846e4e..a9a39a37e 100644 --- a/service/src/test/java/org/whispersystems/textsecuregcm/controllers/AttachmentControllerV4Test.java +++ b/service/src/test/java/org/whispersystems/textsecuregcm/controllers/AttachmentControllerV4Test.java @@ -92,7 +92,7 @@ class AttachmentControllerV4Test { static { try { final GcsAttachmentGenerator gcsAttachmentGenerator = new GcsAttachmentGenerator("some-cdn.signal.org", - "signal@example.com", 1000, "/attach-here", RSA_PRIVATE_KEY_PEM); + "signal@example.com", "/attach-here", RSA_PRIVATE_KEY_PEM); resources = ResourceExtension.builder() .addProvider(AuthHelper.getAuthFilter()) .addProvider(new AuthValueFactoryProvider.Binder<>(AuthenticatedDevice.class)) @@ -100,8 +100,8 @@ class AttachmentControllerV4Test { .setTestContainerFactory(new GrizzlyWebTestContainerFactory()) .addProvider(new AttachmentControllerV4(RATE_LIMITERS, gcsAttachmentGenerator, - new TusAttachmentGenerator(new TusConfiguration(new SecretBytes(TUS_SECRET), TUS_URL, MAX_UPLOAD_LENGTH)), - EXPERIMENT_MANAGER)) + new TusAttachmentGenerator(new TusConfiguration(new SecretBytes(TUS_SECRET), TUS_URL)), + EXPERIMENT_MANAGER, MAX_UPLOAD_LENGTH)) .build(); } catch (IOException | InvalidKeyException | InvalidKeySpecException e) { throw new AssertionError(e); diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/grpc/AttachmentsGrpcServiceTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/grpc/AttachmentsGrpcServiceTest.java index 293dea5d3..c2c23fe27 100644 --- a/service/src/test/java/org/whispersystems/textsecuregcm/grpc/AttachmentsGrpcServiceTest.java +++ b/service/src/test/java/org/whispersystems/textsecuregcm/grpc/AttachmentsGrpcServiceTest.java @@ -73,16 +73,17 @@ class AttachmentsGrpcServiceTest extends Base64.getMimeEncoder().encodeToString(keyPair.getPrivate().getEncoded()) + "\n" + "-----END PRIVATE KEY-----"; final GcsAttachmentGenerator gcsAttachmentGenerator = new GcsAttachmentGenerator( - "some-cdn.signal.org", "signal@example.com", 1000, "/attach-here", gcsPrivateKeyPem); + "some-cdn.signal.org", "signal@example.com", "/attach-here", gcsPrivateKeyPem); final TusAttachmentGenerator tusAttachmentGenerator = - new TusAttachmentGenerator(new TusConfiguration(new SecretBytes(TUS_SECRET), TUS_URL, MAX_UPLOAD_LENGTH)); + new TusAttachmentGenerator(new TusConfiguration(new SecretBytes(TUS_SECRET), TUS_URL)); return new AttachmentsGrpcService( experimentEnrollmentManager, MockUtils.buildMock(RateLimiters.class, rateLimiters -> when(rateLimiters.getAttachmentLimiter()).thenReturn(rateLimiter)), gcsAttachmentGenerator, - tusAttachmentGenerator); + tusAttachmentGenerator, + MAX_UPLOAD_LENGTH); } catch (NoSuchAlgorithmException | IOException | InvalidKeyException | InvalidKeySpecException e) { throw new AssertionError(e); } diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/grpc/BackupsAnonymousGrpcServiceTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/grpc/BackupsAnonymousGrpcServiceTest.java index fbe9a9d11..b96e82d3c 100644 --- a/service/src/test/java/org/whispersystems/textsecuregcm/grpc/BackupsAnonymousGrpcServiceTest.java +++ b/service/src/test/java/org/whispersystems/textsecuregcm/grpc/BackupsAnonymousGrpcServiceTest.java @@ -7,7 +7,9 @@ package org.whispersystems.textsecuregcm.grpc; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatExceptionOfType; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.when; @@ -90,7 +92,7 @@ class BackupsAnonymousGrpcServiceTest extends @Override protected BackupsAnonymousGrpcService createServiceBeforeEachTest() { - return new BackupsAnonymousGrpcService(backupManager, new BackupMetrics()); + return new BackupsAnonymousGrpcService(backupManager, new BackupMetrics(), MAX_MESSAGE_BACKUP_OBJECT_SIZE); } @BeforeEach @@ -98,7 +100,6 @@ class BackupsAnonymousGrpcServiceTest extends try { when(backupManager.authenticateBackupUser(any(), any(), any())) .thenReturn(backupUser(presentation.getBackupId(), BackupCredentialType.MESSAGES, BackupLevel.PAID)); - when(backupManager.maxMessageBackupUploadSize()).thenReturn(MAX_MESSAGE_BACKUP_OBJECT_SIZE); } catch (BackupFailedZkAuthenticationException e) { Assertions.fail(e); } @@ -323,55 +324,66 @@ class BackupsAnonymousGrpcServiceTest extends } @Test - void mediaUploadForm() throws RateLimitExceededException, BackupException { - when(backupManager.createTemporaryAttachmentUploadDescriptor(any())) - .thenReturn(new BackupUploadDescriptor(3, "abc", Map.of("k", "v"), "example.org")); + void mediaUploadFormRateLimit() throws RateLimitExceededException, BackupException { final GetUploadFormRequest request = GetUploadFormRequest.newBuilder() .setMedia(GetUploadFormRequest.MediaUploadType.getDefaultInstance()) .setSignedPresentation(signedPresentation(presentation)) + .setUploadLength(100) .build(); - - final GetUploadFormResponse uploadForm = unauthenticatedServiceStub().getUploadForm(request); - assertThat(uploadForm.getUploadForm().getCdn()).isEqualTo(3); - assertThat(uploadForm.getUploadForm().getKey()).isEqualTo("abc"); - assertThat(uploadForm.getUploadForm().getHeadersMap()).containsExactlyEntriesOf(Map.of("k", "v")); - assertThat(uploadForm.getUploadForm().getSignedUploadLocation()).isEqualTo("example.org"); - - // rate limit Duration duration = Duration.ofSeconds(10); - when(backupManager.createTemporaryAttachmentUploadDescriptor(any())) + when(backupManager.createTemporaryAttachmentUploadDescriptor(any(), anyLong())) .thenThrow(new RateLimitExceededException(duration)); GrpcTestUtils.assertRateLimitExceeded(duration, () -> unauthenticatedServiceStub().getUploadForm(request)); } - static Stream messagesUploadForm() { - return Stream.of( - Arguments.of(Optional.empty(), true), - Arguments.of(Optional.of(MAX_MESSAGE_BACKUP_OBJECT_SIZE), true), - Arguments.of(Optional.of(MAX_MESSAGE_BACKUP_OBJECT_SIZE + 1), false) - ); - } - @ParameterizedTest - @MethodSource - public void messagesUploadForm(Optional uploadLength, boolean allowedSize) throws BackupException { - when(backupManager.createMessageBackupUploadDescriptor(any())) - .thenReturn(new BackupUploadDescriptor(3, "abc", Map.of("k", "v"), "example.org")); - final GetUploadFormRequest.MessagesUploadType.Builder builder = GetUploadFormRequest.MessagesUploadType.newBuilder(); - uploadLength.ifPresent(builder::setUploadLength); - final GetUploadFormRequest request = GetUploadFormRequest.newBuilder() - .setMessages(builder.build()) + @EnumSource(value = GetUploadFormRequest.UploadTypeCase.class, names = "UPLOADTYPE_NOT_SET", mode = EnumSource.Mode.EXCLUDE) + public void uploadForm(GetUploadFormRequest.UploadTypeCase uploadType) + throws BackupException, RateLimitExceededException { + final long uploadLength = 100; + final BackupUploadDescriptor result = + new BackupUploadDescriptor(3, "abc", Map.of("k", "v"), "example.org"); + final GetUploadFormRequest.Builder builder = switch (uploadType) { + case MESSAGES -> { + when(backupManager.createMessageBackupUploadDescriptor(any(), eq(uploadLength))) + .thenReturn(result); + yield GetUploadFormRequest.newBuilder().setMessages(GetUploadFormRequest.MessagesUploadType.getDefaultInstance()); + } + case MEDIA -> { + when(backupManager.createTemporaryAttachmentUploadDescriptor(any(), eq(uploadLength))) + .thenReturn(result); + yield GetUploadFormRequest.newBuilder().setMedia(GetUploadFormRequest.MediaUploadType.getDefaultInstance()); + } + default -> throw new IllegalArgumentException("Unknown upload type: " + uploadType); + }; + final GetUploadFormRequest request = builder .setSignedPresentation(signedPresentation(presentation)) + .setUploadLength(uploadLength) .build(); final GetUploadFormResponse response = unauthenticatedServiceStub().getUploadForm(request); - if (allowedSize) { assertThat(response.getUploadForm().getCdn()).isEqualTo(3); assertThat(response.getUploadForm().getKey()).isEqualTo("abc"); assertThat(response.getUploadForm().getHeadersMap()).containsExactlyEntriesOf(Map.of("k", "v")); assertThat(response.getUploadForm().getSignedUploadLocation()).isEqualTo("example.org"); - } else { - assertThat(response.hasExceedsMaxUploadLength()).isTrue(); - } + } + + @ParameterizedTest + @EnumSource(value = GetUploadFormRequest.UploadTypeCase.class, names = "UPLOADTYPE_NOT_SET", mode = EnumSource.Mode.EXCLUDE) + public void uploadFormExceedsMax(GetUploadFormRequest.UploadTypeCase uploadType) throws BackupException { + final GetUploadFormRequest.Builder builder = switch (uploadType) { + case MESSAGES -> GetUploadFormRequest.newBuilder() + .setMessages(GetUploadFormRequest.MessagesUploadType.getDefaultInstance()); + case MEDIA -> GetUploadFormRequest.newBuilder() + .setMedia(GetUploadFormRequest.MediaUploadType.getDefaultInstance()); + default -> throw new IllegalArgumentException("Unknown upload type: " + uploadType); + }; + final GetUploadFormRequest request = builder + .setSignedPresentation(signedPresentation(presentation)) + .setUploadLength(MAX_MESSAGE_BACKUP_OBJECT_SIZE + 1) + .build(); + + final GetUploadFormResponse response = unauthenticatedServiceStub().getUploadForm(request); + assertThat(response.hasExceedsMaxUploadLength()).isTrue(); } diff --git a/service/src/test/resources/config/test.yml b/service/src/test/resources/config/test.yml index 2fd3b02b7..5f387f3fa 100644 --- a/service/src/test/resources/config/test.yml +++ b/service/src/test/resources/config/test.yml @@ -239,17 +239,18 @@ messageCache: # Redis server configuration for message store cache cluster: type: local +attachments: + maxUploadSizeInBytes: 1024 + gcpAttachments: # GCP Storage configuration domain: example.com email: user@example.cocm - maxSizeInBytes: 1024 pathPrefix: rsaSigningKey: secret://gcpAttachments.rsaSigningKey tus: uploadUri: https://example.org/upload userAuthenticationTokenSharedSecret: secret://tus.userAuthenticationTokenSharedSecret - maxSizeInBytes: 1024 apn: # Apple Push Notifications configuration sandbox: true