Signal/Signal-Server
1
0
Fork 0
mirror of https://github.com/signalapp/Signal-Server synced 2026-04-20 02:58:04 +01:00
Code Issues Packages Projects Releases Wiki Activity

Add and enforce uploadLength in backup endpoints

Browse Source
This commit is contained in:
ravi-signal
2026-03-31 11:08:08 -05:00
committed by GitHub
parent 771c98fd92
commit f9d3cd8d82
23 changed files with 264 additions and 210 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerConfiguration.java
View File

@@ -18,6 +18,7 @@ import org.whispersystems.textsecuregcm.attachments.TusConfiguration;
import org.whispersystems.textsecuregcm.configuration.ApnConfiguration;
import org.whispersystems.textsecuregcm.configuration.AppleAppStoreConfiguration;
import org.whispersystems.textsecuregcm.configuration.AppleDeviceCheckConfiguration;
import org.whispersystems.textsecuregcm.configuration.AttachmentsConfiguration;
import org.whispersystems.textsecuregcm.configuration.AwsCredentialsProviderFactory;
import org.whispersystems.textsecuregcm.configuration.BadgesConfiguration;
import org.whispersystems.textsecuregcm.configuration.BraintreeConfiguration;
@@ -122,6 +123,11 @@ public class WhisperServerConfiguration extends Configuration {
@JsonProperty
private DynamoDbTables dynamoDbTables;
@NotNull
@Valid
@JsonProperty
private AttachmentsConfiguration attachments;
@NotNull
@Valid
@JsonProperty
@@ -412,6 +418,10 @@ public class WhisperServerConfiguration extends Configuration {
return webSocket;
}
public AttachmentsConfiguration getAttachments() {
return attachments;
}
public GcpAttachmentsConfiguration getGcpAttachmentsConfiguration() {
return gcpAttachments;
}
@@ -610,4 +620,5 @@ public class WhisperServerConfiguration extends Configuration {
public HlrLookupConfiguration getHlrLookupConfiguration() {
return hlrLookup;
}
}

10
service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java
View File

@@ -827,7 +827,6 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
final GcsAttachmentGenerator gcsAttachmentGenerator = new GcsAttachmentGenerator(
config.getGcpAttachmentsConfiguration().domain(),
config.getGcpAttachmentsConfiguration().email(),
config.getGcpAttachmentsConfiguration().maxSizeInBytes(),
config.getGcpAttachmentsConfiguration().pathPrefix(),
config.getGcpAttachmentsConfiguration().rsaSigningKey().value());
@@ -978,7 +977,8 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
new MessagesGrpcService(accountsManager, rateLimiters, messageSender, messageByteLimitCardinalityEstimator, spamChecker, Clock.systemUTC()),
new BackupsGrpcService(accountsManager, backupAuthManager, backupMetrics),
new DevicesGrpcService(accountsManager),
new AttachmentsGrpcService(experimentEnrollmentManager, rateLimiters, gcsAttachmentGenerator, tusAttachmentGenerator))
new AttachmentsGrpcService(experimentEnrollmentManager, rateLimiters,
gcsAttachmentGenerator, tusAttachmentGenerator, config.getAttachments().maxUploadSizeInBytes()))
.map(bindableService -> ServerInterceptors.intercept(bindableService,
// Note: interceptors run in the reverse order they are added; the remote deprecation filter
// depends on the user-agent context so it has to come first here!
@@ -997,7 +997,7 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
new KeysAnonymousGrpcService(accountsManager, keysManager, zkSecretParams, Clock.systemUTC()),
new PaymentsGrpcService(currencyManager),
new MessagesAnonymousGrpcService(accountsManager, rateLimiters, messageSender, groupSendTokenUtil, messageByteLimitCardinalityEstimator, spamChecker, Clock.systemUTC()),
new BackupsAnonymousGrpcService(backupManager, backupMetrics),
new BackupsAnonymousGrpcService(backupManager, backupMetrics, config.getAttachments().maxUploadSizeInBytes()),
ExternalServiceCredentialsAnonymousGrpcService.create(accountsManager, config))
.map(bindableService -> ServerInterceptors.intercept(bindableService,
// Note: interceptors run in the reverse order they are added; the remote deprecation filter
@@ -1100,8 +1100,8 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
new AccountControllerV2(accountsManager, changeNumberManager, phoneVerificationTokenManager,
registrationLockVerificationManager, rateLimiters),
new AttachmentControllerV4(rateLimiters, gcsAttachmentGenerator, tusAttachmentGenerator,
experimentEnrollmentManager),
new ArchiveController(accountsManager, backupAuthManager, backupManager, backupMetrics),
experimentEnrollmentManager, config.getAttachments().maxUploadSizeInBytes()),
new ArchiveController(accountsManager, backupAuthManager, backupManager, backupMetrics, config.getAttachments().maxUploadSizeInBytes()),
new CallRoutingControllerV2(rateLimiters, cloudflareTurnCredentialsManager),
new CallLinkController(rateLimiters, callingGenericZkSecretParams),
new CallQualitySurveyController(callQualitySurveyManager),

2
service/src/main/java/org/whispersystems/textsecuregcm/attachments/AttachmentGenerator.java
View File

@@ -12,6 +12,4 @@ public interface AttachmentGenerator {
Descriptor generateAttachment(final String key, final long uploadLength);
long maxUploadSizeInBytes();
}

12
service/src/main/java/org/whispersystems/textsecuregcm/attachments/GcsAttachmentGenerator.java
View File

@@ -22,31 +22,21 @@ public class GcsAttachmentGenerator implements AttachmentGenerator {
@Nonnull
private final CanonicalRequestSigner canonicalRequestSigner;
private final long maxSizeInBytes;
public GcsAttachmentGenerator(@Nonnull String domain, @Nonnull String email,
long maxSizeInBytes, @Nonnull String pathPrefix, @Nonnull String rsaSigningKey)
@Nonnull String pathPrefix, @Nonnull String rsaSigningKey)
throws IOException, InvalidKeyException, InvalidKeySpecException {
this.maxSizeInBytes = maxSizeInBytes;
this.canonicalRequestGenerator = new CanonicalRequestGenerator(domain, email, pathPrefix);
this.canonicalRequestSigner = new CanonicalRequestSigner(rsaSigningKey);
}
@Override
public Descriptor generateAttachment(final String key, final long uploadLength) {
if (uploadLength > maxSizeInBytes) {
throw new IllegalArgumentException("uploadLength " + uploadLength + " exceeds maximum " + maxSizeInBytes);
}
final ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC);
final CanonicalRequest canonicalRequest = canonicalRequestGenerator.createFor(key, now, uploadLength);
return new Descriptor(getHeaderMap(canonicalRequest), getSignedUploadLocation(canonicalRequest));
}
@Override
public long maxUploadSizeInBytes() {
return this.maxSizeInBytes;
}
private String getSignedUploadLocation(@Nonnull CanonicalRequest canonicalRequest) {
return "https://" + canonicalRequest.getDomain() + canonicalRequest.getResourcePath()
+ '?' + canonicalRequest.getCanonicalQuery()

11
service/src/main/java/org/whispersystems/textsecuregcm/attachments/TusAttachmentGenerator.java
View File

@@ -19,20 +19,14 @@ public class TusAttachmentGenerator implements AttachmentGenerator {
private final JwtGenerator jwtGenerator;
private final String tusUri;
private final long maxUploadSize;
public TusAttachmentGenerator(final TusConfiguration cfg) {
this.tusUri = cfg.uploadUri();
this.jwtGenerator = new JwtGenerator(cfg.userAuthenticationTokenSharedSecret().value(), Clock.systemUTC());
this.maxUploadSize = cfg.maxSizeInBytes();
}
@Override
public Descriptor generateAttachment(final String key, final long uploadLength) {
if (uploadLength > maxUploadSize) {
throw new IllegalArgumentException("uploadLength " + uploadLength + " exceeds maximum " + maxUploadSize);
}
final String token = jwtGenerator.generateJwt(ATTACHMENTS, key,
builder -> builder.withClaim(JwtGenerator.MAX_LENGTH_CLAIM_KEY, uploadLength));
final String b64Key = Base64.getEncoder().encodeToString(key.getBytes(StandardCharsets.UTF_8));
@@ -43,9 +37,4 @@ public class TusAttachmentGenerator implements AttachmentGenerator {
);
return new Descriptor(headers, tusUri + "/" + ATTACHMENTS);
}
@Override
public long maxUploadSizeInBytes() {
return maxUploadSize;
}
}

4
service/src/main/java/org/whispersystems/textsecuregcm/attachments/TusConfiguration.java
View File

@@ -12,6 +12,4 @@ import org.whispersystems.textsecuregcm.util.ExactlySize;
public record TusConfiguration(
@ExactlySize(32) SecretBytes userAuthenticationTokenSharedSecret,
@NotEmpty String uploadUri,
@Positive long maxSizeInBytes
){}
@NotEmpty String uploadUri){}

26
service/src/main/java/org/whispersystems/textsecuregcm/backup/BackupManager.java
View File

@@ -155,28 +155,33 @@ public class BackupManager {
* If successful, this also updates the TTL of the backup.
*
* @param backupUser an already ZK authenticated backup user
* @param uploadSize the maximum size of a message backup that can be uploaded with the returned form
* @return the upload form
* @throws BackupPermissionException if the credential does not have the correct level
* @throws BackupWrongCredentialTypeException if the credential does not have the messages type
*/
public BackupUploadDescriptor createMessageBackupUploadDescriptor(
final AuthenticatedBackupUser backupUser) throws BackupPermissionException, BackupWrongCredentialTypeException {
final AuthenticatedBackupUser backupUser,
final long uploadSize) throws BackupPermissionException, BackupWrongCredentialTypeException {
checkBackupLevel(backupUser, BackupLevel.FREE);
checkBackupCredentialType(backupUser, BackupCredentialType.MESSAGES);
// this could race with concurrent updates, but the only effect would be last-writer-wins on the timestamp
backupsDb.addMessageBackup(backupUser).join();
return cdn3BackupCredentialGenerator.generateUpload(cdnMessageBackupName(backupUser));
return cdn3BackupCredentialGenerator.generateUpload(cdnMessageBackupName(backupUser), uploadSize);
}
public BackupUploadDescriptor createTemporaryAttachmentUploadDescriptor(final AuthenticatedBackupUser backupUser)
public BackupUploadDescriptor createTemporaryAttachmentUploadDescriptor(
final AuthenticatedBackupUser backupUser,
final long uploadSize)
throws RateLimitExceededException, BackupPermissionException, BackupWrongCredentialTypeException {
checkBackupLevel(backupUser, BackupLevel.PAID);
checkBackupCredentialType(backupUser, BackupCredentialType.MEDIA);
rateLimiters.forDescriptor(RateLimiters.For.BACKUP_ATTACHMENT).validate(rateLimitKey(backupUser));
final String attachmentKey = AttachmentUtil.generateAttachmentKey(secureRandom);
final AttachmentGenerator.Descriptor descriptor = tusAttachmentGenerator.generateAttachment(attachmentKey, tusAttachmentGenerator.maxUploadSizeInBytes());
final AttachmentGenerator.Descriptor descriptor =
tusAttachmentGenerator.generateAttachment(attachmentKey, uploadSize);
return new BackupUploadDescriptor(3, attachmentKey, descriptor.headers(), descriptor.signedUploadLocation());
}
@@ -194,9 +199,9 @@ public class BackupManager {
final long maxTotalMediaSize =
dynamicConfigurationManager.getConfiguration().getBackupConfiguration().maxTotalMediaSize();
// Report that the backup is out of quota if it cannot store a max size media object
// Report that the backup is out of quota if it is within 100MiB of the limit
final boolean quotaExhausted = storedBackupAttributes.bytesUsed() >=
(maxTotalMediaSize - tusAttachmentGenerator.maxUploadSizeInBytes());
(maxTotalMediaSize - 1024 * 1024 * 100);
final Tags tags = Tags.of(
UserAgentTagUtil.getPlatformTag(backupUser.userAgent()),
@@ -329,13 +334,14 @@ public class BackupManager {
*/
public CopyQuota getCopyQuota(
final AuthenticatedBackupUser backupUser,
final List<CopyParameters> toCopy)
final List<CopyParameters> toCopy,
final long maximumSourceObjectSize)
throws BackupWrongCredentialTypeException, BackupPermissionException, BackupInvalidArgumentException {
checkBackupLevel(backupUser, BackupLevel.PAID);
checkBackupCredentialType(backupUser, BackupCredentialType.MEDIA);
for (CopyParameters copyParameters : toCopy) {
if (copyParameters.sourceLength() > tusAttachmentGenerator.maxUploadSizeInBytes() || copyParameters.sourceLength() < 0) {
if (copyParameters.sourceLength() > maximumSourceObjectSize || copyParameters.sourceLength() < 0) {
throw new BackupInvalidArgumentException("Invalid sourceObject size");
}
}
@@ -729,10 +735,6 @@ public class BackupManager {
.toFuture();
}
public long maxMessageBackupUploadSize() {
return tusAttachmentGenerator.maxUploadSizeInBytes();
}
interface PresentationSignatureVerifier {
Pair<BackupCredentialType, BackupLevel> verifySignature(byte[] signature, ECPublicKey publicKey) throws BackupFailedZkAuthenticationException;

6
service/src/main/java/org/whispersystems/textsecuregcm/backup/Cdn3BackupCredentialGenerator.java
View File

@@ -21,21 +21,19 @@ public class Cdn3BackupCredentialGenerator {
private final String tusUri;
private final JwtGenerator jwtGenerator;
private final long maxUploadSize;
public Cdn3BackupCredentialGenerator(final TusConfiguration cfg) {
this.tusUri = cfg.uploadUri();
this.jwtGenerator = new JwtGenerator(cfg.userAuthenticationTokenSharedSecret().value(), Clock.systemUTC());
this.maxUploadSize = cfg.maxSizeInBytes();
}
public BackupUploadDescriptor generateUpload(final String key) {
public BackupUploadDescriptor generateUpload(final String key, final long uploadSize) {
if (key.isBlank()) {
throw new IllegalArgumentException("Upload descriptors must have non-empty keys");
}
final String token = jwtGenerator.generateJwt(CDN_PATH, key, builder -> builder
.withClaim(JwtGenerator.MAX_LENGTH_CLAIM_KEY, maxUploadSize)
.withClaim(JwtGenerator.MAX_LENGTH_CLAIM_KEY, uploadSize)
.withClaim(JwtGenerator.SCOPE_CLAIM_KEY, "write"));
final String b64Key = Base64.getEncoder().encodeToString(key.getBytes(StandardCharsets.UTF_8));

10
service/src/main/java/org/whispersystems/textsecuregcm/configuration/AttachmentsConfiguration.java Normal file
View File

@@ -0,0 +1,10 @@
/*
* Copyright 2026 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.configuration;
import jakarta.validation.constraints.Positive;
public record AttachmentsConfiguration(@Positive long maxUploadSizeInBytes) {
}

1
service/src/main/java/org/whispersystems/textsecuregcm/configuration/GcpAttachmentsConfiguration.java
View File

@@ -14,7 +14,6 @@ import org.whispersystems.textsecuregcm.configuration.secrets.SecretString;
public record GcpAttachmentsConfiguration(@NotBlank String domain,
@NotBlank String email,
@Min(1) int maxSizeInBytes,
String pathPrefix,
@NotNull SecretString rsaSigningKey) {
@SuppressWarnings("unused")

38
service/src/main/java/org/whispersystems/textsecuregcm/controllers/ArchiveController.java
View File

@@ -102,17 +102,20 @@ public class ArchiveController {
private final BackupAuthManager backupAuthManager;
private final BackupManager backupManager;
private final BackupMetrics backupMetrics;
private final long maxAttachmentSize;
public ArchiveController(
final AccountsManager accountsManager,
final BackupAuthManager backupAuthManager,
final BackupManager backupManager,
final BackupMetrics backupMetrics) {
final BackupMetrics backupMetrics,
final long maxAttachmentSize) {
this.accountsManager = accountsManager;
this.backupAuthManager = backupAuthManager;
this.backupManager = backupManager;
this.backupMetrics = backupMetrics;
this.maxAttachmentSize = maxAttachmentSize;
}
public record SetBackupIdRequest(
@@ -566,8 +569,11 @@ public class ArchiveController {
@Path("/upload/form")
@Produces(MediaType.APPLICATION_JSON)
@Operation(
summary = "Fetch message backup upload form",
description = "Retrieve an upload form that can be used to perform a resumable upload of a message backup.")
summary = "Fetch message backup upload form", description = """
Retrieve an upload form that can be used to perform a resumable upload of a message backup.
Uploads with the returned form will be limited to a maximum size of the provided uploadLength.
""")
@ApiResponse(responseCode = "200", content = @Content(schema = @Schema(implementation = UploadDescriptorResponse.class)))
@ApiResponse(responseCode = "429", description = "Rate limited.")
@ApiResponse(responseCode = "413", description = "The provided uploadLength is larger than the maximum supported upload size. The maximum upload size is subject to change.")
@@ -596,7 +602,7 @@ public class ArchiveController {
backupManager.authenticateBackupUser(presentation.presentation, signature.signature, userAgent);
final boolean oversize = uploadLength
.map(length -> length > backupManager.maxMessageBackupUploadSize())
.map(length -> length > maxAttachmentSize)
.orElse(false);
backupMetrics.updateMessageBackupSizeDistribution(backupUser, oversize, uploadLength);
@@ -604,7 +610,7 @@ public class ArchiveController {
throw new ClientErrorException("exceeded maximum uploadLength", Response.Status.REQUEST_ENTITY_TOO_LARGE);
}
final BackupUploadDescriptor uploadDescriptor =
backupManager.createMessageBackupUploadDescriptor(backupUser);
backupManager.createMessageBackupUploadDescriptor(backupUser, uploadLength.orElse(maxAttachmentSize));
return new UploadDescriptorResponse(
uploadDescriptor.cdn(),
uploadDescriptor.key(),
@@ -621,32 +627,42 @@ public class ArchiveController {
Retrieve an upload form that can be used to perform a resumable upload of an attachment. After uploading, the
attachment can be copied into the backup at PUT /archives/media/.
Like the account authenticated version at /attachments, the uploaded object is only temporary.
Like the account authenticated version at /attachments, the uploaded object is only temporary. Uploads with
the returned form will be limited to a maximum size of the provided uploadLength.
""")
@ApiResponse(responseCode = "200", content = @Content(schema = @Schema(implementation = UploadDescriptorResponse.class)))
@ApiResponse(responseCode = "429", description = "Rate limited.")
@ApiResponse(responseCode = "413", description = "The provided uploadLength is larger than the maximum supported upload size. The maximum upload size is subject to change and is governed by `global.attachments.maxBytes`.")
@ApiResponseZkAuth
@ManagedAsync
public UploadDescriptorResponse uploadTemporaryAttachment(
@Auth final Optional<AuthenticatedDevice> account,
@HeaderParam(HttpHeaders.USER_AGENT) final String userAgent,
@Parameter(description = BackupAuthCredentialPresentationHeader.DESCRIPTION, schema = @Schema(implementation = String.class))
@NotNull
@HeaderParam(X_SIGNAL_ZK_AUTH) final ArchiveController.BackupAuthCredentialPresentationHeader presentation,
@Parameter(description = BackupAuthCredentialPresentationSignature.DESCRIPTION, schema = @Schema(implementation = String.class))
@NotNull
@HeaderParam(X_SIGNAL_ZK_AUTH_SIGNATURE) final BackupAuthCredentialPresentationSignature signature)
@HeaderParam(X_SIGNAL_ZK_AUTH_SIGNATURE) final BackupAuthCredentialPresentationSignature signature,
@Parameter(description = "The size of the temporary attachment to upload in bytes")
@QueryParam("uploadLength") final Optional<Long> uploadLength)
throws RateLimitExceededException, BackupFailedZkAuthenticationException, BackupWrongCredentialTypeException, BackupPermissionException {
if (account.isPresent()) {
throw new BadRequestException("must not use authenticated connection for anonymous operations");
}
final AuthenticatedBackupUser backupUser =
backupManager.authenticateBackupUser(presentation.presentation, signature.signature, userAgent);
final boolean oversize = uploadLength.map(length -> length > maxAttachmentSize).orElse(false);
if (oversize) {
throw new ClientErrorException("exceeded maximum uploadLength", Response.Status.REQUEST_ENTITY_TOO_LARGE);
}
final BackupUploadDescriptor uploadDescriptor =
backupManager.createTemporaryAttachmentUploadDescriptor(backupUser);
backupManager.createTemporaryAttachmentUploadDescriptor(backupUser, uploadLength.orElse(maxAttachmentSize));
return new UploadDescriptorResponse(
uploadDescriptor.cdn(),
uploadDescriptor.key(),
@@ -741,7 +757,7 @@ public class ArchiveController {
final AuthenticatedBackupUser backupUser =
backupManager.authenticateBackupUser(presentation.presentation, signature.signature, userAgent);
final BackupManager.CopyQuota copyQuota =
backupManager.getCopyQuota(backupUser, List.of(copyMediaRequest.toCopyParameters()));
backupManager.getCopyQuota(backupUser, List.of(copyMediaRequest.toCopyParameters()), maxAttachmentSize);
final CopyResult copyResult = backupManager.copyToBackup(copyQuota).next()
.blockOptional()
.orElseThrow(() -> new IllegalStateException("Non empty copy request must return result"));
@@ -844,7 +860,7 @@ public class ArchiveController {
final Stream<CopyParameters> copyParams = copyMediaRequest.items().stream().map(CopyMediaRequest::toCopyParameters);
final AuthenticatedBackupUser backupUser =
backupManager.authenticateBackupUser(presentation.presentation, signature.signature, userAgent);
final BackupManager.CopyQuota copyQuota = backupManager.getCopyQuota(backupUser, copyParams.toList());
final BackupManager.CopyQuota copyQuota = backupManager.getCopyQuota(backupUser, copyParams.toList(), maxAttachmentSize);
final List<CopyMediaBatchResponse.Entry> copyResults = backupManager.copyToBackup(copyQuota)
.doOnNext(result -> backupMetrics.updateCopyCounter(result, UserAgentTagUtil.getPlatformTag(userAgent)))
.map(CopyMediaBatchResponse.Entry::fromCopyResult)

20
service/src/main/java/org/whispersystems/textsecuregcm/controllers/AttachmentControllerV4.java
View File

@@ -45,6 +45,7 @@ public class AttachmentControllerV4 {
private final ExperimentEnrollmentManager experimentEnrollmentManager;
private final RateLimiter rateLimiter;
private final long maxUploadLength;
private final Map<Integer, AttachmentGenerator> attachmentGenerators;
@@ -55,9 +56,11 @@ public class AttachmentControllerV4 {
final RateLimiters rateLimiters,
final GcsAttachmentGenerator gcsAttachmentGenerator,
final TusAttachmentGenerator tusAttachmentGenerator,
final ExperimentEnrollmentManager experimentEnrollmentManager) {
final ExperimentEnrollmentManager experimentEnrollmentManager,
final long maxUploadLength) {
this.rateLimiter = rateLimiters.getAttachmentLimiter();
this.experimentEnrollmentManager = experimentEnrollmentManager;
this.maxUploadLength = maxUploadLength;
this.secureRandom = new SecureRandom();
this.attachmentGenerators = Map.of(
2, gcsAttachmentGenerator,
@@ -88,17 +91,16 @@ public class AttachmentControllerV4 {
@Parameter(description = "The size of the attachment to upload in bytes")
@QueryParam("uploadLength") final @Valid Optional<@Positive Long> maybeUploadLength)
throws RateLimitExceededException {
final long uploadLength = maybeUploadLength.orElse(maxUploadLength);
if (uploadLength > maxUploadLength) {
throw new ClientErrorException("exceeded maximum uploadLength", Response.Status.REQUEST_ENTITY_TOO_LARGE);
}
rateLimiter.validate(auth.accountIdentifier());
final String key = AttachmentUtil.generateAttachmentKey(secureRandom);
final boolean useCdn3 = this.experimentEnrollmentManager.isEnrolled(auth.accountIdentifier(), AttachmentUtil.CDN3_EXPERIMENT_NAME);
int cdn = useCdn3 ? 3 : 2;
final AttachmentGenerator attachmentGenerator = this.attachmentGenerators.get(cdn);
final long uploadLength = maybeUploadLength.orElse(attachmentGenerator.maxUploadSizeInBytes());
if (uploadLength > attachmentGenerator.maxUploadSizeInBytes()) {
throw new ClientErrorException("exceeded maximum uploadLength", Response.Status.REQUEST_ENTITY_TOO_LARGE);
}
rateLimiter.validate(auth.accountIdentifier());
final AttachmentGenerator.Descriptor descriptor = attachmentGenerator.generateAttachment(key, uploadLength);
final AttachmentGenerator.Descriptor descriptor = this.attachmentGenerators.get(cdn).generateAttachment(key, uploadLength);
return new AttachmentDescriptorV3(cdn, key, descriptor.headers(), descriptor.signedUploadLocation());
}

23
service/src/main/java/org/whispersystems/textsecuregcm/grpc/AttachmentsGrpcService.java
View File

@@ -27,6 +27,7 @@ public class AttachmentsGrpcService extends SimpleAttachmentsGrpc.AttachmentsImp
private final ExperimentEnrollmentManager experimentEnrollmentManager;
private final RateLimiter rateLimiter;
private final long maxUploadLength;
private final Map<Integer, AttachmentGenerator> attachmentGenerators;
private final SecureRandom secureRandom;
@@ -34,9 +35,11 @@ public class AttachmentsGrpcService extends SimpleAttachmentsGrpc.AttachmentsImp
final ExperimentEnrollmentManager experimentEnrollmentManager,
final RateLimiters rateLimiters,
final GcsAttachmentGenerator gcsAttachmentGenerator,
final TusAttachmentGenerator tusAttachmentGenerator) {
final TusAttachmentGenerator tusAttachmentGenerator,
final long maxUploadLength) {
this.experimentEnrollmentManager = experimentEnrollmentManager;
this.rateLimiter = rateLimiters.getAttachmentLimiter();
this.maxUploadLength = maxUploadLength;
this.secureRandom = new SecureRandom();
this.attachmentGenerators = Map.of(
2, gcsAttachmentGenerator,
@@ -45,20 +48,20 @@ public class AttachmentsGrpcService extends SimpleAttachmentsGrpc.AttachmentsImp
@Override
public GetUploadFormResponse getUploadForm(final GetUploadFormRequest request) throws RateLimitExceededException {
final AuthenticatedDevice auth = AuthenticationUtil.requireAuthenticatedDevice();
final String key = AttachmentUtil.generateAttachmentKey(secureRandom);
final boolean useCdn3 = this.experimentEnrollmentManager.isEnrolled(auth.accountIdentifier(),
AttachmentUtil.CDN3_EXPERIMENT_NAME);
final int cdn = useCdn3 ? 3 : 2;
final AttachmentGenerator attachmentGenerator = this.attachmentGenerators.get(cdn);
if (request.getUploadLength() > attachmentGenerator.maxUploadSizeInBytes()) {
if (request.getUploadLength() > maxUploadLength) {
return GetUploadFormResponse.newBuilder()
.setExceedsMaxUploadLength(FailedPrecondition.getDefaultInstance())
.build();
}
final AuthenticatedDevice auth = AuthenticationUtil.requireAuthenticatedDevice();
rateLimiter.validate(auth.accountIdentifier());
final AttachmentGenerator.Descriptor descriptor = attachmentGenerator.generateAttachment(key, request.getUploadLength());
final String key = AttachmentUtil.generateAttachmentKey(secureRandom);
final boolean useCdn3 = this.experimentEnrollmentManager.isEnrolled(auth.accountIdentifier(),
AttachmentUtil.CDN3_EXPERIMENT_NAME);
final int cdn = useCdn3 ? 3 : 2;
final AttachmentGenerator.Descriptor descriptor =
this.attachmentGenerators.get(cdn).generateAttachment(key, request.getUploadLength());
return GetUploadFormResponse.newBuilder().setUploadForm(UploadForm.newBuilder()
.setCdn(cdn)
.setKey(key)

51
service/src/main/java/org/whispersystems/textsecuregcm/grpc/BackupsAnonymousGrpcService.java
View File

@@ -32,6 +32,7 @@ import org.signal.chat.backup.SetPublicKeyRequest;
import org.signal.chat.backup.SetPublicKeyResponse;
import org.signal.chat.backup.SignedPresentation;
import org.signal.chat.backup.SimpleBackupsAnonymousGrpc;
import org.signal.chat.common.UploadForm;
import org.signal.chat.errors.FailedPrecondition;
import org.signal.chat.errors.FailedZkAuthentication;
import org.signal.libsignal.protocol.InvalidKeyException;
@@ -60,10 +61,12 @@ public class BackupsAnonymousGrpcService extends SimpleBackupsAnonymousGrpc.Back
private final BackupManager backupManager;
private final BackupMetrics backupMetrics;
private final long maxAttachmentSize;
public BackupsAnonymousGrpcService(final BackupManager backupManager, final BackupMetrics backupMetrics) {
public BackupsAnonymousGrpcService(final BackupManager backupManager, final BackupMetrics backupMetrics, final long maxAttachmentSize) {
this.backupManager = backupManager;
this.backupMetrics = backupMetrics;
this.maxAttachmentSize = maxAttachmentSize;
}
@Override
@@ -178,35 +181,29 @@ public class BackupsAnonymousGrpcService extends SimpleBackupsAnonymousGrpc.Back
.setFailedAuthentication(FailedZkAuthentication.newBuilder().setDescription(e.getMessage()).build())
.build();
}
final GetUploadFormResponse.Builder builder = GetUploadFormResponse.newBuilder();
switch (request.getUploadTypeCase()) {
case MESSAGES -> {
final long uploadLength = request.getMessages().getUploadLength();
final boolean oversize = uploadLength > backupManager.maxMessageBackupUploadSize();
backupMetrics.updateMessageBackupSizeDistribution(backupUser, oversize, Optional.of(uploadLength));
if (oversize) {
builder.setExceedsMaxUploadLength(FailedPrecondition.getDefaultInstance());
} else {
final BackupUploadDescriptor uploadDescriptor = backupManager.createMessageBackupUploadDescriptor(backupUser);
builder.setUploadForm(builder.getUploadFormBuilder()
.setCdn(uploadDescriptor.cdn())
.setKey(uploadDescriptor.key())
.setSignedUploadLocation(uploadDescriptor.signedUploadLocation())
.putAllHeaders(uploadDescriptor.headers())).build();
}
final long uploadLength = request.getUploadLength();
if (uploadLength > maxAttachmentSize) {
if (request.getUploadTypeCase() == GetUploadFormRequest.UploadTypeCase.MESSAGES) {
backupMetrics.updateMessageBackupSizeDistribution(backupUser, true, Optional.of(uploadLength));
}
case MEDIA -> {
final BackupUploadDescriptor uploadDescriptor = backupManager.createTemporaryAttachmentUploadDescriptor(
backupUser);
builder.setUploadForm(builder.getUploadFormBuilder()
return GetUploadFormResponse.newBuilder().setExceedsMaxUploadLength(FailedPrecondition.getDefaultInstance()).build();
}
final BackupUploadDescriptor uploadDescriptor = switch (request.getUploadTypeCase()) {
case MESSAGES -> {
backupMetrics.updateMessageBackupSizeDistribution(backupUser, false, Optional.of(uploadLength));
yield backupManager.createMessageBackupUploadDescriptor(backupUser, uploadLength);
}
case MEDIA -> backupManager.createTemporaryAttachmentUploadDescriptor(backupUser, uploadLength);
case UPLOADTYPE_NOT_SET -> throw GrpcExceptions.fieldViolation("upload_type", "Must set upload_type");
};
return GetUploadFormResponse.newBuilder()
.setUploadForm(UploadForm.newBuilder()
.setCdn(uploadDescriptor.cdn())
.setKey(uploadDescriptor.key())
.setSignedUploadLocation(uploadDescriptor.signedUploadLocation())
.putAllHeaders(uploadDescriptor.headers())).build();
}
case UPLOADTYPE_NOT_SET -> throw GrpcExceptions.fieldViolation("upload_type", "Must set upload_type");
}
return builder.build();
.putAllHeaders(uploadDescriptor.headers()))
.build();
}
@Override
@@ -221,7 +218,7 @@ public class BackupsAnonymousGrpcService extends SimpleBackupsAnonymousGrpc.Back
// uint32 in proto, make sure it fits in a signed int
fromUnsignedExact(item.getObjectLength()),
new MediaEncryptionParameters(item.getEncryptionKey().toByteArray(), item.getHmacKey().toByteArray()),
item.getMediaId().toByteArray())).toList());
item.getMediaId().toByteArray())).toList(), maxAttachmentSize);
} catch (BackupFailedZkAuthenticationException e) {
return JdkFlowAdapter.publisherToFlowPublisher(Mono.just(CopyMediaResponse
.newBuilder()

11
service/src/main/proto/org/signal/chat/backups.proto
View File

@@ -349,9 +349,7 @@ message RefreshResponse {
message GetUploadFormRequest {
SignedPresentation signed_presentation = 1;
message MessagesUploadType {
uint64 upload_length = 1;
}
message MessagesUploadType {}
message MediaUploadType {}
oneof upload_type {
// Retrieve an upload form that can be used to perform a resumable upload of
@@ -365,6 +363,10 @@ message GetUploadFormRequest {
// Behaves identically to the account authenticated version at /attachments.
MediaUploadType media = 3;
}
// The length of the attachment for the requested upload form. Uploads
// performed with this form will be limited to the provided length.
uint64 uploadLength = 4 [(require.range) = {min: 1}];
}
message GetUploadFormResponse {
oneof outcome {
@@ -377,7 +379,8 @@ message GetUploadFormResponse {
errors.FailedZkAuthentication failed_authentication = 2 [(tag.reason) = "failed_authentication"];
// The request size was larger than the maximum supported upload size. The
// maximum upload size is subject to change.
// maximum upload size is subject to change and is governed by
// `global.attachments.maxBytes`
errors.FailedPrecondition exceeds_max_upload_length = 3 [(tag.reason) = "oversize_upload"];
}
}

50
service/src/test/java/org/whispersystems/textsecuregcm/backup/BackupManagerTest.java
View File

@@ -152,8 +152,6 @@ public class BackupManagerTest {
when(dynamicConfiguration.getBackupConfiguration()).thenReturn(backupConfiguration);
when(dynamicConfigurationManager.getConfiguration()).thenReturn(dynamicConfiguration);
when(tusAttachmentGenerator.maxUploadSizeInBytes()).thenReturn(MAX_UPLOAD_SIZE);
this.backupManager = new BackupManager(
backupsDb,
backupAuthTestUtil.params,
@@ -224,9 +222,9 @@ public class BackupManagerTest {
final AuthenticatedBackupUser backupUser = backupUser(TestRandomUtil.nextBytes(16), BackupCredentialType.MESSAGES, backupLevel);
backupManager.createMessageBackupUploadDescriptor(backupUser);
backupManager.createMessageBackupUploadDescriptor(backupUser, 17);
verify(tusCredentialGenerator, times(1))
.generateUpload("%s/%s".formatted(backupUser.backupDir(), BackupManager.MESSAGE_BACKUP_NAME));
.generateUpload("%s/%s".formatted(backupUser.backupDir(), BackupManager.MESSAGE_BACKUP_NAME), 17);
final BackupManager.BackupInfo info = backupManager.backupInfo(backupUser);
assertThat(info.backupSubdir()).isEqualTo(backupUser.backupDir()).isNotBlank();
@@ -247,7 +245,7 @@ public class BackupManagerTest {
final AuthenticatedBackupUser backupUser = backupUser(TestRandomUtil.nextBytes(16), BackupCredentialType.MEDIA, backupLevel);
assertThatExceptionOfType(BackupWrongCredentialTypeException.class)
.isThrownBy(() -> backupManager.createMessageBackupUploadDescriptor(backupUser));
.isThrownBy(() -> backupManager.createMessageBackupUploadDescriptor(backupUser, MAX_UPLOAD_SIZE));
}
@Test
@@ -256,21 +254,21 @@ public class BackupManagerTest {
doThrow(new RateLimitExceededException(null))
.when(mediaUploadLimiter).validate(eq(BackupManager.rateLimitKey(backupUser)));
assertThatExceptionOfType(RateLimitExceededException.class)
.isThrownBy(() -> backupManager.createTemporaryAttachmentUploadDescriptor(backupUser));
.isThrownBy(() -> backupManager.createTemporaryAttachmentUploadDescriptor(backupUser, MAX_UPLOAD_SIZE));
}
@Test
public void createTemporaryMediaAttachmentWrongTier() {
final AuthenticatedBackupUser backupUser = backupUser(TestRandomUtil.nextBytes(16), BackupCredentialType.MEDIA, BackupLevel.FREE);
assertThatExceptionOfType(BackupPermissionException.class)
.isThrownBy(() -> backupManager.createTemporaryAttachmentUploadDescriptor(backupUser));
.isThrownBy(() -> backupManager.createTemporaryAttachmentUploadDescriptor(backupUser, MAX_UPLOAD_SIZE));
}
@Test
public void createTemporaryMediaAttachmentWrongCredentialType() {
final AuthenticatedBackupUser backupUser = backupUser(TestRandomUtil.nextBytes(16), BackupCredentialType.MESSAGES, BackupLevel.PAID);
assertThatExceptionOfType(BackupWrongCredentialTypeException.class)
.isThrownBy(() -> backupManager.createTemporaryAttachmentUploadDescriptor(backupUser));
.isThrownBy(() -> backupManager.createTemporaryAttachmentUploadDescriptor(backupUser, MAX_UPLOAD_SIZE));
}
@ParameterizedTest
@@ -283,7 +281,7 @@ public class BackupManagerTest {
// create backup at t=tstart
testClock.pin(tstart);
backupManager.createMessageBackupUploadDescriptor(backupUser);
backupManager.createMessageBackupUploadDescriptor(backupUser, MAX_UPLOAD_SIZE);
// refresh at t=tnext
testClock.pin(tnext);
@@ -305,11 +303,11 @@ public class BackupManagerTest {
// create backup at t=tstart
testClock.pin(tstart);
backupManager.createMessageBackupUploadDescriptor(backupUser);
backupManager.createMessageBackupUploadDescriptor(backupUser, MAX_UPLOAD_SIZE);
// create again at t=tnext
testClock.pin(tnext);
backupManager.createMessageBackupUploadDescriptor(backupUser);
backupManager.createMessageBackupUploadDescriptor(backupUser, MAX_UPLOAD_SIZE);
checkExpectedExpirations(
tnext.truncatedTo(ChronoUnit.DAYS),
@@ -478,7 +476,7 @@ public class BackupManagerTest {
.thenReturn(slow);
final ArrayBlockingQueue<CopyResult> copyResults = new ArrayBlockingQueue<>(100);
final CompletableFuture<Void> future = backupManager
.copyToBackup(backupManager.getCopyQuota(backupUser, toCopy))
.copyToBackup(backupManager.getCopyQuota(backupUser, toCopy, MAX_UPLOAD_SIZE))
.doOnNext(copyResults::add).then().toFuture();
for (int i = 0; i < slowIndex; i++) {
@@ -525,7 +523,7 @@ public class BackupManagerTest {
new CopyParameters(3, "missing", 200, COPY_ENCRYPTION_PARAM, TestRandomUtil.nextBytes(15)),
new CopyParameters(3, "badlength", 300, COPY_ENCRYPTION_PARAM, TestRandomUtil.nextBytes(15)));
when(tusCredentialGenerator.generateUpload(any()))
when(tusCredentialGenerator.generateUpload(any(), anyLong()))
.thenReturn(new BackupUploadDescriptor(3, "", Collections.emptyMap(), ""));
when(remoteStorageManager.copy(eq(3), eq("success"), eq(100), any(), any()))
.thenReturn(CompletableFuture.completedFuture(null));
@@ -534,7 +532,8 @@ public class BackupManagerTest {
when(remoteStorageManager.copy(eq(3), eq("badlength"), eq(300), any(), any()))
.thenReturn(CompletableFuture.failedFuture(new InvalidLengthException("")));
final List<CopyResult> results = backupManager.copyToBackup(backupManager.getCopyQuota(backupUser, toCopy))
final List<CopyResult> results = backupManager
.copyToBackup(backupManager.getCopyQuota(backupUser, toCopy, MAX_UPLOAD_SIZE))
.collectList().block();
assertThat(results).hasSize(3);
@@ -871,7 +870,7 @@ public class BackupManagerTest {
.toList();
for (int i = 0; i < backupUsers.size(); i++) {
testClock.pin(days(i));
backupManager.createMessageBackupUploadDescriptor(backupUsers.get(i));
backupManager.createMessageBackupUploadDescriptor(backupUsers.get(i), MAX_UPLOAD_SIZE);
}
// set of backup-id hashes that should be expired (initially t=0)
@@ -906,11 +905,11 @@ public class BackupManagerTest {
// refreshed media timestamp at t=5
testClock.pin(days(5));
backupManager.createMessageBackupUploadDescriptor(backupUser(backupId, BackupCredentialType.MESSAGES, BackupLevel.PAID));
backupManager.createMessageBackupUploadDescriptor(backupUser(backupId, BackupCredentialType.MESSAGES, BackupLevel.PAID), MAX_UPLOAD_SIZE);
// refreshed messages timestamp at t=6
testClock.pin(days(6));
backupManager.createMessageBackupUploadDescriptor(backupUser(backupId, BackupCredentialType.MESSAGES, BackupLevel.FREE));
backupManager.createMessageBackupUploadDescriptor(backupUser(backupId, BackupCredentialType.MESSAGES, BackupLevel.FREE), MAX_UPLOAD_SIZE);
Function<Instant, List<ExpiredBackup>> getExpired = time -> backupManager
.getExpiredBackups(1, Schedulers.immediate(), time)
@@ -931,7 +930,7 @@ public class BackupManagerTest {
@EnumSource(mode = EnumSource.Mode.INCLUDE, names = {"MEDIA", "ALL"})
public void expireBackup(ExpiredBackup.ExpirationType expirationType) throws BackupException {
final AuthenticatedBackupUser backupUser = backupUser(TestRandomUtil.nextBytes(16), BackupCredentialType.MESSAGES, BackupLevel.PAID);
backupManager.createMessageBackupUploadDescriptor(backupUser);
backupManager.createMessageBackupUploadDescriptor(backupUser, MAX_UPLOAD_SIZE);
final String expectedPrefixToDelete = switch (expirationType) {
case ALL -> backupUser.backupDir();
@@ -975,7 +974,7 @@ public class BackupManagerTest {
@Test
public void deleteBackupPaginated() throws BackupException {
final AuthenticatedBackupUser backupUser = backupUser(TestRandomUtil.nextBytes(16), BackupCredentialType.MESSAGES, BackupLevel.PAID);
backupManager.createMessageBackupUploadDescriptor(backupUser);
backupManager.createMessageBackupUploadDescriptor(backupUser, MAX_UPLOAD_SIZE);
final ExpiredBackup expiredBackup = expiredBackup(ExpiredBackup.ExpirationType.MEDIA, backupUser);
final String mediaPrefix = expiredBackup.prefixToDelete() + "/";
@@ -1033,21 +1032,24 @@ public class BackupManagerTest {
}
private CopyResult copyError(final AuthenticatedBackupUser backupUser, Throwable copyException) throws BackupException {
when(tusCredentialGenerator.generateUpload(any()))
when(tusCredentialGenerator.generateUpload(any(), anyLong()))
.thenReturn(new BackupUploadDescriptor(3, "def", Collections.emptyMap(), ""));
when(remoteStorageManager.copy(eq(3), eq(COPY_PARAM.sourceKey()), eq(COPY_PARAM.sourceLength()), any(), any()))
.thenReturn(CompletableFuture.failedFuture(copyException));
return backupManager.copyToBackup(backupManager.getCopyQuota(backupUser, List.of(COPY_PARAM))).single().block();
return backupManager
.copyToBackup(backupManager.getCopyQuota(backupUser, List.of(COPY_PARAM), MAX_UPLOAD_SIZE))
.single().block();
}
private CopyResult copy(final AuthenticatedBackupUser backupUser) throws BackupException {
when(tusCredentialGenerator.generateUpload(any()))
when(tusCredentialGenerator.generateUpload(any(), anyLong()))
.thenReturn(new BackupUploadDescriptor(3, "def", Collections.emptyMap(), ""));
when(tusCredentialGenerator.generateUpload(any()))
when(tusCredentialGenerator.generateUpload(any(), anyLong()))
.thenReturn(new BackupUploadDescriptor(3, "def", Collections.emptyMap(), ""));
when(remoteStorageManager.copy(eq(3), eq(COPY_PARAM.sourceKey()), eq(COPY_PARAM.sourceLength()), any(), any()))
.thenReturn(CompletableFuture.completedFuture(null));
return backupManager.copyToBackup(backupManager.getCopyQuota(backupUser, List.of(COPY_PARAM))).single().block();
return backupManager.copyToBackup(backupManager.getCopyQuota(backupUser, List.of(COPY_PARAM), MAX_UPLOAD_SIZE))
.single().block();
}
private static ExpiredBackup expiredBackup(final ExpiredBackup.ExpirationType expirationType,

8
service/src/test/java/org/whispersystems/textsecuregcm/backup/Cdn3BackupCredentialGeneratorTest.java
View File

@@ -28,10 +28,9 @@ public class Cdn3BackupCredentialGeneratorTest {
public void uploadGenerator() {
final Cdn3BackupCredentialGenerator generator = new Cdn3BackupCredentialGenerator(new TusConfiguration(
new SecretBytes(SECRET),
"https://example.org/upload",
MAX_UPLOAD_LENGTH));
"https://example.org/upload"));
final BackupUploadDescriptor messageBackupUploadDescriptor = generator.generateUpload("subdir/key");
final BackupUploadDescriptor messageBackupUploadDescriptor = generator.generateUpload("subdir/key", MAX_UPLOAD_LENGTH);
assertThat(messageBackupUploadDescriptor.signedUploadLocation()).isEqualTo("https://example.org/upload/backups");
assertThat(messageBackupUploadDescriptor.key()).isEqualTo("subdir/key");
assertThat(messageBackupUploadDescriptor.headers()).containsKey("Authorization");
@@ -50,8 +49,7 @@ public class Cdn3BackupCredentialGeneratorTest {
public void readCredential() {
final Cdn3BackupCredentialGenerator generator = new Cdn3BackupCredentialGenerator(new TusConfiguration(
new SecretBytes(SECRET),
"https://example.org/upload",
MAX_UPLOAD_LENGTH));
"https://example.org/upload"));
final Map<String, String> headers = generator.readHeaders("subdir");
assertThat(headers).containsKey("Authorization");

77
service/src/test/java/org/whispersystems/textsecuregcm/controllers/ArchiveControllerTest.java
View File

@@ -7,6 +7,7 @@ package org.whispersystems.textsecuregcm.controllers;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
@@ -73,6 +74,7 @@ import org.whispersystems.textsecuregcm.backup.BackupManager;
import org.whispersystems.textsecuregcm.backup.BackupNotFoundException;
import org.whispersystems.textsecuregcm.backup.BackupPermissionException;
import org.whispersystems.textsecuregcm.backup.BackupUploadDescriptor;
import org.whispersystems.textsecuregcm.backup.BackupWrongCredentialTypeException;
import org.whispersystems.textsecuregcm.backup.CopyResult;
import org.whispersystems.textsecuregcm.entities.RemoteAttachment;
import org.whispersystems.textsecuregcm.mappers.BackupExceptionMapper;
@@ -90,7 +92,7 @@ import reactor.core.publisher.Flux;
@ExtendWith(DropwizardExtensionsSupport.class)
public class ArchiveControllerTest {
private static final long MAX_MESSAGE_BACKUP_OBJECT_SIZE = 1000L;
private static final long MAX_ATTACHMENT_SIZE = 1000L;
private static final AccountsManager accountsManager = mock(AccountsManager.class);
private static final BackupAuthManager backupAuthManager = mock(BackupAuthManager.class);
private static final BackupManager backupManager = mock(BackupManager.class);
@@ -106,7 +108,7 @@ public class ArchiveControllerTest {
.addProvider(new RateLimitExceededExceptionMapper())
.setMapper(SystemMapper.jsonMapper())
.setTestContainerFactory(new GrizzlyWebTestContainerFactory())
.addResource(new ArchiveController(accountsManager, backupAuthManager, backupManager, new BackupMetrics()))
.addResource(new ArchiveController(accountsManager, backupAuthManager, backupManager, new BackupMetrics(), MAX_ATTACHMENT_SIZE))
.build();
private final UUID aci = UUID.randomUUID();
@@ -118,8 +120,6 @@ public class ArchiveControllerTest {
reset(backupAuthManager);
reset(backupManager);
when(backupManager.maxMessageBackupUploadSize()).thenReturn(MAX_MESSAGE_BACKUP_OBJECT_SIZE);
when(accountsManager.getByAccountIdentifier(AuthHelper.VALID_UUID))
.thenReturn(Optional.of(AuthHelper.VALID_ACCOUNT));
}
@@ -615,8 +615,8 @@ public class ArchiveControllerTest {
static Stream<Arguments> messagesUploadForm() {
return Stream.of(
Arguments.of(Optional.empty(), true),
Arguments.of(Optional.of(MAX_MESSAGE_BACKUP_OBJECT_SIZE), true),
Arguments.of(Optional.of(MAX_MESSAGE_BACKUP_OBJECT_SIZE + 1), false)
Arguments.of(Optional.of(MAX_ATTACHMENT_SIZE), true),
Arguments.of(Optional.of(MAX_ATTACHMENT_SIZE + 1), false)
);
}
@@ -627,7 +627,7 @@ public class ArchiveControllerTest {
backupAuthTestUtil.getPresentation(BackupLevel.PAID, messagesBackupKey, aci);
when(backupManager.authenticateBackupUser(any(), any(), any()))
.thenReturn(backupUser(presentation.getBackupId(), BackupCredentialType.MESSAGES, BackupLevel.PAID));
when(backupManager.createMessageBackupUploadDescriptor(any()))
when(backupManager.createMessageBackupUploadDescriptor(any(), anyLong()))
.thenReturn(new BackupUploadDescriptor(3, "abc", Map.of("k", "v"), "example.org"));
final WebTarget builder = resources.getJerseyTest().target("v1/archives/upload/form");
@@ -641,36 +641,59 @@ public class ArchiveControllerTest {
if (expectSuccess) {
assertThat(response.getStatus()).isEqualTo(200);
ArchiveController.UploadDescriptorResponse desc = response.readEntity(ArchiveController.UploadDescriptorResponse.class);
assertThat(desc.cdn()).isEqualTo(3);
assertThat(desc.key()).isEqualTo("abc");
assertThat(desc.headers()).containsExactlyEntriesOf(Map.of("k", "v"));
assertThat(desc.signedUploadLocation()).isEqualTo("example.org");
assertThat(desc)
.isEqualTo(new ArchiveController.UploadDescriptorResponse(3, "abc", Map.of("k", "v"), "example.org"));
verify(backupManager).createMessageBackupUploadDescriptor(any(), eq(uploadLength.orElse(MAX_ATTACHMENT_SIZE)));
} else {
assertThat(response.getStatus()).isEqualTo(413);
}
}
static Stream<Arguments> mediaUploadForm() {
return Stream.of(
Arguments.of(Optional.empty(), true),
Arguments.of(Optional.of(MAX_ATTACHMENT_SIZE), true),
Arguments.of(Optional.of(MAX_ATTACHMENT_SIZE + 1), false)
);
}
@ParameterizedTest
@MethodSource
public void mediaUploadForm(Optional<Long> uploadLength, boolean expectSuccess) throws VerificationFailedException, BackupException, RateLimitExceededException {
final BackupAuthCredentialPresentation presentation =
backupAuthTestUtil.getPresentation(BackupLevel.PAID, messagesBackupKey, aci);
when(backupManager.authenticateBackupUser(any(), any(), any()))
.thenReturn(backupUser(presentation.getBackupId(), BackupCredentialType.MESSAGES, BackupLevel.PAID));
when(backupManager.createTemporaryAttachmentUploadDescriptor(any(), anyLong()))
.thenReturn(new BackupUploadDescriptor(3, "abc", Map.of("k", "v"), "example.org"));
final WebTarget builder = resources.getJerseyTest().target("v1/archives/media/upload/form");
final Response response = uploadLength
.map(length -> builder.queryParam("uploadLength", length))
.orElse(builder)
.request()
.header("X-Signal-ZK-Auth", Base64.getEncoder().encodeToString(presentation.serialize()))
.header("X-Signal-ZK-Auth-Signature", "aaa")
.get();
if (expectSuccess) {
assertThat(response.getStatus()).isEqualTo(200);
final ArchiveController.UploadDescriptorResponse desc =
response.readEntity(ArchiveController.UploadDescriptorResponse.class);
assertThat(desc)
.isEqualTo(new ArchiveController.UploadDescriptorResponse(3, "abc", Map.of("k", "v"), "example.org"));
verify(backupManager).createTemporaryAttachmentUploadDescriptor(any(), eq(uploadLength.orElse(MAX_ATTACHMENT_SIZE)));
} else {
assertThat(response.getStatus()).isEqualTo(413);
}
}
@Test
public void mediaUploadForm() throws VerificationFailedException, BackupException, RateLimitExceededException {
public void rateLimitMediaUploadForm()
throws BackupWrongCredentialTypeException, RateLimitExceededException, BackupPermissionException, VerificationFailedException, BackupFailedZkAuthenticationException {
final BackupAuthCredentialPresentation presentation =
backupAuthTestUtil.getPresentation(BackupLevel.PAID, messagesBackupKey, aci);
when(backupManager.authenticateBackupUser(any(), any(), any()))
.thenReturn(backupUser(presentation.getBackupId(), BackupCredentialType.MESSAGES, BackupLevel.PAID));
when(backupManager.createTemporaryAttachmentUploadDescriptor(any()))
.thenReturn(new BackupUploadDescriptor(3, "abc", Map.of("k", "v"), "example.org"));
final ArchiveController.UploadDescriptorResponse desc = resources.getJerseyTest()
.target("v1/archives/media/upload/form")
.request()
.header("X-Signal-ZK-Auth", Base64.getEncoder().encodeToString(presentation.serialize()))
.header("X-Signal-ZK-Auth-Signature", "aaa")
.get(ArchiveController.UploadDescriptorResponse.class);
assertThat(desc.cdn()).isEqualTo(3);
assertThat(desc.key()).isEqualTo("abc");
assertThat(desc.headers()).containsExactlyEntriesOf(Map.of("k", "v"));
assertThat(desc.signedUploadLocation()).isEqualTo("example.org");
// rate limit
when(backupManager.createTemporaryAttachmentUploadDescriptor(any())).thenThrow(new RateLimitExceededException(null));
when(backupManager.createTemporaryAttachmentUploadDescriptor(any(), anyLong())).thenThrow(new RateLimitExceededException(null));
final Response response = resources.getJerseyTest()
.target("v1/archives/media/upload/form")
.request()

6
service/src/test/java/org/whispersystems/textsecuregcm/controllers/AttachmentControllerV4Test.java
View File

@@ -92,7 +92,7 @@ class AttachmentControllerV4Test {
static {
try {
final GcsAttachmentGenerator gcsAttachmentGenerator = new GcsAttachmentGenerator("some-cdn.signal.org",
"signal@example.com", 1000, "/attach-here", RSA_PRIVATE_KEY_PEM);
"signal@example.com", "/attach-here", RSA_PRIVATE_KEY_PEM);
resources = ResourceExtension.builder()
.addProvider(AuthHelper.getAuthFilter())
.addProvider(new AuthValueFactoryProvider.Binder<>(AuthenticatedDevice.class))
@@ -100,8 +100,8 @@ class AttachmentControllerV4Test {
.setTestContainerFactory(new GrizzlyWebTestContainerFactory())
.addProvider(new AttachmentControllerV4(RATE_LIMITERS,
gcsAttachmentGenerator,
new TusAttachmentGenerator(new TusConfiguration(new SecretBytes(TUS_SECRET), TUS_URL, MAX_UPLOAD_LENGTH)),
EXPERIMENT_MANAGER))
new TusAttachmentGenerator(new TusConfiguration(new SecretBytes(TUS_SECRET), TUS_URL)),
EXPERIMENT_MANAGER, MAX_UPLOAD_LENGTH))
.build();
} catch (IOException | InvalidKeyException | InvalidKeySpecException e) {
throw new AssertionError(e);

7
service/src/test/java/org/whispersystems/textsecuregcm/grpc/AttachmentsGrpcServiceTest.java
View File

@@ -73,16 +73,17 @@ class AttachmentsGrpcServiceTest extends
Base64.getMimeEncoder().encodeToString(keyPair.getPrivate().getEncoded()) + "\n" +
"-----END PRIVATE KEY-----";
final GcsAttachmentGenerator gcsAttachmentGenerator = new GcsAttachmentGenerator(
"some-cdn.signal.org", "signal@example.com", 1000, "/attach-here", gcsPrivateKeyPem);
"some-cdn.signal.org", "signal@example.com", "/attach-here", gcsPrivateKeyPem);
final TusAttachmentGenerator tusAttachmentGenerator =
new TusAttachmentGenerator(new TusConfiguration(new SecretBytes(TUS_SECRET), TUS_URL, MAX_UPLOAD_LENGTH));
new TusAttachmentGenerator(new TusConfiguration(new SecretBytes(TUS_SECRET), TUS_URL));
return new AttachmentsGrpcService(
experimentEnrollmentManager,
MockUtils.buildMock(RateLimiters.class, rateLimiters ->
when(rateLimiters.getAttachmentLimiter()).thenReturn(rateLimiter)),
gcsAttachmentGenerator,
tusAttachmentGenerator);
tusAttachmentGenerator,
MAX_UPLOAD_LENGTH);
} catch (NoSuchAlgorithmException | IOException | InvalidKeyException | InvalidKeySpecException e) {
throw new AssertionError(e);
}

80
service/src/test/java/org/whispersystems/textsecuregcm/grpc/BackupsAnonymousGrpcServiceTest.java
View File

@@ -7,7 +7,9 @@ package org.whispersystems.textsecuregcm.grpc;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatExceptionOfType;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.when;
@@ -90,7 +92,7 @@ class BackupsAnonymousGrpcServiceTest extends
@Override
protected BackupsAnonymousGrpcService createServiceBeforeEachTest() {
return new BackupsAnonymousGrpcService(backupManager, new BackupMetrics());
return new BackupsAnonymousGrpcService(backupManager, new BackupMetrics(), MAX_MESSAGE_BACKUP_OBJECT_SIZE);
}
@BeforeEach
@@ -98,7 +100,6 @@ class BackupsAnonymousGrpcServiceTest extends
try {
when(backupManager.authenticateBackupUser(any(), any(), any()))
.thenReturn(backupUser(presentation.getBackupId(), BackupCredentialType.MESSAGES, BackupLevel.PAID));
when(backupManager.maxMessageBackupUploadSize()).thenReturn(MAX_MESSAGE_BACKUP_OBJECT_SIZE);
} catch (BackupFailedZkAuthenticationException e) {
Assertions.fail(e);
}
@@ -323,55 +324,66 @@ class BackupsAnonymousGrpcServiceTest extends
}
@Test
void mediaUploadForm() throws RateLimitExceededException, BackupException {
when(backupManager.createTemporaryAttachmentUploadDescriptor(any()))
.thenReturn(new BackupUploadDescriptor(3, "abc", Map.of("k", "v"), "example.org"));
void mediaUploadFormRateLimit() throws RateLimitExceededException, BackupException {
final GetUploadFormRequest request = GetUploadFormRequest.newBuilder()
.setMedia(GetUploadFormRequest.MediaUploadType.getDefaultInstance())
.setSignedPresentation(signedPresentation(presentation))
.setUploadLength(100)
.build();
final GetUploadFormResponse uploadForm = unauthenticatedServiceStub().getUploadForm(request);
assertThat(uploadForm.getUploadForm().getCdn()).isEqualTo(3);
assertThat(uploadForm.getUploadForm().getKey()).isEqualTo("abc");
assertThat(uploadForm.getUploadForm().getHeadersMap()).containsExactlyEntriesOf(Map.of("k", "v"));
assertThat(uploadForm.getUploadForm().getSignedUploadLocation()).isEqualTo("example.org");
// rate limit
Duration duration = Duration.ofSeconds(10);
when(backupManager.createTemporaryAttachmentUploadDescriptor(any()))
when(backupManager.createTemporaryAttachmentUploadDescriptor(any(), anyLong()))
.thenThrow(new RateLimitExceededException(duration));
GrpcTestUtils.assertRateLimitExceeded(duration, () -> unauthenticatedServiceStub().getUploadForm(request));
}
static Stream<Arguments> messagesUploadForm() {
return Stream.of(
Arguments.of(Optional.empty(), true),
Arguments.of(Optional.of(MAX_MESSAGE_BACKUP_OBJECT_SIZE), true),
Arguments.of(Optional.of(MAX_MESSAGE_BACKUP_OBJECT_SIZE + 1), false)
);
}
@ParameterizedTest
@MethodSource
public void messagesUploadForm(Optional<Long> uploadLength, boolean allowedSize) throws BackupException {
when(backupManager.createMessageBackupUploadDescriptor(any()))
.thenReturn(new BackupUploadDescriptor(3, "abc", Map.of("k", "v"), "example.org"));
final GetUploadFormRequest.MessagesUploadType.Builder builder = GetUploadFormRequest.MessagesUploadType.newBuilder();
uploadLength.ifPresent(builder::setUploadLength);
final GetUploadFormRequest request = GetUploadFormRequest.newBuilder()
.setMessages(builder.build())
@EnumSource(value = GetUploadFormRequest.UploadTypeCase.class, names = "UPLOADTYPE_NOT_SET", mode = EnumSource.Mode.EXCLUDE)
public void uploadForm(GetUploadFormRequest.UploadTypeCase uploadType)
throws BackupException, RateLimitExceededException {
final long uploadLength = 100;
final BackupUploadDescriptor result =
new BackupUploadDescriptor(3, "abc", Map.of("k", "v"), "example.org");
final GetUploadFormRequest.Builder builder = switch (uploadType) {
case MESSAGES -> {
when(backupManager.createMessageBackupUploadDescriptor(any(), eq(uploadLength)))
.thenReturn(result);
yield GetUploadFormRequest.newBuilder().setMessages(GetUploadFormRequest.MessagesUploadType.getDefaultInstance());
}
case MEDIA -> {
when(backupManager.createTemporaryAttachmentUploadDescriptor(any(), eq(uploadLength)))
.thenReturn(result);
yield GetUploadFormRequest.newBuilder().setMedia(GetUploadFormRequest.MediaUploadType.getDefaultInstance());
}
default -> throw new IllegalArgumentException("Unknown upload type: " + uploadType);
};
final GetUploadFormRequest request = builder
.setSignedPresentation(signedPresentation(presentation))
.setUploadLength(uploadLength)
.build();
final GetUploadFormResponse response = unauthenticatedServiceStub().getUploadForm(request);
if (allowedSize) {
assertThat(response.getUploadForm().getCdn()).isEqualTo(3);
assertThat(response.getUploadForm().getKey()).isEqualTo("abc");
assertThat(response.getUploadForm().getHeadersMap()).containsExactlyEntriesOf(Map.of("k", "v"));
assertThat(response.getUploadForm().getSignedUploadLocation()).isEqualTo("example.org");
} else {
assertThat(response.hasExceedsMaxUploadLength()).isTrue();
}
}
@ParameterizedTest
@EnumSource(value = GetUploadFormRequest.UploadTypeCase.class, names = "UPLOADTYPE_NOT_SET", mode = EnumSource.Mode.EXCLUDE)
public void uploadFormExceedsMax(GetUploadFormRequest.UploadTypeCase uploadType) throws BackupException {
final GetUploadFormRequest.Builder builder = switch (uploadType) {
case MESSAGES -> GetUploadFormRequest.newBuilder()
.setMessages(GetUploadFormRequest.MessagesUploadType.getDefaultInstance());
case MEDIA -> GetUploadFormRequest.newBuilder()
.setMedia(GetUploadFormRequest.MediaUploadType.getDefaultInstance());
default -> throw new IllegalArgumentException("Unknown upload type: " + uploadType);
};
final GetUploadFormRequest request = builder
.setSignedPresentation(signedPresentation(presentation))
.setUploadLength(MAX_MESSAGE_BACKUP_OBJECT_SIZE + 1)
.build();
final GetUploadFormResponse response = unauthenticatedServiceStub().getUploadForm(request);
assertThat(response.hasExceedsMaxUploadLength()).isTrue();
}

5
service/src/test/resources/config/test.yml
View File

@@ -239,17 +239,18 @@ messageCache: # Redis server configuration for message store cache
cluster:
type: local
attachments:
maxUploadSizeInBytes: 1024
gcpAttachments: # GCP Storage configuration
domain: example.com
email: user@example.cocm
maxSizeInBytes: 1024
pathPrefix:
rsaSigningKey: secret://gcpAttachments.rsaSigningKey
tus:
uploadUri: https://example.org/upload
userAuthenticationTokenSharedSecret: secret://tus.userAuthenticationTokenSharedSecret
maxSizeInBytes: 1024
apn: # Apple Push Notifications configuration
sandbox: true