Signal/Signal-Server
1
0
Fork 0
mirror of https://github.com/signalapp/Signal-Server synced 2026-04-20 09:57:59 +01:00
Code Issues Packages Projects Releases Wiki Activity

Add new upload-for-copy backup endpoint

Browse Source
This commit is contained in:
ravi-signal
2024-04-15 13:47:46 -05:00
committed by GitHub
parent e5d654f0c7
commit d36df3eaa9
13 changed files with 202 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

104
service/src/main/java/org/whispersystems/textsecuregcm/backup/BackupManager.java
View File

@@ -11,6 +11,7 @@ import io.micrometer.core.instrument.DistributionSummary;
import io.micrometer.core.instrument.Metrics;
import java.io.IOException;
import java.net.URI;
import java.security.SecureRandom;
import java.time.Clock;
import java.time.Duration;
import java.time.Instant;
@@ -30,7 +31,12 @@ import org.signal.libsignal.zkgroup.VerificationFailedException;
import org.signal.libsignal.zkgroup.backups.BackupAuthCredentialPresentation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.whispersystems.textsecuregcm.attachments.AttachmentGenerator;
import org.whispersystems.textsecuregcm.attachments.TusAttachmentGenerator;
import org.whispersystems.textsecuregcm.auth.AuthenticatedBackupUser;
import org.whispersystems.textsecuregcm.controllers.RateLimitExceededException;
import org.whispersystems.textsecuregcm.limits.RateLimiter;
import org.whispersystems.textsecuregcm.limits.RateLimiters;
import org.whispersystems.textsecuregcm.metrics.MetricsUtil;
import org.whispersystems.textsecuregcm.util.ExceptionUtils;
import reactor.core.publisher.Flux;
@@ -59,21 +65,28 @@ public class BackupManager {
private final BackupsDb backupsDb;
private final GenericServerSecretParams serverSecretParams;
private final RateLimiters rateLimiters;
private final TusAttachmentGenerator tusAttachmentGenerator;
private final Cdn3BackupCredentialGenerator cdn3BackupCredentialGenerator;
private final RemoteStorageManager remoteStorageManager;
private final Map<Integer, String> attachmentCdnBaseUris;
private final SecureRandom secureRandom = new SecureRandom();
private final Clock clock;
public BackupManager(
final BackupsDb backupsDb,
final GenericServerSecretParams serverSecretParams,
final RateLimiters rateLimiters,
final TusAttachmentGenerator tusAttachmentGenerator,
final Cdn3BackupCredentialGenerator cdn3BackupCredentialGenerator,
final RemoteStorageManager remoteStorageManager,
final Map<Integer, String> attachmentCdnBaseUris,
final Clock clock) {
this.backupsDb = backupsDb;
this.serverSecretParams = serverSecretParams;
this.rateLimiters = rateLimiters;
this.tusAttachmentGenerator = tusAttachmentGenerator;
this.cdn3BackupCredentialGenerator = cdn3BackupCredentialGenerator;
this.remoteStorageManager = remoteStorageManager;
this.clock = clock;
@@ -131,26 +144,38 @@ public class BackupManager {
* @param backupUser an already ZK authenticated backup user
* @return the upload form
*/
public CompletableFuture<MessageBackupUploadDescriptor> createMessageBackupUploadDescriptor(
public CompletableFuture<BackupUploadDescriptor> createMessageBackupUploadDescriptor(
final AuthenticatedBackupUser backupUser) {
checkBackupTier(backupUser, BackupTier.MESSAGES);
// this could race with concurrent updates, but the only effect would be last-writer-wins on the timestamp
return backupsDb
.addMessageBackup(backupUser)
.thenApply(result -> cdn3BackupCredentialGenerator.generateUpload(cdnMessageBackupName(backupUser)));
}
public BackupUploadDescriptor createTemporaryAttachmentUploadDescriptor(final AuthenticatedBackupUser backupUser)
throws RateLimitExceededException {
checkBackupTier(backupUser, BackupTier.MEDIA);
RateLimiter.adaptLegacyException(() -> rateLimiters
.forDescriptor(RateLimiters.For.BACKUP_ATTACHMENT)
.validate(rateLimitKey(backupUser)));
final byte[] bytes = new byte[15];
secureRandom.nextBytes(bytes);
final String attachmentKey = Base64.getUrlEncoder().encodeToString(bytes);
final AttachmentGenerator.Descriptor descriptor = tusAttachmentGenerator.generateAttachment(attachmentKey);
return new BackupUploadDescriptor(3, attachmentKey, descriptor.headers(), descriptor.signedUploadLocation());
}
/**
* Update the last update timestamps for the backupId in the presentation
*
* @param backupUser an already ZK authenticated backup user
*/
public CompletableFuture<Void> ttlRefresh(final AuthenticatedBackupUser backupUser) {
if (backupUser.backupTier().compareTo(BackupTier.MESSAGES) < 0) {
Metrics.counter(ZK_AUTHZ_FAILURE_COUNTER_NAME).increment();
throw Status.PERMISSION_DENIED
.withDescription("credential does not support ttl operation")
.asRuntimeException();
}
checkBackupTier(backupUser, BackupTier.MESSAGES);
// update message backup TTL
return backupsDb.ttlRefresh(backupUser);
}
@@ -165,11 +190,7 @@ public class BackupManager {
* @return Information about the existing backup
*/
public CompletableFuture<BackupInfo> backupInfo(final AuthenticatedBackupUser backupUser) {
if (backupUser.backupTier().compareTo(BackupTier.MESSAGES) < 0) {
Metrics.counter(ZK_AUTHZ_FAILURE_COUNTER_NAME).increment();
throw Status.PERMISSION_DENIED.withDescription("credential does not support info operation")
.asRuntimeException();
}
checkBackupTier(backupUser, BackupTier.MESSAGES);
return backupsDb.describeBackup(backupUser)
.thenApply(backupDescription -> new BackupInfo(
backupDescription.cdn(),
@@ -187,12 +208,7 @@ public class BackupManager {
* @return true if mediaLength bytes can be stored
*/
public CompletableFuture<Boolean> canStoreMedia(final AuthenticatedBackupUser backupUser, final long mediaLength) {
if (backupUser.backupTier().compareTo(BackupTier.MEDIA) < 0) {
Metrics.counter(ZK_AUTHZ_FAILURE_COUNTER_NAME).increment();
throw Status.PERMISSION_DENIED
.withDescription("credential does not support storing media")
.asRuntimeException();
}
checkBackupTier(backupUser, BackupTier.MEDIA);
return backupsDb.getMediaUsage(backupUser)
.thenComposeAsync(info -> {
final boolean canStore = MAX_TOTAL_BACKUP_MEDIA_BYTES - info.usageInfo().bytesUsed() >= mediaLength;
@@ -243,12 +259,7 @@ public class BackupManager {
final int sourceLength,
final MediaEncryptionParameters encryptionParameters,
final byte[] destinationMediaId) {
if (backupUser.backupTier().compareTo(BackupTier.MEDIA) < 0) {
Metrics.counter(ZK_AUTHZ_FAILURE_COUNTER_NAME).increment();
throw Status.PERMISSION_DENIED
.withDescription("credential does not support storing media")
.asRuntimeException();
}
checkBackupTier(backupUser, BackupTier.MEDIA);
if (sourceLength > MAX_MEDIA_OBJECT_SIZE) {
throw Status.INVALID_ARGUMENT
.withDescription("Invalid sourceObject size")
@@ -262,7 +273,7 @@ public class BackupManager {
return CompletableFuture.failedFuture(e);
}
final MessageBackupUploadDescriptor dst = cdn3BackupCredentialGenerator.generateUpload(
final BackupUploadDescriptor dst = cdn3BackupCredentialGenerator.generateUpload(
cdnMediaPath(backupUser, destinationMediaId));
final int destinationLength = encryptionParameters.outputSize(sourceLength);
@@ -309,12 +320,7 @@ public class BackupManager {
* @return A map of headers to include with CDN requests
*/
public Map<String, String> generateReadAuth(final AuthenticatedBackupUser backupUser) {
if (backupUser.backupTier().compareTo(BackupTier.MESSAGES) < 0) {
Metrics.counter(ZK_AUTHZ_FAILURE_COUNTER_NAME).increment();
throw Status.PERMISSION_DENIED
.withDescription("credential does not support read auth operation")
.asRuntimeException();
}
checkBackupTier(backupUser, BackupTier.MESSAGES);
return cdn3BackupCredentialGenerator.readHeaders(backupUser.backupDir());
}
@@ -339,12 +345,7 @@ public class BackupManager {
final AuthenticatedBackupUser backupUser,
final Optional<String> cursor,
final int limit) {
if (backupUser.backupTier().compareTo(BackupTier.MESSAGES) < 0) {
Metrics.counter(ZK_AUTHZ_FAILURE_COUNTER_NAME).increment();
throw Status.PERMISSION_DENIED
.withDescription("credential does not support list operation")
.asRuntimeException();
}
checkBackupTier(backupUser, BackupTier.MESSAGES);
return remoteStorageManager.list(cdnMediaDirectory(backupUser), cursor, limit)
.thenApply(result ->
new ListMediaResult(
@@ -370,12 +371,7 @@ public class BackupManager {
public CompletableFuture<Void> delete(final AuthenticatedBackupUser backupUser,
final List<StorageDescriptor> storageDescriptors) {
if (backupUser.backupTier().compareTo(BackupTier.MESSAGES) < 0) {
Metrics.counter(ZK_AUTHZ_FAILURE_COUNTER_NAME).increment();
throw Status.PERMISSION_DENIED
.withDescription("credential does not support list operation")
.asRuntimeException();
}
checkBackupTier(backupUser, BackupTier.MESSAGES);
if (storageDescriptors.stream().anyMatch(sd -> sd.cdn() != remoteStorageManager.cdnNumber())) {
throw Status.INVALID_ARGUMENT
@@ -430,6 +426,7 @@ public class BackupManager {
}
private static final ECPublicKey INVALID_PUBLIC_KEY = Curve.generateKeyPair().getPublicKey();
/**
* Authenticate the ZK anonymous backup credential's presentation
* <p>
@@ -532,6 +529,7 @@ public class BackupManager {
}
interface PresentationSignatureVerifier {
BackupTier verifySignature(byte[] signature, ECPublicKey publicKey);
}
@@ -576,6 +574,22 @@ public class BackupManager {
};
}
/**
* Check that the authenticated backup user is authorized to use the provided backupTier
*
* @param backupUser The backup user to check
* @param backupTier The authorization level to verify the backupUser has access to
* @throws {@link Status#PERMISSION_DENIED} error if the backup user is not authorized to access {@code backupTier}
*/
private static void checkBackupTier(final AuthenticatedBackupUser backupUser, final BackupTier backupTier) {
if (backupUser.backupTier().compareTo(backupTier) < 0) {
Metrics.counter(ZK_AUTHZ_FAILURE_COUNTER_NAME).increment();
throw Status.PERMISSION_DENIED
.withDescription("credential does not support the requested operation")
.asRuntimeException();
}
}
@VisibleForTesting
static String encodeMediaIdForCdn(final byte[] bytes) {
return Base64.getUrlEncoder().encodeToString(bytes);
@@ -596,4 +610,8 @@ public class BackupManager {
private static String cdnMediaPath(final AuthenticatedBackupUser backupUser, final byte[] mediaId) {
return "%s%s".formatted(cdnMediaDirectory(backupUser), encodeMediaIdForCdn(mediaId));
}
static String rateLimitKey(final AuthenticatedBackupUser backupUser) {
return Base64.getEncoder().encodeToString(BackupsDb.hashedBackupId(backupUser.backupId()));
}
}

2
service/src/main/java/org/whispersystems/textsecuregcm/backup/MessageBackupUploadDescriptor.java → service/src/main/java/org/whispersystems/textsecuregcm/backup/BackupUploadDescriptor.java
View File

@@ -7,7 +7,7 @@ package org.whispersystems.textsecuregcm.backup;
import java.util.Map;
public record MessageBackupUploadDescriptor(
public record BackupUploadDescriptor(
int cdn,
String key,
Map<String, String> headers,

4
service/src/main/java/org/whispersystems/textsecuregcm/backup/Cdn3BackupCredentialGenerator.java
View File

@@ -49,7 +49,7 @@ public class Cdn3BackupCredentialGenerator {
.build();
}
public MessageBackupUploadDescriptor generateUpload(final String key) {
public BackupUploadDescriptor generateUpload(final String key) {
if (key.isBlank()) {
throw new IllegalArgumentException("Upload descriptors must have non-empty keys");
}
@@ -60,7 +60,7 @@ public class Cdn3BackupCredentialGenerator {
HttpHeaders.AUTHORIZATION, HeaderUtils.basicAuthHeader(credentials),
"Upload-Metadata", String.format("filename %s", b64Key));
return new MessageBackupUploadDescriptor(
return new BackupUploadDescriptor(
BACKUP_CDN,
key,
headers,

4
service/src/main/java/org/whispersystems/textsecuregcm/backup/Cdn3RemoteStorageManager.java
View File

@@ -113,7 +113,7 @@ public class Cdn3RemoteStorageManager implements RemoteStorageManager {
final URI sourceUri,
final int expectedSourceLength,
final MediaEncryptionParameters encryptionParameters,
final MessageBackupUploadDescriptor uploadDescriptor) {
final BackupUploadDescriptor uploadDescriptor) {
if (uploadDescriptor.cdn() != cdnNumber()) {
throw new IllegalArgumentException("Cdn3RemoteStorageManager can only copy to cdn3");
@@ -152,7 +152,7 @@ public class Cdn3RemoteStorageManager implements RemoteStorageManager {
private HttpRequest createCopyRequest(
final int expectedSourceLength,
final MessageBackupUploadDescriptor uploadDescriptor,
final BackupUploadDescriptor uploadDescriptor,
BackupMediaEncrypter encrypter,
HttpResponse<Flow.Publisher<List<ByteBuffer>>> response) throws IOException {
if (response.statusCode() == Response.Status.NOT_FOUND.getStatusCode()) {

2
service/src/main/java/org/whispersystems/textsecuregcm/backup/RemoteStorageManager.java
View File

@@ -36,7 +36,7 @@ public interface RemoteStorageManager {
URI sourceUri,
int expectedSourceLength,
MediaEncryptionParameters encryptionParameters,
MessageBackupUploadDescriptor uploadDescriptor);
BackupUploadDescriptor uploadDescriptor);
/**
* Result of a {@link #list} operation