Signal/Signal-Server
1
0
Fork 0
mirror of https://github.com/signalapp/Signal-Server synced 2026-04-20 19:28:10 +01:00
Code Issues Packages Projects Releases Wiki Activity

Make copy/delete streaming friendly

Browse Source
This commit is contained in:
ravi-signal
2024-06-20 16:00:09 -05:00
committed by GitHub
parent c27898a993
commit 4aadabfac0
6 changed files with 436 additions and 321 deletions
Download Patch File Download Diff File Expand all files Collapse all files

281
service/src/main/java/org/whispersystems/textsecuregcm/backup/BackupManager.java
View File

@@ -14,13 +14,14 @@ import java.security.SecureRandom;
import java.time.Clock;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Base64;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionStage;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Function;
import org.signal.libsignal.protocol.ecc.Curve;
import org.signal.libsignal.protocol.ecc.ECPublicKey;
import org.signal.libsignal.zkgroup.GenericServerSecretParams;
@@ -32,7 +33,6 @@ import org.slf4j.LoggerFactory;
import org.whispersystems.textsecuregcm.attachments.AttachmentGenerator;
import org.whispersystems.textsecuregcm.attachments.TusAttachmentGenerator;
import org.whispersystems.textsecuregcm.auth.AuthenticatedBackupUser;
import org.whispersystems.textsecuregcm.controllers.RateLimitExceededException;
import org.whispersystems.textsecuregcm.limits.RateLimiter;
import org.whispersystems.textsecuregcm.limits.RateLimiters;
import org.whispersystems.textsecuregcm.metrics.MetricsUtil;
@@ -56,6 +56,9 @@ public class BackupManager {
// How many cdn object deletion requests can be outstanding at a time per backup deletion operation
private static final int DELETION_CONCURRENCY = 10;
// How many cdn object copy requests can be outstanding at a time per batch copy-to-backup operation
private static final int COPY_CONCURRENCY = 10;
private static final String ZK_AUTHN_COUNTER_NAME = MetricsUtil.name(BackupManager.class, "authentication");
private static final String ZK_AUTHZ_FAILURE_COUNTER_NAME = MetricsUtil.name(BackupManager.class,
@@ -148,7 +151,7 @@ public class BackupManager {
.thenApply(result -> cdn3BackupCredentialGenerator.generateUpload(cdnMessageBackupName(backupUser)));
}
public CompletionStage<BackupUploadDescriptor> createTemporaryAttachmentUploadDescriptor(
public CompletableFuture<BackupUploadDescriptor> createTemporaryAttachmentUploadDescriptor(
final AuthenticatedBackupUser backupUser) {
checkBackupLevel(backupUser, BackupLevel.MEDIA);
@@ -160,7 +163,7 @@ public class BackupManager {
final String attachmentKey = Base64.getUrlEncoder().encodeToString(bytes);
final AttachmentGenerator.Descriptor descriptor = tusAttachmentGenerator.generateAttachment(attachmentKey);
return new BackupUploadDescriptor(3, attachmentKey, descriptor.headers(), descriptor.signedUploadLocation());
});
}).toCompletableFuture();
}
/**
@@ -195,19 +198,96 @@ public class BackupManager {
}
/**
* Check if there is enough capacity to store the requested amount of media
* Copy an encrypted object to the backup cdn, adding a layer of encryption
* <p>
* Implementation notes: <p> This method guarantees that any object that gets successfully copied to the backup cdn
* will also be deducted from the user's quota. </p>
* <p>
* However, the converse isn't true. It's possible we may charge the user for media they failed to copy. As a result,
* the quota may be over reported. It should be recalculated before taking quota enforcement actions.
*
* @param backupUser an already ZK authenticated backup user
* @param mediaLength the desired number of media bytes to store
* @return true if mediaLength bytes can be stored
* @return A Flux that emits the locations of the double-encrypted objects on the backup cdn, or includes an error
* detailing why the object could not be copied.
*/
public CompletableFuture<Boolean> canStoreMedia(final AuthenticatedBackupUser backupUser, final long mediaLength) {
public Flux<CopyResult> copyToBackup(final AuthenticatedBackupUser backupUser, List<CopyParameters> toCopy) {
checkBackupLevel(backupUser, BackupLevel.MEDIA);
return Mono
// Figure out how many objects we're allowed to copy, updating the quota usage for the amount we are allowed
.fromFuture(enforceQuota(backupUser, toCopy))
// Copy the ones we have enough quota to hold
.flatMapMany(quotaResult -> Flux.concat(
// These fit in our remaining quota, so perform the copy. If the copy fails, our estimated quota usage may not
// be exact since we already updated our usage. We make a best-effort attempt to undo the usage update if we
// know that the copied failed for sure though.
Flux.fromIterable(quotaResult.requestsToCopy()).flatMapSequential(
copyParams -> copyToBackup(backupUser, copyParams)
.flatMap(copyResult -> switch (copyResult.outcome()) {
case SUCCESS -> Mono.just(copyResult);
case SOURCE_WRONG_LENGTH, SOURCE_NOT_FOUND, OUT_OF_QUOTA -> Mono
.fromFuture(this.backupsDb.trackMedia(backupUser, -1, -copyParams.destinationObjectSize()))
.thenReturn(copyResult);
}),
COPY_CONCURRENCY),
// There wasn't enough quota remaining to perform these copies
Flux.fromIterable(quotaResult.requestsToReject())
.map(arg -> new CopyResult(CopyResult.Outcome.OUT_OF_QUOTA, arg.destinationMediaId(), null))));
}
private Mono<CopyResult> copyToBackup(final AuthenticatedBackupUser backupUser, final CopyParameters copyParameters) {
return Mono.fromCompletionStage(() -> remoteStorageManager.copy(
copyParameters.sourceCdn(), copyParameters.sourceKey(), copyParameters.sourceLength(),
copyParameters.encryptionParameters(),
cdnMediaPath(backupUser, copyParameters.destinationMediaId())))
// Successfully copied!
.thenReturn(new CopyResult(
CopyResult.Outcome.SUCCESS, copyParameters.destinationMediaId(), remoteStorageManager.cdnNumber()))
// Otherwise, squash per-item copy errors that don't fail the entire operation
.onErrorResume(
// If the error maps to an explicit result type
throwable ->
CopyResult.fromCopyError(throwable, copyParameters.destinationMediaId()).isPresent(),
// return that result type instead of propagating the error
throwable ->
Mono.just(CopyResult.fromCopyError(throwable, copyParameters.destinationMediaId()).orElseThrow()));
}
private record QuotaResult(List<CopyParameters> requestsToCopy, List<CopyParameters> requestsToReject) {}
/**
* Determine which copy requests can be performed with the user's remaining quota and update the used quota. If a copy
* request subsequently fails, the caller should attempt to restore the quota for the failed copy.
*
* @param backupUser The user quota to update
* @param toCopy The proposed copy requests
* @return QuotaResult indicating which requests fit into the remaining quota and which requests should be rejected
* with {@link CopyResult.Outcome#OUT_OF_QUOTA}
*/
private CompletableFuture<QuotaResult> enforceQuota(
final AuthenticatedBackupUser backupUser,
final List<CopyParameters> toCopy) {
final long totalBytesAdded = toCopy.stream()
.mapToLong(copyParameters -> {
if (copyParameters.sourceLength() > MAX_MEDIA_OBJECT_SIZE) {
throw Status.INVALID_ARGUMENT
.withDescription("Invalid sourceObject size")
.asRuntimeException();
}
return copyParameters.destinationObjectSize();
})
.sum();
return backupsDb.getMediaUsage(backupUser)
.thenComposeAsync(info -> {
final boolean canStore = MAX_TOTAL_BACKUP_MEDIA_BYTES - info.usageInfo().bytesUsed() >= mediaLength;
long remainingQuota = MAX_TOTAL_BACKUP_MEDIA_BYTES - info.usageInfo().bytesUsed();
final boolean canStore = remainingQuota >= totalBytesAdded;
if (canStore || info.lastRecalculationTime().isAfter(clock.instant().minus(MAX_QUOTA_STALENESS))) {
return CompletableFuture.completedFuture(canStore);
return CompletableFuture.completedFuture(remainingQuota);
}
// The user is out of quota, and we have not recently recalculated the user's usage. Double check by doing a
@@ -221,69 +301,46 @@ public class BackupManager {
Metrics.counter(USAGE_RECALCULATION_COUNTER_NAME, "usageChanged", String.valueOf(usageChanged))
.increment();
})
.thenApply(newUsage -> MAX_TOTAL_BACKUP_MEDIA_BYTES - newUsage.bytesUsed() >= mediaLength);
.thenApply(newUsage -> MAX_TOTAL_BACKUP_MEDIA_BYTES - newUsage.bytesUsed());
})
.thenCompose(remainingQuota -> {
// Figure out how many of the requested objects fit in the remaining quota
final int index = indexWhereTotalExceeds(toCopy, CopyParameters::destinationObjectSize,
remainingQuota);
final QuotaResult result = new QuotaResult(toCopy.subList(0, index),
toCopy.subList(index, toCopy.size()));
if (index == 0) {
// Skip the usage update if we're not able to write anything
return CompletableFuture.completedFuture(result);
}
// Update the usage
final long quotaToConsume = result.requestsToCopy.stream()
.mapToLong(CopyParameters::destinationObjectSize)
.sum();
return backupsDb.trackMedia(backupUser, index, quotaToConsume).thenApply(ignored -> result);
});
}
/**
* @return the largest index i such that sum(ts[0],...ts[i - 1]) <= max
*/
private static <T> int indexWhereTotalExceeds(List<T> ts, Function<T, Long> valueFunction, long max) {
long sum = 0;
for (int index = 0; index < ts.size(); index++) {
sum += valueFunction.apply(ts.get(index));
if (sum > max) {
return index;
}
}
return ts.size();
}
public record StorageDescriptor(int cdn, byte[] key) {}
public record StorageDescriptorWithLength(int cdn, byte[] key, long length) {}
/**
* Copy an encrypted object to the backup cdn, adding a layer of encryption
* <p>
* Implementation notes: <p> This method guarantees that any object that gets successfully copied to the backup cdn
* will also be deducted from the user's quota. </p>
* <p>
* However, the converse isn't true. It's possible we may charge the user for media they failed to copy. As a result,
* the quota may be over reported and it should be recalculated before taking quota enforcement actions.
*
* @return A stage that completes successfully with location of the twice-encrypted object on the backup cdn. The
* returned CompletionStage can be completed exceptionally with the following exceptions.
* <ul>
* <li> {@link InvalidLengthException} If the expectedSourceLength does not match the length of the sourceUri </li>
* <li> {@link SourceObjectNotFoundException} If the no object at sourceUri is found </li>
* <li> {@link java.io.IOException} If there was a generic IO issue </li>
* </ul>
*/
public CompletableFuture<StorageDescriptor> copyToBackup(
final AuthenticatedBackupUser backupUser,
final int sourceCdn,
final String sourceKey,
final int sourceLength,
final MediaEncryptionParameters encryptionParameters,
final byte[] destinationMediaId) {
checkBackupLevel(backupUser, BackupLevel.MEDIA);
if (sourceLength > MAX_MEDIA_OBJECT_SIZE) {
throw Status.INVALID_ARGUMENT
.withDescription("Invalid sourceObject size")
.asRuntimeException();
}
final String destination = cdnMediaPath(backupUser, destinationMediaId);
final int destinationLength = encryptionParameters.outputSize(sourceLength);
return this.backupsDb
// Write the ddb updates before actually updating backing storage
.trackMedia(backupUser, 1, destinationLength)
// Actually copy the objects. If the copy fails, our estimated quota usage may not be exact
.thenComposeAsync(ignored ->
remoteStorageManager.copy(sourceCdn, sourceKey, sourceLength, encryptionParameters, destination))
.exceptionallyCompose(throwable -> {
final Throwable unwrapped = ExceptionUtils.unwrap(throwable);
if (!(unwrapped instanceof SourceObjectNotFoundException) && !(unwrapped instanceof InvalidLengthException)) {
throw ExceptionUtils.wrap(unwrapped);
}
// In cases where we know the copy fails without writing anything, we can try to restore the user's quota
return this.backupsDb.trackMedia(backupUser, -1, -destinationLength).whenComplete((ignored, ignoredEx) -> {
throw ExceptionUtils.wrap(unwrapped);
});
})
// indicates where the backup was stored
.thenApply(ignore -> new StorageDescriptor(remoteStorageManager.cdnNumber(), destinationMediaId));
}
/**
* Generate credentials that can be used to read from the backup CDN
*
@@ -348,66 +405,60 @@ public class BackupManager {
deletePrefix(backupUser.backupDir(), DELETION_CONCURRENCY))));
}
private sealed interface Either permits DeleteSuccess, DeleteFailure {}
private record DeleteSuccess(long usage) implements Either {}
private record DeleteFailure(Throwable e) implements Either {}
public CompletableFuture<Void> delete(final AuthenticatedBackupUser backupUser,
public Flux<StorageDescriptor> deleteMedia(final AuthenticatedBackupUser backupUser,
final List<StorageDescriptor> storageDescriptors) {
checkBackupLevel(backupUser, BackupLevel.MESSAGES);
// Check for a cdn we don't know how to process
if (storageDescriptors.stream().anyMatch(sd -> sd.cdn() != remoteStorageManager.cdnNumber())) {
throw Status.INVALID_ARGUMENT
.withDescription("unsupported media cdn provided")
.asRuntimeException();
}
return Flux
.fromIterable(storageDescriptors)
return Flux.usingWhen(
// Issue deletes for all storage descriptors (proceeds with default flux concurrency)
.flatMap(descriptor -> Mono.fromCompletionStage(
remoteStorageManager
.delete(cdnMediaPath(backupUser, descriptor.key))
// Squash errors/success into a single type
.handle((bytesDeleted, throwable) -> throwable != null
? new DeleteFailure(throwable)
: new DeleteSuccess(bytesDeleted))
))
// Gather usage updates into the UsageBatcher to apply during the cleanup operation
Mono.just(new UsageBatcher()),
// Update backupsDb with the change in usage
.collectList()
.<Void>flatMap(eithers -> {
// count up usage changes
long totalBytesDeleted = 0;
long totalCountDeleted = 0;
final List<Throwable> toThrow = new ArrayList<>();
for (Either either : eithers) {
switch (either) {
case DeleteFailure f:
toThrow.add(f.e());
break;
case DeleteSuccess s when s.usage() > 0:
totalBytesDeleted += s.usage();
totalCountDeleted++;
break;
default:
break;
}
}
final Mono<Void> result = toThrow.isEmpty()
? Mono.empty()
: Mono.error(toThrow.stream().reduce((t1, t2) -> {
t1.addSuppressed(t2);
return t1;
}).get());
return Mono
.fromCompletionStage(this.backupsDb.trackMedia(backupUser, -totalCountDeleted, -totalBytesDeleted))
.then(result);
})
.toFuture();
// Deletes the objects, returning their former location. Tracks bytes removed so the quota can be updated on
// completion
batcher -> Flux.fromIterable(storageDescriptors)
.flatMapSequential(sd -> Mono
// Delete the object
.fromCompletionStage(remoteStorageManager.delete(cdnMediaPath(backupUser, sd.key())))
// Track how much the remote storage manager indicated was deleted as part of the operation
.doOnNext(deletedBytes -> batcher.update(-deletedBytes))
.thenReturn(sd), DELETION_CONCURRENCY),
// On cleanup, update the quota using whatever updates were accumulated in the batcher
batcher ->
Mono.fromFuture(backupsDb.trackMedia(backupUser, batcher.countDelta.get(), batcher.usageDelta.get())));
}
/**
* Track pending media usage updates
*/
private static class UsageBatcher {
AtomicLong countDelta = new AtomicLong();
AtomicLong usageDelta = new AtomicLong();
/**
* Stage a usage update that will be applied later
*
* @param bytesDelta The amount of bytes that should be tracked as used (or if negative, freed). If the delta is
* non-zero, the count will also be updated.
*/
void update(long bytesDelta) {
if (bytesDelta < 0) {
countDelta.decrementAndGet();
} else if (bytesDelta > 0) {
countDelta.incrementAndGet();
}
usageDelta.addAndGet(bytesDelta);
}
}
private static final ECPublicKey INVALID_PUBLIC_KEY = Curve.generateKeyPair().getPublicKey();

29
service/src/main/java/org/whispersystems/textsecuregcm/backup/CopyParameters.java Normal file
View File

@@ -0,0 +1,29 @@
/*
* Copyright 2024 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.backup;
/**
* Descriptor for a single copy-and-encrypt operation
*
* @param sourceCdn The cdn of the object to copy
* @param sourceKey The mediaId within the cdn of the object to copy
* @param sourceLength The length of the object to copy
* @param encryptionParameters Encryption parameters to double encrypt the object
* @param destinationMediaId The mediaId of the destination object
*/
public record CopyParameters(
int sourceCdn,
String sourceKey,
int sourceLength,
MediaEncryptionParameters encryptionParameters,
byte[] destinationMediaId) {
/**
* @return The size of the double-encrypted destination object after it is copied
*/
long destinationObjectSize() {
return encryptionParameters().outputSize(sourceLength());
}
}

46
service/src/main/java/org/whispersystems/textsecuregcm/backup/CopyResult.java Normal file
View File

@@ -0,0 +1,46 @@
/*
* Copyright 2024 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.backup;
import org.whispersystems.textsecuregcm.util.ExceptionUtils;
import javax.annotation.Nullable;
import java.util.Optional;
/**
* The result of a copy operation
*
* @param outcome Whether the copy was a success
* @param mediaId The destination mediaId
* @param cdn On success, the destination cdn
*/
public record CopyResult(Outcome outcome, byte[] mediaId, @Nullable Integer cdn) {
public enum Outcome {
SUCCESS,
SOURCE_NOT_FOUND,
SOURCE_WRONG_LENGTH,
OUT_OF_QUOTA
}
/**
* Map an exception returned by {@link RemoteStorageManager#copy} to CopyResult with the appropriate outcome.
*
* @param throwable result of a failed copy operation
* @param key the copy destination mediaId
* @return The appropriate CopyResult, or empty if the exception does not match to an Outcome.
*/
static Optional<CopyResult> fromCopyError(final Throwable throwable, final byte[] key) {
final Throwable unwrapped = ExceptionUtils.unwrap(throwable);
if (unwrapped instanceof SourceObjectNotFoundException) {
return Optional.of(new CopyResult(Outcome.SOURCE_NOT_FOUND, key, null));
} else if (unwrapped instanceof InvalidLengthException) {
return Optional.of(new CopyResult(Outcome.SOURCE_WRONG_LENGTH, key, null));
} else {
return Optional.empty();
}
}
}

136
service/src/main/java/org/whispersystems/textsecuregcm/controllers/ArchiveController.java
View File

@@ -27,8 +27,8 @@ import java.util.Base64;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionStage;
import java.util.stream.Stream;
import javax.validation.Valid;
import javax.validation.constraints.Max;
import javax.validation.constraints.Min;
@@ -54,22 +54,19 @@ import org.signal.libsignal.zkgroup.backups.BackupAuthCredentialPresentation;
import org.signal.libsignal.zkgroup.backups.BackupAuthCredentialRequest;
import org.signal.libsignal.zkgroup.receipts.ReceiptCredentialPresentation;
import org.whispersystems.textsecuregcm.auth.AuthenticatedAccount;
import org.whispersystems.textsecuregcm.auth.AuthenticatedBackupUser;
import org.whispersystems.textsecuregcm.backup.BackupAuthManager;
import org.whispersystems.textsecuregcm.backup.BackupManager;
import org.whispersystems.textsecuregcm.backup.InvalidLengthException;
import org.whispersystems.textsecuregcm.backup.CopyParameters;
import org.whispersystems.textsecuregcm.backup.CopyResult;
import org.whispersystems.textsecuregcm.backup.MediaEncryptionParameters;
import org.whispersystems.textsecuregcm.backup.SourceObjectNotFoundException;
import org.whispersystems.textsecuregcm.util.BackupAuthCredentialAdapter;
import org.whispersystems.textsecuregcm.util.ByteArrayAdapter;
import org.whispersystems.textsecuregcm.util.ByteArrayBase64UrlAdapter;
import org.whispersystems.textsecuregcm.util.ECPublicKeyAdapter;
import org.whispersystems.textsecuregcm.util.ExactlySize;
import org.whispersystems.textsecuregcm.util.ExceptionUtils;
import org.whispersystems.textsecuregcm.util.Util;
import org.whispersystems.websocket.auth.Mutable;
import org.whispersystems.websocket.auth.ReadOnly;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
@Path("/v1/archives")
@@ -124,7 +121,7 @@ public class ArchiveController {
public record RedeemBackupReceiptRequest(
@Schema(description = "Presentation of a ZK receipt encoded in standard padded base64", implementation = String.class)
@JsonDeserialize(using = RedeemBackupReceiptRequest.Deserializer.class)
@JsonDeserialize(using = Deserializer.class)
@NotNull
ReceiptCredentialPresentation receiptCredentialPresentation) {
@@ -503,7 +500,16 @@ public class ArchiveController {
@JsonDeserialize(using = ByteArrayAdapter.Deserializing.class)
@NotNull
@ExactlySize(16)
byte[] iv) {}
byte[] iv) {
CopyParameters toCopyParameters() {
return new CopyParameters(
sourceAttachment.cdn, sourceAttachment.key,
objectLength,
new MediaEncryptionParameters(encryptionKey, hmacKey, iv),
mediaId);
}
}
public record CopyMediaResponse(
@Schema(description = "The backup cdn where this media object is stored")
@@ -547,48 +553,20 @@ public class ArchiveController {
throw new BadRequestException("must not use authenticated connection for anonymous operations");
}
return backupManager
.authenticateBackupUser(presentation.presentation, signature.signature)
.thenCompose(backupUser -> checkMediaFits(backupUser, copyMediaRequest.objectLength)
.thenCompose(ignored -> copyMediaImpl(backupUser, copyMediaRequest)))
.thenApply(result -> new CopyMediaResponse(result.cdn()))
.exceptionally(e -> {
final Throwable unwrapped = ExceptionUtils.unwrap(e);
if (unwrapped instanceof SourceObjectNotFoundException) {
throw new ClientErrorException("Source object not found " + unwrapped.getMessage(), Response.Status.GONE);
} else if (unwrapped instanceof InvalidLengthException) {
throw new BadRequestException("Invalid length " + unwrapped.getMessage());
} else {
throw ExceptionUtils.wrap(e);
}
});
return Mono
.fromFuture(backupManager.authenticateBackupUser(presentation.presentation, signature.signature))
.flatMap(backupUser -> backupManager.copyToBackup(backupUser, List.of(copyMediaRequest.toCopyParameters()))
.next()
.map(copyResult -> switch (copyResult.outcome()) {
case SUCCESS -> new CopyMediaResponse(copyResult.cdn());
case SOURCE_WRONG_LENGTH -> throw new BadRequestException("Invalid length");
case SOURCE_NOT_FOUND -> throw new ClientErrorException("Source object not found", Response.Status.GONE);
case OUT_OF_QUOTA ->
throw new ClientErrorException("Media quota exhausted", Response.Status.REQUEST_ENTITY_TOO_LARGE);
}))
.toFuture();
}
private CompletableFuture<Void> checkMediaFits(AuthenticatedBackupUser backupUser, long amountToStore) {
return backupManager.canStoreMedia(backupUser, amountToStore)
.thenApply(fits -> {
if (!fits) {
throw new ClientErrorException("Media quota exhausted", Response.Status.REQUEST_ENTITY_TOO_LARGE);
}
return null;
});
}
private CompletionStage<BackupManager.StorageDescriptor> copyMediaImpl(final AuthenticatedBackupUser backupUser,
final CopyMediaRequest copyMediaRequest) {
return this.backupManager.copyToBackup(
backupUser,
copyMediaRequest.sourceAttachment.cdn,
copyMediaRequest.sourceAttachment.key,
copyMediaRequest.objectLength,
new MediaEncryptionParameters(
copyMediaRequest.encryptionKey,
copyMediaRequest.hmacKey,
copyMediaRequest.iv),
copyMediaRequest.mediaId);
}
public record CopyMediaBatchRequest(
@Schema(description = "A list of media objects to copy from the attachments CDN to the backup CDN")
@NotNull
@@ -606,6 +584,7 @@ public class ArchiveController {
A 200 indicates the object was successfully copied.
A 400 indicates an invalid argument in the request
A 410 indicates that the source object was not found
A 413 indicates that the media quota was exhausted
""")
int status,
@@ -620,7 +599,17 @@ public class ArchiveController {
@JsonDeserialize(using = ByteArrayBase64UrlAdapter.Deserializing.class)
@NotNull
@ExactlySize(15)
byte[] mediaId) {}
byte[] mediaId) {
static Entry fromCopyResult(final CopyResult copyResult) {
return switch (copyResult.outcome()) {
case SUCCESS -> new Entry(200, null, copyResult.cdn(), copyResult.mediaId());
case SOURCE_WRONG_LENGTH -> new Entry(400, "Invalid source length", null, copyResult.mediaId());
case SOURCE_NOT_FOUND -> new Entry(410, "Source not found", null, copyResult.mediaId());
case OUT_OF_QUOTA -> new Entry(413, "Media quota exhausted", null, copyResult.mediaId());
};
}
}
}
@PUT
@@ -661,37 +650,13 @@ public class ArchiveController {
if (account.isPresent()) {
throw new BadRequestException("must not use authenticated connection for anonymous operations");
}
// If the entire batch won't fit in the user's remaining quota, reject the whole request.
final long expectedStorage = copyMediaRequest.items().stream().mapToLong(CopyMediaRequest::objectLength).sum();
return backupManager.authenticateBackupUser(presentation.presentation, signature.signature)
.thenCompose(backupUser -> checkMediaFits(backupUser, expectedStorage).thenCompose(
ignored -> Flux.fromIterable(copyMediaRequest.items)
// Operate sequentially, waiting for one copy to finish before starting the next one. At least right now,
// copying concurrently will introduce contention over the metadata.
.concatMap(request -> Mono
.fromCompletionStage(copyMediaImpl(backupUser, request))
.map(result -> new CopyMediaBatchResponse.Entry(200, null, result.cdn(), result.key()))
.onErrorResume(throwable -> ExceptionUtils.unwrap(throwable) instanceof IOException, throwable -> {
final Throwable unwrapped = ExceptionUtils.unwrap(throwable);
int status;
String error;
if (unwrapped instanceof SourceObjectNotFoundException) {
status = 410;
error = "Source object not found " + unwrapped.getMessage();
} else if (unwrapped instanceof InvalidLengthException) {
status = 400;
error = "Invalid length " + unwrapped.getMessage();
} else {
throw ExceptionUtils.wrap(throwable);
}
return Mono.just(new CopyMediaBatchResponse.Entry(status, error, null, request.mediaId));
}))
.collectList()
.map(list -> Response.status(207).entity(new CopyMediaBatchResponse(list)).build())
.toFuture()));
final Stream<CopyParameters> copyParams = copyMediaRequest.items().stream().map(CopyMediaRequest::toCopyParameters);
return Mono.fromFuture(backupManager.authenticateBackupUser(presentation.presentation, signature.signature))
.flatMapMany(backupUser -> backupManager.copyToBackup(backupUser, copyParams.toList()))
.map(CopyMediaBatchResponse.Entry::fromCopyResult)
.collectList()
.map(list -> Response.status(207).entity(new CopyMediaBatchResponse(list)).build())
.toFuture();
}
@POST
@@ -842,12 +807,15 @@ public class ArchiveController {
throw new BadRequestException("must not use authenticated connection for anonymous operations");
}
final List<BackupManager.StorageDescriptor> toDelete = deleteMedia.mediaToDelete().stream()
.map(media -> new BackupManager.StorageDescriptor(media.cdn(), media.mediaId))
.toList();
return backupManager
.authenticateBackupUser(presentation.presentation, signature.signature)
.thenCompose(authenticatedBackupUser -> backupManager.delete(authenticatedBackupUser,
deleteMedia.mediaToDelete().stream()
.map(media -> new BackupManager.StorageDescriptor(media.cdn(), media.mediaId))
.toList()))
.thenCompose(authenticatedBackupUser -> backupManager
.deleteMedia(authenticatedBackupUser, toDelete)
.then().toFuture())
.thenApply(Util.ASYNC_EMPTY_RESPONSE);
}