Signal/Signal-Server
1
0
Fork 0
mirror of https://github.com/signalapp/Signal-Server synced 2026-04-20 10:38:01 +01:00
Code Issues Packages Projects Releases Wiki Activity

Make max total backup media configurable

Browse Source
This commit is contained in:
Ravi Khadiwala
2025-09-15 10:29:10 -05:00
committed by ravi-signal
parent e50dcd185d
commit 35ffb208e3
9 changed files with 115 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files

49
service/src/main/java/org/whispersystems/textsecuregcm/backup/BackupManager.java
View File

@@ -17,6 +17,7 @@ import java.security.SecureRandom;
import java.time.Clock;
import java.time.Duration;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.Base64;
import java.util.HexFormat;
import java.util.List;
@@ -60,7 +61,6 @@ import reactor.core.scheduler.Scheduler;
public class BackupManager {
static final String MESSAGE_BACKUP_NAME = "messageBackup";
public static final long MAX_TOTAL_BACKUP_MEDIA_BYTES = DataSize.gibibytes(100).toBytes();
public static final long MAX_MESSAGE_BACKUP_OBJECT_SIZE = DataSize.mebibytes(101).toBytes();
public static final long MAX_MEDIA_OBJECT_SIZE = DataSize.mebibytes(101).toBytes();
@@ -74,6 +74,10 @@ public class BackupManager {
private static final Timer SYNCHRONOUS_DELETE_TIMER =
Metrics.timer(MetricsUtil.name(BackupManager.class, "synchronousDelete"));
private static final String NUM_OBJECTS_SUMMARY_NAME = MetricsUtil.name(BackupsDb.class, "numObjects");
private static final String BYTES_USED_SUMMARY_NAME = MetricsUtil.name(BackupsDb.class, "bytesUsed");
private static final String BACKUPS_COUNTER_NAME = MetricsUtil.name(BackupsDb.class, "backups");
private static final String SUCCESS_TAG_NAME = "success";
private static final String FAILURE_REASON_TAG_NAME = "reason";
@@ -161,11 +165,42 @@ public class BackupManager {
final AuthenticatedBackupUser backupUser) {
checkBackupLevel(backupUser, BackupLevel.FREE);
checkBackupCredentialType(backupUser, BackupCredentialType.MESSAGES);
final Instant today = clock.instant().truncatedTo(ChronoUnit.DAYS);
final long maxTotalMediaSize =
dynamicConfigurationManager.getConfiguration().getBackupConfiguration().maxTotalMediaSize();
// this could race with concurrent updates, but the only effect would be last-writer-wins on the timestamp
return backupsDb
.addMessageBackup(backupUser)
.thenApply(result -> cdn3BackupCredentialGenerator.generateUpload(cdnMessageBackupName(backupUser)));
.thenApply(storedBackupAttributes -> {
final Instant previousRefreshTime = storedBackupAttributes.lastRefresh();
// Only publish a metric update once per day
if (previousRefreshTime.isBefore(today)) {
final Tags tags = Tags.of(
UserAgentTagUtil.getPlatformTag(backupUser.userAgent()),
Tag.of("tier", backupUser.backupLevel().name()));
DistributionSummary.builder(NUM_OBJECTS_SUMMARY_NAME)
.tags(tags)
.publishPercentileHistogram()
.register(Metrics.globalRegistry)
.record(storedBackupAttributes.numObjects());
DistributionSummary.builder(BYTES_USED_SUMMARY_NAME)
.tags(tags)
.publishPercentileHistogram()
.register(Metrics.globalRegistry)
.record(storedBackupAttributes.bytesUsed());
// Report that the backup is out of quota if it cannot store a max size media object
final boolean quotaExhausted = storedBackupAttributes.bytesUsed() >=
(maxTotalMediaSize - BackupManager.MAX_MEDIA_OBJECT_SIZE);
Metrics.counter(BACKUPS_COUNTER_NAME,
tags.and("quotaExhausted", String.valueOf(quotaExhausted)))
.increment();
}
return cdn3BackupCredentialGenerator.generateUpload(cdnMessageBackupName(backupUser));
});
}
public CompletableFuture<BackupUploadDescriptor> createTemporaryAttachmentUploadDescriptor(
@@ -312,12 +347,14 @@ public class BackupManager {
})
.sum();
final Duration maxQuotaStaleness =
dynamicConfigurationManager.getConfiguration().getBackupConfiguration().maxQuotaStaleness();
final DynamicBackupConfiguration backupConfiguration =
dynamicConfigurationManager.getConfiguration().getBackupConfiguration();
final Duration maxQuotaStaleness = backupConfiguration.maxQuotaStaleness();
final long maxTotalMediaSize = backupConfiguration.maxTotalMediaSize();
return backupsDb.getMediaUsage(backupUser)
.thenComposeAsync(info -> {
long remainingQuota = MAX_TOTAL_BACKUP_MEDIA_BYTES - info.usageInfo().bytesUsed();
long remainingQuota = maxTotalMediaSize - info.usageInfo().bytesUsed();
final boolean canStore = remainingQuota >= totalBytesAdded;
if (canStore || info.lastRecalculationTime().isAfter(clock.instant().minus(maxQuotaStaleness))) {
return CompletableFuture.completedFuture(remainingQuota);
@@ -336,7 +373,7 @@ public class BackupManager {
Tag.of("usageChanged", String.valueOf(usageChanged))))
.increment();
})
.thenApply(newUsage -> MAX_TOTAL_BACKUP_MEDIA_BYTES - newUsage.bytesUsed());
.thenApply(newUsage -> maxTotalMediaSize - newUsage.bytesUsed());
})
.thenApply(remainingQuota -> {
// Figure out how many of the requested objects fit in the remaining quota

63
service/src/main/java/org/whispersystems/textsecuregcm/backup/BackupsDb.java
View File

@@ -87,10 +87,6 @@ public class BackupsDb {
private final SecureRandom secureRandom;
private static final String NUM_OBJECTS_SUMMARY_NAME = MetricsUtil.name(BackupsDb.class, "numObjects");
private static final String BYTES_USED_SUMMARY_NAME = MetricsUtil.name(BackupsDb.class, "bytesUsed");
private static final String BACKUPS_COUNTER_NAME = MetricsUtil.name(BackupsDb.class, "backups");
// The backups table
// B: 16 bytes that identifies the backup
@@ -257,12 +253,14 @@ public class BackupsDb {
.thenRun(Util.NOOP);
}
/**
* Track that a backup will be stored for the user
*
* @param backupUser an already authorized backup user
* @return A future that completes with the attributes of the backup before the update
*/
CompletableFuture<Void> addMessageBackup(final AuthenticatedBackupUser backupUser) {
CompletableFuture<StoredBackupAttributes> addMessageBackup(final AuthenticatedBackupUser backupUser) {
final Instant today = clock.instant().truncatedTo(ChronoUnit.DAYS);
// this could race with concurrent updates, but the only effect would be last-writer-wins on the timestamp
return dynamoClient.updateItem(
@@ -272,40 +270,7 @@ public class BackupsDb {
.updateItemBuilder()
.returnValues(ReturnValue.ALL_OLD)
.build())
.thenAccept(updateItemResponse ->
updateMetricsAfterUpload(backupUser, today, updateItemResponse.attributes()));
}
private void updateMetricsAfterUpload(final AuthenticatedBackupUser backupUser, final Instant today, final Map<String, AttributeValue> item) {
final Instant previousRefreshTime = Instant.ofEpochSecond(
AttributeValues.getLong(item, ATTR_LAST_REFRESH, 0L));
// Only publish a metric update once per day
if (previousRefreshTime.isBefore(today)) {
final long mediaCount = AttributeValues.getLong(item, ATTR_MEDIA_COUNT, 0L);
final long bytesUsed = AttributeValues.getLong(item, ATTR_MEDIA_BYTES_USED, 0L);
final Tags tags = Tags.of(
UserAgentTagUtil.getPlatformTag(backupUser.userAgent()),
Tag.of("tier", backupUser.backupLevel().name()));
DistributionSummary.builder(NUM_OBJECTS_SUMMARY_NAME)
.tags(tags)
.publishPercentileHistogram()
.register(Metrics.globalRegistry)
.record(mediaCount);
DistributionSummary.builder(BYTES_USED_SUMMARY_NAME)
.tags(tags)
.publishPercentileHistogram()
.register(Metrics.globalRegistry)
.record(bytesUsed);
// Report that the backup is out of quota if it cannot store a max size media object
final boolean quotaExhausted = bytesUsed >=
(BackupManager.MAX_TOTAL_BACKUP_MEDIA_BYTES - BackupManager.MAX_MEDIA_OBJECT_SIZE);
Metrics.counter(BACKUPS_COUNTER_NAME,
tags.and("quotaExhausted", String.valueOf(quotaExhausted)))
.increment();
}
.thenApply(updateItemResponse -> fromItem(updateItemResponse.attributes()));
}
/**
@@ -520,14 +485,18 @@ public class BackupsDb {
// Don't use the SDK's item publisher, works around https://github.com/aws/aws-sdk-java-v2/issues/6411
.concatMap(page -> Flux.fromIterable(page.items()))
.filter(item -> item.containsKey(KEY_BACKUP_ID_HASH))
.map(item -> new StoredBackupAttributes(
AttributeValues.getByteArray(item, KEY_BACKUP_ID_HASH, null),
AttributeValues.getString(item, ATTR_BACKUP_DIR, null),
AttributeValues.getString(item, ATTR_MEDIA_DIR, null),
Instant.ofEpochSecond(AttributeValues.getLong(item, ATTR_LAST_REFRESH, 0L)),
Instant.ofEpochSecond(AttributeValues.getLong(item, ATTR_LAST_MEDIA_REFRESH, 0L)),
AttributeValues.getLong(item, ATTR_MEDIA_BYTES_USED, 0L),
AttributeValues.getLong(item, ATTR_MEDIA_COUNT, 0L)));
.map(BackupsDb::fromItem);
}
private static StoredBackupAttributes fromItem(Map<String, AttributeValue> item) {
return new StoredBackupAttributes(
AttributeValues.getByteArray(item, KEY_BACKUP_ID_HASH, null),
AttributeValues.getString(item, ATTR_BACKUP_DIR, null),
AttributeValues.getString(item, ATTR_MEDIA_DIR, null),
Instant.ofEpochSecond(AttributeValues.getLong(item, ATTR_LAST_REFRESH, 0L)),
Instant.ofEpochSecond(AttributeValues.getLong(item, ATTR_LAST_MEDIA_REFRESH, 0L)),
AttributeValues.getLong(item, ATTR_MEDIA_BYTES_USED, 0L),
AttributeValues.getLong(item, ATTR_MEDIA_COUNT, 0L));
}
Flux<ExpiredBackup> getExpiredBackups(final int segments, final Scheduler scheduler, final Instant purgeTime) {