Signal/Signal-Server
1
0
Fork 0
mirror of https://github.com/signalapp/Signal-Server synced 2026-04-22 16:38:04 +01:00
Code Issues Packages Projects Releases Wiki Activity

Track backup metrics on refreshes

Browse Source
This commit is contained in:
Ravi Khadiwala
2025-05-27 18:10:24 -05:00
committed by ravi-signal
parent 030d8e8dd4
commit 4dc3b19d2a
10 changed files with 198 additions and 131 deletions
Download Patch File Download Diff File Expand all files Collapse all files

54
service/src/test/java/org/whispersystems/textsecuregcm/backup/BackupManagerTest.java
View File

@@ -30,6 +30,7 @@ import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.time.Duration;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Base64;
@@ -261,7 +262,7 @@ public class BackupManagerTest {
final AuthenticatedBackupUser backupUser = backupUser(TestRandomUtil.nextBytes(16), BackupCredentialType.MESSAGES, backupLevel);
final Instant tstart = Instant.ofEpochSecond(1).plus(Duration.ofDays(1));
final Instant tnext = tstart.plus(Duration.ofSeconds(1));
final Instant tnext = tstart.plus(Duration.ofDays(1));
// create backup at t=tstart
testClock.pin(tstart);
@@ -272,8 +273,8 @@ public class BackupManagerTest {
backupManager.ttlRefresh(backupUser).join();
checkExpectedExpirations(
tnext,
backupLevel == BackupLevel.PAID ? tnext : null,
tnext.truncatedTo(ChronoUnit.DAYS),
backupLevel == BackupLevel.PAID ? tnext.truncatedTo(ChronoUnit.DAYS) : null,
backupUser);
}
@@ -281,7 +282,7 @@ public class BackupManagerTest {
@EnumSource
public void createBackupRefreshesTtl(final BackupLevel backupLevel) {
final Instant tstart = Instant.ofEpochSecond(1).plus(Duration.ofDays(1));
final Instant tnext = tstart.plus(Duration.ofSeconds(1));
final Instant tnext = tstart.plus(Duration.ofDays(1));
final AuthenticatedBackupUser backupUser = backupUser(TestRandomUtil.nextBytes(16), BackupCredentialType.MESSAGES, backupLevel);
@@ -294,8 +295,8 @@ public class BackupManagerTest {
backupManager.createMessageBackupUploadDescriptor(backupUser).join();
checkExpectedExpirations(
tnext,
backupLevel == BackupLevel.PAID ? tnext : null,
tnext.truncatedTo(ChronoUnit.DAYS),
backupLevel == BackupLevel.PAID ? tnext.truncatedTo(ChronoUnit.DAYS) : null,
backupUser);
}
@@ -311,7 +312,8 @@ public class BackupManagerTest {
assertThatExceptionOfType(StatusRuntimeException.class)
.isThrownBy(() -> backupManager.authenticateBackupUser(
invalidPresentation,
keyPair.getPrivateKey().calculateSignature(invalidPresentation.serialize())))
keyPair.getPrivateKey().calculateSignature(invalidPresentation.serialize()),
null))
.extracting(StatusRuntimeException::getStatus)
.extracting(Status::getCode)
.isEqualTo(Status.UNAUTHENTICATED.getCode());
@@ -335,7 +337,8 @@ public class BackupManagerTest {
assertThatExceptionOfType(StatusRuntimeException.class)
.isThrownBy(() -> backupManager.authenticateBackupUser(
invalidPresentation,
keyPair.getPrivateKey().calculateSignature(invalidPresentation.serialize())))
keyPair.getPrivateKey().calculateSignature(invalidPresentation.serialize()),
null))
.extracting(StatusRuntimeException::getStatus)
.extracting(Status::getCode)
.isEqualTo(Status.UNAUTHENTICATED.getCode());
@@ -352,7 +355,7 @@ public class BackupManagerTest {
// haven't set a public key yet
assertThat(CompletableFutureTestUtil.assertFailsWithCause(
StatusRuntimeException.class,
backupManager.authenticateBackupUser(presentation, signature))
backupManager.authenticateBackupUser(presentation, signature, null))
.getStatus().getCode())
.isEqualTo(Status.UNAUTHENTICATED.getCode());
}
@@ -403,12 +406,12 @@ public class BackupManagerTest {
// shouldn't be able to authenticate with an invalid signature
assertThat(CompletableFutureTestUtil.assertFailsWithCause(
StatusRuntimeException.class,
backupManager.authenticateBackupUser(presentation, wrongSignature))
backupManager.authenticateBackupUser(presentation, wrongSignature, null))
.getStatus().getCode())
.isEqualTo(Status.UNAUTHENTICATED.getCode());
// correct signature
final AuthenticatedBackupUser user = backupManager.authenticateBackupUser(presentation, signature).join();
final AuthenticatedBackupUser user = backupManager.authenticateBackupUser(presentation, signature, null).join();
assertThat(user.backupId()).isEqualTo(presentation.getBackupId());
assertThat(user.backupLevel()).isEqualTo(BackupLevel.FREE);
}
@@ -426,16 +429,16 @@ public class BackupManagerTest {
// should be accepted the day before to forgive clock skew
testClock.pin(Instant.ofEpochSecond(1));
assertThatNoException().isThrownBy(() -> backupManager.authenticateBackupUser(oldCredential, signature).join());
assertThatNoException().isThrownBy(() -> backupManager.authenticateBackupUser(oldCredential, signature, null).join());
// should be accepted the day after to forgive clock skew
testClock.pin(Instant.ofEpochSecond(1).plus(Duration.ofDays(2)));
assertThatNoException().isThrownBy(() -> backupManager.authenticateBackupUser(oldCredential, signature).join());
assertThatNoException().isThrownBy(() -> backupManager.authenticateBackupUser(oldCredential, signature, null).join());
// should be rejected the day after that
testClock.pin(Instant.ofEpochSecond(1).plus(Duration.ofDays(3)));
assertThatExceptionOfType(StatusRuntimeException.class)
.isThrownBy(() -> backupManager.authenticateBackupUser(oldCredential, signature))
.isThrownBy(() -> backupManager.authenticateBackupUser(oldCredential, signature, null))
.extracting(StatusRuntimeException::getStatus)
.extracting(Status::getCode)
.isEqualTo(Status.UNAUTHENTICATED.getCode());
@@ -856,7 +859,7 @@ public class BackupManagerTest {
.mapToObj(i -> backupUser(TestRandomUtil.nextBytes(16), BackupCredentialType.MESSAGES, BackupLevel.PAID))
.toList();
for (int i = 0; i < backupUsers.size(); i++) {
testClock.pin(Instant.ofEpochSecond(i));
testClock.pin(days(i));
backupManager.createMessageBackupUploadDescriptor(backupUsers.get(i)).join();
}
@@ -864,11 +867,12 @@ public class BackupManagerTest {
final Set<ByteBuffer> expectedHashes = new HashSet<>();
for (int i = 0; i < backupUsers.size(); i++) {
testClock.pin(Instant.ofEpochSecond(i));
final Instant day = days(i);
testClock.pin(day);
// get backups expired at t=i
final List<ExpiredBackup> expired = backupManager
.getExpiredBackups(1, Schedulers.immediate(), Instant.ofEpochSecond(i))
.getExpiredBackups(1, Schedulers.immediate(), day)
.collectList()
.block();
@@ -890,24 +894,24 @@ public class BackupManagerTest {
final byte[] backupId = TestRandomUtil.nextBytes(16);
// refreshed media timestamp at t=5
testClock.pin(Instant.ofEpochSecond(5));
testClock.pin(days(5));
backupManager.createMessageBackupUploadDescriptor(backupUser(backupId, BackupCredentialType.MESSAGES, BackupLevel.PAID)).join();
// refreshed messages timestamp at t=6
testClock.pin(Instant.ofEpochSecond(6));
testClock.pin(days(6));
backupManager.createMessageBackupUploadDescriptor(backupUser(backupId, BackupCredentialType.MESSAGES, BackupLevel.FREE)).join();
Function<Instant, List<ExpiredBackup>> getExpired = time -> backupManager
.getExpiredBackups(1, Schedulers.immediate(), time)
.collectList().block();
assertThat(getExpired.apply(Instant.ofEpochSecond(5))).isEmpty();
assertThat(getExpired.apply(days(5))).isEmpty();
assertThat(getExpired.apply(Instant.ofEpochSecond(6)))
assertThat(getExpired.apply(days(6)))
.hasSize(1).first()
.matches(eb -> eb.expirationType() == ExpiredBackup.ExpirationType.MEDIA, "is media tier");
assertThat(getExpired.apply(Instant.ofEpochSecond(7)))
assertThat(getExpired.apply(days(7)))
.hasSize(1).first()
.matches(eb -> eb.expirationType() == ExpiredBackup.ExpirationType.ALL, "is messages tier");
}
@@ -1075,6 +1079,10 @@ public class BackupManagerTest {
*/
private AuthenticatedBackupUser retrieveBackupUser(final byte[] backupId, final BackupCredentialType credentialType, final BackupLevel backupLevel) {
final BackupsDb.AuthenticationData authData = backupsDb.retrieveAuthenticationData(backupId).join().get();
return new AuthenticatedBackupUser(backupId, credentialType, backupLevel, authData.backupDir(), authData.mediaDir());
return new AuthenticatedBackupUser(backupId, credentialType, backupLevel, authData.backupDir(), authData.mediaDir(), null);
}
private static Instant days(int n) {
return Instant.EPOCH.plus(Duration.ofDays(n));
}
}

78
service/src/test/java/org/whispersystems/textsecuregcm/backup/BackupsDbTest.java
View File

@@ -10,12 +10,16 @@ import static org.assertj.core.api.Assertions.assertThat;
import io.grpc.Status;
import io.grpc.StatusRuntimeException;
import java.time.Duration;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.Comparator;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
import java.util.function.Function;
import java.util.stream.Stream;
import org.assertj.core.util.Streams;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.RegisterExtension;
@@ -89,13 +93,13 @@ public class BackupsDbTest {
@Test
public void expirationDetectedOnce() {
final byte[] backupId = TestRandomUtil.nextBytes(16);
// Refresh media/messages at t=0
testClock.pin(Instant.ofEpochSecond(0L));
// Refresh media/messages at t=0D
testClock.pin(days(0));
backupsDb.setPublicKey(backupId, BackupLevel.PAID, Curve.generateKeyPair().getPublicKey()).join();
this.backupsDb.ttlRefresh(backupUser(backupId, BackupCredentialType.MEDIA, BackupLevel.PAID)).join();
// refresh only messages at t=2
testClock.pin(Instant.ofEpochSecond(2L));
// refresh only messages on t=2D
testClock.pin(days(2).plus(Duration.ofSeconds(123)));
this.backupsDb.ttlRefresh(backupUser(backupId, BackupCredentialType.MEDIA, BackupLevel.FREE)).join();
final Function<Instant, List<ExpiredBackup>> expiredBackups = purgeTime -> backupsDb
@@ -103,7 +107,8 @@ public class BackupsDbTest {
.collectList()
.block();
List<ExpiredBackup> expired = expiredBackups.apply(Instant.ofEpochSecond(1));
// the media should be expired at t=1D
List<ExpiredBackup> expired = expiredBackups.apply(days(1));
assertThat(expired).hasSize(1).first()
.matches(eb -> eb.expirationType() == ExpiredBackup.ExpirationType.MEDIA);
@@ -111,11 +116,11 @@ public class BackupsDbTest {
backupsDb.startExpiration(expired.getFirst()).join();
backupsDb.finishExpiration(expired.getFirst()).join();
// should be nothing to expire at t=1
assertThat(expiredBackups.apply(Instant.ofEpochSecond(1))).isEmpty();
// should be nothing left to expire at t=1D
assertThat(expiredBackups.apply(days(1))).isEmpty();
// at t=3, should now expire messages as well
expired = expiredBackups.apply(Instant.ofEpochSecond(3));
// at t=3D, should now expire messages as well
expired = expiredBackups.apply(days(3));
assertThat(expired).hasSize(1).first()
.matches(eb -> eb.expirationType() == ExpiredBackup.ExpirationType.ALL);
@@ -124,21 +129,21 @@ public class BackupsDbTest {
backupsDb.finishExpiration(expired.getFirst()).join();
// should be nothing to expire at t=3
assertThat(expiredBackups.apply(Instant.ofEpochSecond(3))).isEmpty();
assertThat(expiredBackups.apply(days(3))).isEmpty();
}
@ParameterizedTest
@EnumSource(names = {"MEDIA", "ALL"})
public void expirationFailed(ExpiredBackup.ExpirationType expirationType) {
final byte[] backupId = TestRandomUtil.nextBytes(16);
// Refresh media/messages at t=0
testClock.pin(Instant.ofEpochSecond(0L));
// Refresh media/messages at t=0D
testClock.pin(days(0));
backupsDb.setPublicKey(backupId, BackupLevel.PAID, Curve.generateKeyPair().getPublicKey()).join();
this.backupsDb.ttlRefresh(backupUser(backupId, BackupCredentialType.MEDIA, BackupLevel.PAID)).join();
if (expirationType == ExpiredBackup.ExpirationType.MEDIA) {
// refresh only messages at t=2 so that we only expire media at t=1
testClock.pin(Instant.ofEpochSecond(2L));
// refresh only messages at t=2D so that we only expire media at t=1D
testClock.pin(days(2));
this.backupsDb.ttlRefresh(backupUser(backupId, BackupCredentialType.MEDIA, BackupLevel.FREE)).join();
}
@@ -155,7 +160,7 @@ public class BackupsDbTest {
final String originalBackupDir = info.backupDir();
final String originalMediaDir = info.mediaDir();
ExpiredBackup expired = expiredBackups.apply(Instant.ofEpochSecond(1)).get();
ExpiredBackup expired = expiredBackups.apply(days(1)).get();
assertThat(expired).matches(eb -> eb.expirationType() == expirationType);
// expire but fail (don't call finishExpiration)
@@ -179,7 +184,7 @@ public class BackupsDbTest {
final String expiredPrefix = expired.prefixToDelete();
// We failed, so we should see the same prefix on the next expiration listing
expired = expiredBackups.apply(Instant.ofEpochSecond(1)).get();
expired = expiredBackups.apply(days(1)).get();
assertThat(expired).matches(eb -> eb.expirationType() == ExpiredBackup.ExpirationType.GARBAGE_COLLECTION,
"Expiration should be garbage collection ");
assertThat(expired.prefixToDelete()).isEqualTo(expiredPrefix);
@@ -188,7 +193,7 @@ public class BackupsDbTest {
// Successfully finish the expiration
backupsDb.finishExpiration(expired).join();
Optional<ExpiredBackup> opt = expiredBackups.apply(Instant.ofEpochSecond(1));
Optional<ExpiredBackup> opt = expiredBackups.apply(days(1));
if (expirationType == ExpiredBackup.ExpirationType.MEDIA) {
// should be nothing to expire at t=1
assertThat(opt).isEmpty();
@@ -212,19 +217,24 @@ public class BackupsDbTest {
@Test
public void list() {
final AuthenticatedBackupUser u1 = backupUser(TestRandomUtil.nextBytes(16), BackupCredentialType.MEDIA, BackupLevel.FREE);
final AuthenticatedBackupUser u2 = backupUser(TestRandomUtil.nextBytes(16), BackupCredentialType.MEDIA, BackupLevel.PAID);
final AuthenticatedBackupUser u3 = backupUser(TestRandomUtil.nextBytes(16), BackupCredentialType.MEDIA, BackupLevel.PAID);
final List<AuthenticatedBackupUser> users = List.of(
backupUser(TestRandomUtil.nextBytes(16), BackupCredentialType.MEDIA, BackupLevel.FREE),
backupUser(TestRandomUtil.nextBytes(16), BackupCredentialType.MEDIA, BackupLevel.PAID),
backupUser(TestRandomUtil.nextBytes(16), BackupCredentialType.MEDIA, BackupLevel.PAID));
final List<Instant> lastRefreshTimes = List.of(
days(1).plus(Duration.ofSeconds(12)),
days(2).plus(Duration.ofSeconds(34)),
days(3).plus(Duration.ofSeconds(56)));
// add at least one message backup, so we can describe it
testClock.pin(Instant.ofEpochSecond(10));
Stream.of(u1, u2, u3).forEach(u -> backupsDb.addMessageBackup(u).join());
for (int i = 0; i < users.size(); i++) {
testClock.pin(lastRefreshTimes.get(i));
backupsDb.addMessageBackup(users.get(i)).join();
}
testClock.pin(Instant.ofEpochSecond(20));
backupsDb.trackMedia(u2, 10, 100).join();
testClock.pin(Instant.ofEpochSecond(30));
backupsDb.trackMedia(u3, 1, 1000).join();
backupsDb.trackMedia(users.get(1), 10, 100).join();
backupsDb.trackMedia(users.get(2), 1, 1000).join();
final List<StoredBackupAttributes> sbms = backupsDb.listBackupAttributes(1, Schedulers.immediate())
.sort(Comparator.comparing(StoredBackupAttributes::lastRefresh))
@@ -234,22 +244,28 @@ public class BackupsDbTest {
final StoredBackupAttributes sbm1 = sbms.get(0);
assertThat(sbm1.bytesUsed()).isEqualTo(0);
assertThat(sbm1.numObjects()).isEqualTo(0);
assertThat(sbm1.lastRefresh()).isEqualTo(Instant.ofEpochSecond(10));
assertThat(sbm1.lastRefresh()).isEqualTo(lastRefreshTimes.get(0).truncatedTo(ChronoUnit.DAYS));
assertThat(sbm1.lastMediaRefresh()).isEqualTo(Instant.EPOCH);
final StoredBackupAttributes sbm2 = sbms.get(1);
assertThat(sbm2.bytesUsed()).isEqualTo(100);
assertThat(sbm2.numObjects()).isEqualTo(10);
assertThat(sbm2.lastRefresh()).isEqualTo(sbm2.lastMediaRefresh()).isEqualTo(Instant.ofEpochSecond(20));
assertThat(sbm2.lastRefresh()).isEqualTo(lastRefreshTimes.get(1).truncatedTo(ChronoUnit.DAYS));
assertThat(sbm2.lastMediaRefresh()).isEqualTo(lastRefreshTimes.get(1).truncatedTo(ChronoUnit.DAYS));
final StoredBackupAttributes sbm3 = sbms.get(2);
assertThat(sbm3.bytesUsed()).isEqualTo(1000);
assertThat(sbm3.numObjects()).isEqualTo(1);
assertThat(sbm3.lastRefresh()).isEqualTo(sbm3.lastMediaRefresh()).isEqualTo(Instant.ofEpochSecond(30));
assertThat(sbm3.lastRefresh()).isEqualTo(lastRefreshTimes.get(2).truncatedTo(ChronoUnit.DAYS));
assertThat(sbm3.lastMediaRefresh()).isEqualTo(lastRefreshTimes.get(2).truncatedTo(ChronoUnit.DAYS));
}
private static Instant days(int n) {
return Instant.EPOCH.plus(Duration.ofDays(n));
}
private AuthenticatedBackupUser backupUser(final byte[] backupId, final BackupCredentialType credentialType, final BackupLevel backupLevel) {
return new AuthenticatedBackupUser(backupId, credentialType, backupLevel, "myBackupDir", "myMediaDir");
return new AuthenticatedBackupUser(backupId, credentialType, backupLevel, "myBackupDir", "myMediaDir", null);
}
}