Signal/Signal-Server
1
0
Fork 0
mirror of https://github.com/signalapp/Signal-Server synced 2026-04-22 02:48:01 +01:00
Code Issues Packages Projects Releases Wiki Activity

Remove AccountCrawler (and doPeriodicWork) from WhisperServerService

Browse Source
This commit is contained in:
Chris Eager
2023-06-23 12:46:48 -05:00
committed by Chris Eager
parent f26bc70b59
commit f8fefe2e5e
11 changed files with 30 additions and 395 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
service/src/test/java/org/whispersystems/textsecuregcm/storage/AccountCleanerTest.java
View File

@@ -77,7 +77,7 @@ class AccountCleanerTest {
}
@Test
void testAccounts() throws AccountDatabaseCrawlerRestartException, InterruptedException {
void testAccounts() throws InterruptedException {
AccountCleaner accountCleaner = new AccountCleaner(accountsManager, deletionExecutor);
accountCleaner.onCrawlStart();
accountCleaner.timeAndProcessCrawlChunk(Optional.empty(),

50
service/src/test/java/org/whispersystems/textsecuregcm/storage/AccountDatabaseCrawlerIntegrationTest.java
View File

@@ -7,9 +7,7 @@ package org.whispersystems.textsecuregcm.storage;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@@ -20,8 +18,6 @@ import java.util.UUID;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.RegisterExtension;
import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicAccountDatabaseCrawlerConfiguration;
import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicConfiguration;
import org.whispersystems.textsecuregcm.redis.RedisClusterExtension;
class AccountDatabaseCrawlerIntegrationTest {
@@ -60,54 +56,10 @@ class AccountDatabaseCrawlerIntegrationTest {
.thenReturn(new AccountCrawlChunk(List.of(secondAccount), SECOND_UUID))
.thenReturn(new AccountCrawlChunk(Collections.emptyList(), null));
DynamicConfigurationManager<DynamicConfiguration> dynamicConfigurationManager = mock(
DynamicConfigurationManager.class);
final DynamicConfiguration dynamicConfiguration = mock(DynamicConfiguration.class);
DynamicAccountDatabaseCrawlerConfiguration accountDatabaseCrawlerConfiguration = new DynamicAccountDatabaseCrawlerConfiguration(
true, true);
when(dynamicConfiguration.getAccountDatabaseCrawlerConfiguration()).thenReturn(accountDatabaseCrawlerConfiguration);
when(dynamicConfigurationManager.getConfiguration()).thenReturn(dynamicConfiguration);
final AccountDatabaseCrawlerCache crawlerCache = new AccountDatabaseCrawlerCache(
REDIS_CLUSTER_EXTENSION.getRedisCluster(), "test");
accountDatabaseCrawler = new AccountDatabaseCrawler("test", accountsManager, crawlerCache, List.of(listener),
CHUNK_SIZE, dynamicConfigurationManager);
}
@Test
void testCrawlUninterrupted() throws AccountDatabaseCrawlerRestartException {
accountDatabaseCrawler.doPeriodicWork();
accountDatabaseCrawler.doPeriodicWork();
accountDatabaseCrawler.doPeriodicWork();
verify(accountsManager).getAllFromDynamo(CHUNK_SIZE);
verify(accountsManager).getAllFromDynamo(FIRST_UUID, CHUNK_SIZE);
verify(accountsManager).getAllFromDynamo(SECOND_UUID, CHUNK_SIZE);
verify(listener).onCrawlStart();
verify(listener).timeAndProcessCrawlChunk(Optional.empty(), List.of(firstAccount));
verify(listener).timeAndProcessCrawlChunk(Optional.of(FIRST_UUID), List.of(secondAccount));
verify(listener).onCrawlEnd();
}
@Test
void testCrawlWithReset() throws AccountDatabaseCrawlerRestartException {
doThrow(new AccountDatabaseCrawlerRestartException("OH NO")).doNothing()
.when(listener).timeAndProcessCrawlChunk(Optional.empty(), List.of(firstAccount));
accountDatabaseCrawler.doPeriodicWork();
accountDatabaseCrawler.doPeriodicWork();
accountDatabaseCrawler.doPeriodicWork();
accountDatabaseCrawler.doPeriodicWork();
verify(accountsManager, times(2)).getAllFromDynamo(CHUNK_SIZE);
verify(accountsManager).getAllFromDynamo(FIRST_UUID, CHUNK_SIZE);
verify(accountsManager).getAllFromDynamo(SECOND_UUID, CHUNK_SIZE);
verify(listener, times(2)).onCrawlStart();
verify(listener, times(2)).timeAndProcessCrawlChunk(Optional.empty(), List.of(firstAccount));
verify(listener).timeAndProcessCrawlChunk(Optional.of(FIRST_UUID), List.of(secondAccount));
verify(listener).onCrawlEnd();
CHUNK_SIZE);
}
@Test

135
service/src/test/java/org/whispersystems/textsecuregcm/tests/storage/AccountDatabaseCrawlerTest.java
View File

@@ -9,11 +9,9 @@ import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoInteractions;
import static org.mockito.Mockito.verifyNoMoreInteractions;
import static org.mockito.Mockito.when;
@@ -23,16 +21,12 @@ import java.util.Optional;
import java.util.UUID;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicAccountDatabaseCrawlerConfiguration;
import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicConfiguration;
import org.whispersystems.textsecuregcm.storage.Account;
import org.whispersystems.textsecuregcm.storage.AccountCrawlChunk;
import org.whispersystems.textsecuregcm.storage.AccountDatabaseCrawler;
import org.whispersystems.textsecuregcm.storage.AccountDatabaseCrawlerCache;
import org.whispersystems.textsecuregcm.storage.AccountDatabaseCrawlerListener;
import org.whispersystems.textsecuregcm.storage.AccountDatabaseCrawlerRestartException;
import org.whispersystems.textsecuregcm.storage.AccountsManager;
import org.whispersystems.textsecuregcm.storage.DynamicConfigurationManager;
class AccountDatabaseCrawlerTest {
@@ -48,11 +42,8 @@ class AccountDatabaseCrawlerTest {
private final AccountDatabaseCrawlerListener listener = mock(AccountDatabaseCrawlerListener.class);
private final AccountDatabaseCrawlerCache cache = mock(AccountDatabaseCrawlerCache.class);
private final DynamicConfigurationManager<DynamicConfiguration> dynamicConfigurationManager = mock(
DynamicConfigurationManager.class);
private final AccountDatabaseCrawler crawler =
new AccountDatabaseCrawler("test", accounts, cache, List.of(listener), CHUNK_SIZE, dynamicConfigurationManager);
new AccountDatabaseCrawler("test", accounts, cache, List.of(listener), CHUNK_SIZE);
@BeforeEach
void setup() {
@@ -67,132 +58,10 @@ class AccountDatabaseCrawlerTest {
new AccountCrawlChunk(Collections.emptyList(), null));
when(cache.claimActiveWork(any(), anyLong())).thenReturn(true);
final DynamicConfiguration dynamicConfiguration = mock(DynamicConfiguration.class);
final DynamicAccountDatabaseCrawlerConfiguration accountDatabaseCrawlerConfiguration =
new DynamicAccountDatabaseCrawlerConfiguration(true, true);
when(dynamicConfiguration.getAccountDatabaseCrawlerConfiguration()).thenReturn(accountDatabaseCrawlerConfiguration);
when(dynamicConfigurationManager.getConfiguration()).thenReturn(dynamicConfiguration);
}
@Test
void testCrawlStart() throws AccountDatabaseCrawlerRestartException {
when(cache.getLastUuid()).thenReturn(Optional.empty());
crawler.doPeriodicWork();
verify(cache, times(1)).claimActiveWork(any(String.class), anyLong());
verify(cache, times(1)).getLastUuid();
verify(listener, times(1)).onCrawlStart();
verify(accounts, times(1)).getAllFromDynamo(eq(CHUNK_SIZE));
verify(accounts, times(0)).getAllFromDynamo(any(UUID.class), eq(CHUNK_SIZE));
verify(account1, times(0)).getUuid();
verify(listener, times(1)).timeAndProcessCrawlChunk(eq(Optional.empty()), eq(List.of(account1, account2)));
verify(cache, times(1)).setLastUuid(eq(Optional.of(ACCOUNT2)));
verify(cache, times(1)).releaseActiveWork(any(String.class));
verifyNoMoreInteractions(account1);
verifyNoMoreInteractions(account2);
verifyNoMoreInteractions(accounts);
verifyNoMoreInteractions(listener);
verifyNoMoreInteractions(cache);
}
@Test
void testCrawlChunk() throws AccountDatabaseCrawlerRestartException {
when(cache.getLastUuid()).thenReturn(Optional.of(ACCOUNT1));
crawler.doPeriodicWork();
verify(cache, times(1)).claimActiveWork(any(String.class), anyLong());
verify(cache, times(1)).getLastUuid();
verify(accounts, times(0)).getAllFromDynamo(eq(CHUNK_SIZE));
verify(accounts, times(1)).getAllFromDynamo(eq(ACCOUNT1), eq(CHUNK_SIZE));
verify(listener, times(1)).timeAndProcessCrawlChunk(eq(Optional.of(ACCOUNT1)), eq(List.of(account2)));
verify(cache, times(1)).setLastUuid(eq(Optional.of(ACCOUNT2)));
verify(cache, times(1)).releaseActiveWork(any(String.class));
verifyNoInteractions(account1);
verifyNoMoreInteractions(account2);
verifyNoMoreInteractions(accounts);
verifyNoMoreInteractions(listener);
verifyNoMoreInteractions(cache);
}
@Test
void testCrawlChunkAccelerated() throws AccountDatabaseCrawlerRestartException {
when(cache.getLastUuid()).thenReturn(Optional.of(ACCOUNT1));
crawler.doPeriodicWork();
verify(cache, times(1)).claimActiveWork(any(String.class), anyLong());
verify(cache, times(1)).getLastUuid();
verify(accounts, times(0)).getAllFromDynamo(eq(CHUNK_SIZE));
verify(accounts, times(1)).getAllFromDynamo(eq(ACCOUNT1), eq(CHUNK_SIZE));
verify(listener, times(1)).timeAndProcessCrawlChunk(eq(Optional.of(ACCOUNT1)), eq(List.of(account2)));
verify(cache, times(1)).setLastUuid(eq(Optional.of(ACCOUNT2)));
verify(cache, times(1)).releaseActiveWork(any(String.class));
verifyNoInteractions(account1);
verifyNoMoreInteractions(account2);
verifyNoMoreInteractions(accounts);
verifyNoMoreInteractions(listener);
verifyNoMoreInteractions(cache);
}
@Test
void testCrawlChunkRestart() throws AccountDatabaseCrawlerRestartException {
when(cache.getLastUuid()).thenReturn(Optional.of(ACCOUNT1));
doThrow(AccountDatabaseCrawlerRestartException.class).when(listener)
.timeAndProcessCrawlChunk(eq(Optional.of(ACCOUNT1)), eq(List.of(account2)));
crawler.doPeriodicWork();
verify(cache, times(1)).claimActiveWork(any(String.class), anyLong());
verify(cache, times(1)).getLastUuid();
verify(accounts, times(0)).getAllFromDynamo(eq(CHUNK_SIZE));
verify(accounts, times(1)).getAllFromDynamo(eq(ACCOUNT1), eq(CHUNK_SIZE));
verify(account2, times(0)).getNumber();
verify(listener, times(1)).timeAndProcessCrawlChunk(eq(Optional.of(ACCOUNT1)), eq(List.of(account2)));
verify(cache, times(1)).setLastUuid(eq(Optional.empty()));
verify(cache, times(1)).releaseActiveWork(any(String.class));
verifyNoInteractions(account1);
verifyNoMoreInteractions(account2);
verifyNoMoreInteractions(accounts);
verifyNoMoreInteractions(listener);
verifyNoMoreInteractions(cache);
}
@Test
void testCrawlEnd() {
when(cache.getLastUuid()).thenReturn(Optional.of(ACCOUNT2));
crawler.doPeriodicWork();
verify(cache, times(1)).claimActiveWork(any(String.class), anyLong());
verify(cache, times(1)).getLastUuid();
verify(accounts, times(0)).getAllFromDynamo(eq(CHUNK_SIZE));
verify(accounts, times(1)).getAllFromDynamo(eq(ACCOUNT2), eq(CHUNK_SIZE));
verify(account1, times(0)).getNumber();
verify(account2, times(0)).getNumber();
verify(listener, times(1)).onCrawlEnd();
verify(cache, times(1)).setLastUuid(eq(Optional.empty()));
verify(cache, times(1)).releaseActiveWork(any(String.class));
verifyNoInteractions(account1);
verifyNoInteractions(account2);
verifyNoMoreInteractions(accounts);
verifyNoMoreInteractions(listener);
verifyNoMoreInteractions(cache);
}
@Test
void testCrawlAllAccounts() throws Exception {
void testCrawlAllAccounts() {
when(cache.getLastUuid())
.thenReturn(Optional.empty());

8
service/src/test/java/org/whispersystems/textsecuregcm/tests/storage/PushFeedbackProcessorTest.java
View File

@@ -27,7 +27,6 @@ import java.util.concurrent.TimeUnit;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.whispersystems.textsecuregcm.storage.Account;
import org.whispersystems.textsecuregcm.storage.AccountDatabaseCrawlerRestartException;
import org.whispersystems.textsecuregcm.storage.AccountsManager;
import org.whispersystems.textsecuregcm.storage.Device;
import org.whispersystems.textsecuregcm.storage.PushFeedbackProcessor;
@@ -36,7 +35,7 @@ import org.whispersystems.textsecuregcm.util.Util;
class PushFeedbackProcessorTest {
private AccountsManager accountsManager = mock(AccountsManager.class);
private final AccountsManager accountsManager = mock(AccountsManager.class);
private Account uninstalledAccount = mock(Account.class);
private Account mixedAccount = mock(Account.class);
@@ -99,9 +98,8 @@ class PushFeedbackProcessorTest {
Set.of(uninstalledAccount, mixedAccount, freshAccount, cleanAccount, stillActiveAccount));
}
@Test
void testEmpty() throws AccountDatabaseCrawlerRestartException {
void testEmpty() {
PushFeedbackProcessor processor = new PushFeedbackProcessor(accountsManager, Executors.newSingleThreadExecutor());
processor.timeAndProcessCrawlChunk(Optional.of(UUID.randomUUID()), Collections.emptyList());
@@ -109,7 +107,7 @@ class PushFeedbackProcessorTest {
}
@Test
void testUpdate() throws AccountDatabaseCrawlerRestartException {
void testUpdate() {
PushFeedbackProcessor processor = new PushFeedbackProcessor(accountsManager, Executors.newSingleThreadExecutor());
processor.timeAndProcessCrawlChunk(Optional.of(UUID.randomUUID()),
List.of(uninstalledAccount, mixedAccount, stillActiveAccount, freshAccount, cleanAccount));