Migrate from bounded elastic to dedicated executor for message delivery

service/src/main/java/org/whispersystems/textsecuregcm/websocket/AuthenticatedConnectListener.java

@@ -11,6 +11,8 @@ import com.codahale.metrics.Counter;
 import com.codahale.metrics.MetricRegistry;
 import com.codahale.metrics.SharedMetricRegistries;
 import com.codahale.metrics.Timer;
+import io.micrometer.core.instrument.Metrics;
+import io.micrometer.core.instrument.Tags;
 import java.util.EnumMap;
 import java.util.Map;
 import java.util.concurrent.ScheduledExecutorService;
@@ -18,8 +20,6 @@ import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
-import io.micrometer.core.instrument.Metrics;
-import io.micrometer.core.instrument.Tags;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.whispersystems.textsecuregcm.auth.AuthenticatedAccount;
@@ -38,6 +38,7 @@ import org.whispersystems.textsecuregcm.util.ua.UnrecognizedUserAgentException;
 import org.whispersystems.textsecuregcm.util.ua.UserAgentUtil;
 import org.whispersystems.websocket.session.WebSocketSessionContext;
 import org.whispersystems.websocket.setup.WebSocketConnectListener;
+import reactor.core.scheduler.Scheduler;
 
 public class AuthenticatedConnectListener implements WebSocketConnectListener {
 
@@ -61,6 +62,7 @@ public class AuthenticatedConnectListener implements WebSocketConnectListener {
   private final PushNotificationManager pushNotificationManager;
   private final ClientPresenceManager clientPresenceManager;
   private final ScheduledExecutorService scheduledExecutorService;
+  private final Scheduler messageDeliveryScheduler;
 
   private final Map<ClientPlatform, AtomicInteger> openWebsocketsByClientPlatform;
   private final AtomicInteger openWebsocketsFromUnknownPlatforms;
@@ -69,12 +71,14 @@ public class AuthenticatedConnectListener implements WebSocketConnectListener {
       MessagesManager messagesManager,
       PushNotificationManager pushNotificationManager,
       ClientPresenceManager clientPresenceManager,
-      ScheduledExecutorService scheduledExecutorService) {
+      ScheduledExecutorService scheduledExecutorService,
+      Scheduler messageDeliveryScheduler) {
     this.receiptSender = receiptSender;
     this.messagesManager = messagesManager;
     this.pushNotificationManager = pushNotificationManager;
     this.clientPresenceManager = clientPresenceManager;
     this.scheduledExecutorService = scheduledExecutorService;
+    this.messageDeliveryScheduler = messageDeliveryScheduler;
 
     openWebsocketsByClientPlatform = new EnumMap<>(ClientPlatform.class);
 
@@ -100,7 +104,8 @@ public class AuthenticatedConnectListener implements WebSocketConnectListener {
       final WebSocketConnection connection = new WebSocketConnection(receiptSender,
           messagesManager, auth, device,
           context.getClient(),
-          scheduledExecutorService);
+          scheduledExecutorService,
+          messageDeliveryScheduler);
 
       final AtomicInteger openWebsocketAtomicInteger = getOpenWebsocketCounter(context.getClient().getUserAgent());
 

service/src/main/java/org/whispersystems/textsecuregcm/websocket/WebSocketConnection.java

@@ -56,7 +56,6 @@ import reactor.core.observability.micrometer.Micrometer;
 import reactor.core.publisher.Flux;
 import reactor.core.publisher.Mono;
 import reactor.core.scheduler.Scheduler;
-import reactor.core.scheduler.Schedulers;
 
 public class WebSocketConnection implements MessageAvailabilityListener, DisplacedPresenceListener {
 
@@ -127,7 +126,7 @@ public class WebSocketConnection implements MessageAvailabilityListener, Displac
   private final AtomicReference<Disposable> messageSubscription = new AtomicReference<>();
 
   private final Random random = new Random();
-  private final Scheduler reactiveScheduler;
+  private final Scheduler messageDeliveryScheduler;
 
   private enum StoredMessageState {
     EMPTY,
@@ -136,29 +135,12 @@ public class WebSocketConnection implements MessageAvailabilityListener, Displac
   }
 
   public WebSocketConnection(ReceiptSender receiptSender,
-      MessagesManager messagesManager,
-      AuthenticatedAccount auth,
-      Device device,
-      WebSocketClient client,
-      ScheduledExecutorService scheduledExecutorService) {
-
-    this(receiptSender,
-        messagesManager,
-        auth,
-        device,
-        client,
-        scheduledExecutorService,
-        Schedulers.boundedElastic());
-  }
-
-  @VisibleForTesting
-  WebSocketConnection(ReceiptSender receiptSender,
       MessagesManager messagesManager,
       AuthenticatedAccount auth,
       Device device,
       WebSocketClient client,
       ScheduledExecutorService scheduledExecutorService,
-      Scheduler reactiveScheduler) {
+      Scheduler messageDeliveryScheduler) {
 
     this(receiptSender,
         messagesManager,
@@ -167,7 +149,7 @@ public class WebSocketConnection implements MessageAvailabilityListener, Displac
         client,
         DEFAULT_SEND_FUTURES_TIMEOUT_MILLIS,
         scheduledExecutorService,
-        reactiveScheduler);
+        messageDeliveryScheduler);
   }
 
   @VisibleForTesting
@@ -178,7 +160,7 @@ public class WebSocketConnection implements MessageAvailabilityListener, Displac
       WebSocketClient client,
       int sendFuturesTimeoutMillis,
       ScheduledExecutorService scheduledExecutorService,
-      Scheduler reactiveScheduler) {
+      Scheduler messageDeliveryScheduler) {
 
     this.receiptSender = receiptSender;
     this.messagesManager = messagesManager;
@@ -187,7 +169,7 @@ public class WebSocketConnection implements MessageAvailabilityListener, Displac
     this.client = client;
     this.sendFuturesTimeoutMillis = sendFuturesTimeoutMillis;
     this.scheduledExecutorService = scheduledExecutorService;
-    this.reactiveScheduler = reactiveScheduler;
+    this.messageDeliveryScheduler = messageDeliveryScheduler;
   }
 
   public void start() {
@@ -366,7 +348,7 @@ public class WebSocketConnection implements MessageAvailabilityListener, Displac
         .flatMapSequential(envelope ->
             Mono.fromFuture(sendMessage(envelope)
                 .orTimeout(sendFuturesTimeoutMillis, TimeUnit.MILLISECONDS)))
-        .subscribeOn(reactiveScheduler)
+        .subscribeOn(messageDeliveryScheduler)
         .subscribe(
             // no additional consumer of values - it is Flux<Void> by now
             null,