Improve WebSocket health monitoring.

2026-02-24 03:35:58 +00:00 · 2021-07-27 13:40:33 -04:00
parent fc6db45e59
commit 712b0c147a
20 changed files with 559 additions and 351 deletions
--- a/app/src/main/java/org/thoughtcrime/securesms/conversationlist/ConversationListFragment.java
+++ b/app/src/main/java/org/thoughtcrime/securesms/conversationlist/ConversationListFragment.java
@@ -112,7 +112,6 @@ import org.thoughtcrime.securesms.megaphone.MegaphoneActionController;
 import org.thoughtcrime.securesms.megaphone.MegaphoneViewBuilder;
 import org.thoughtcrime.securesms.megaphone.Megaphones;
 import org.thoughtcrime.securesms.mms.GlideApp;
-import org.thoughtcrime.securesms.net.PipeConnectivityListener;
 import org.thoughtcrime.securesms.notifications.MarkReadReceiver;
 import org.thoughtcrime.securesms.payments.preferences.PaymentsActivity;
 import org.thoughtcrime.securesms.payments.preferences.details.PaymentDetailsFragmentArgs;
@@ -143,6 +142,7 @@ import org.thoughtcrime.securesms.util.concurrent.SimpleTask;
 import org.thoughtcrime.securesms.util.task.SnackbarAsyncTask;
 import org.thoughtcrime.securesms.util.views.Stub;
 import org.whispersystems.libsignal.util.guava.Optional;
+import org.whispersystems.signalservice.api.websocket.WebSocketConnectionState;

 import java.util.Collections;
 import java.util.HashSet;
@@ -968,19 +968,21 @@ public class ConversationListFragment extends MainFragment implements ActionMode
    }
  }

-  private void updateProxyStatus(@NonNull PipeConnectivityListener.State state) {
+  private void updateProxyStatus(@NonNull WebSocketConnectionState state) {
    if (SignalStore.proxy().isProxyEnabled()) {
      proxyStatus.setVisibility(View.VISIBLE);

      switch (state) {
        case CONNECTING:
+        case DISCONNECTING:
        case DISCONNECTED:
          proxyStatus.setImageResource(R.drawable.ic_proxy_connecting_24);
          break;
        case CONNECTED:
          proxyStatus.setImageResource(R.drawable.ic_proxy_connected_24);
          break;
-        case FAILURE:
+        case AUTHENTICATION_FAILED:
+        case FAILED:
          proxyStatus.setImageResource(R.drawable.ic_proxy_failed_24);
          break;
      }
--- a/app/src/main/java/org/thoughtcrime/securesms/conversationlist/ConversationListViewModel.java
+++ b/app/src/main/java/org/thoughtcrime/securesms/conversationlist/ConversationListViewModel.java
@@ -5,6 +5,7 @@ import android.text.TextUtils;

 import androidx.annotation.NonNull;
 import androidx.lifecycle.LiveData;
+import androidx.lifecycle.LiveDataReactiveStreams;
 import androidx.lifecycle.MutableLiveData;
 import androidx.lifecycle.ViewModel;
 import androidx.lifecycle.ViewModelProvider;
@@ -14,7 +15,6 @@ import org.signal.paging.PagedData;
 import org.signal.paging.PagingConfig;
 import org.signal.paging.PagingController;
 import org.thoughtcrime.securesms.conversationlist.model.Conversation;
-import org.thoughtcrime.securesms.search.SearchResult;
 import org.thoughtcrime.securesms.conversationlist.model.UnreadPayments;
 import org.thoughtcrime.securesms.conversationlist.model.UnreadPaymentsLiveData;
 import org.thoughtcrime.securesms.database.DatabaseFactory;
@@ -23,17 +23,20 @@ import org.thoughtcrime.securesms.dependencies.ApplicationDependencies;
 import org.thoughtcrime.securesms.megaphone.Megaphone;
 import org.thoughtcrime.securesms.megaphone.MegaphoneRepository;
 import org.thoughtcrime.securesms.megaphone.Megaphones;
-import org.thoughtcrime.securesms.net.PipeConnectivityListener;
 import org.thoughtcrime.securesms.payments.UnreadPaymentsRepository;
 import org.thoughtcrime.securesms.search.SearchRepository;
+import org.thoughtcrime.securesms.search.SearchResult;
 import org.thoughtcrime.securesms.util.Debouncer;
 import org.thoughtcrime.securesms.util.ThrottledDebouncer;
 import org.thoughtcrime.securesms.util.livedata.LiveDataUtil;
 import org.thoughtcrime.securesms.util.paging.Invalidator;
 import org.whispersystems.libsignal.util.guava.Optional;
+import org.whispersystems.signalservice.api.websocket.WebSocketConnectionState;

 import java.util.List;

+import io.reactivex.rxjava3.core.BackpressureStrategy;
+
 class ConversationListViewModel extends ViewModel {

  private static final String TAG = Log.tag(ConversationListViewModel.class);
@@ -117,8 +120,8 @@ class ConversationListViewModel extends ViewModel {
    return pagedData.getController();
  }

-  @NonNull LiveData<PipeConnectivityListener.State> getPipeState() {
-    return ApplicationDependencies.getPipeListener().getState();
+  @NonNull LiveData<WebSocketConnectionState> getPipeState() {
+    return LiveDataReactiveStreams.fromPublisher(ApplicationDependencies.getSignalWebSocket().getWebSocketState().toFlowable(BackpressureStrategy.LATEST));
  }

  @NonNull LiveData<Optional<UnreadPayments>> getUnreadPaymentsLiveData() {
--- a/app/src/main/java/org/thoughtcrime/securesms/dependencies/ApplicationDependencies.java
+++ b/app/src/main/java/org/thoughtcrime/securesms/dependencies/ApplicationDependencies.java
@@ -19,7 +19,6 @@ import org.thoughtcrime.securesms.megaphone.MegaphoneRepository;
 import org.thoughtcrime.securesms.messages.BackgroundMessageRetriever;
 import org.thoughtcrime.securesms.messages.IncomingMessageObserver;
 import org.thoughtcrime.securesms.messages.IncomingMessageProcessor;
-import org.thoughtcrime.securesms.net.PipeConnectivityListener;
 import org.thoughtcrime.securesms.net.StandardUserAgentInterceptor;
 import org.thoughtcrime.securesms.notifications.MessageNotifier;
 import org.thoughtcrime.securesms.payments.Payments;
@@ -112,10 +111,6 @@ public class ApplicationDependencies {
    return application;
  }

-  public static @NonNull PipeConnectivityListener getPipeListener() {
-    return provider.providePipeListener();
-  }
-
  public static @NonNull SignalServiceAccountManager getSignalServiceAccountManager() {
    SignalServiceAccountManager local = accountManager;

@@ -227,7 +222,6 @@ public class ApplicationDependencies {

  public static void resetNetworkConnectionsAfterProxyChange() {
    synchronized (LOCK) {
-      getPipeListener().reset();
      closeConnections();
    }
  }
@@ -509,7 +503,6 @@ public class ApplicationDependencies {
  }

  public interface Provider {
-    @NonNull PipeConnectivityListener providePipeListener();
    @NonNull GroupsV2Operations provideGroupsV2Operations();
    @NonNull SignalServiceAccountManager provideSignalServiceAccountManager();
    @NonNull SignalServiceMessageSender provideSignalServiceMessageSender(@NonNull SignalWebSocket signalWebSocket);
--- a/app/src/main/java/org/thoughtcrime/securesms/dependencies/ApplicationDependencyProvider.java
+++ b/app/src/main/java/org/thoughtcrime/securesms/dependencies/ApplicationDependencyProvider.java
@@ -6,7 +6,6 @@ import android.content.Context;
 import androidx.annotation.NonNull;

 import org.signal.core.util.concurrent.SignalExecutors;
-import org.signal.core.util.logging.Log;
 import org.thoughtcrime.securesms.BuildConfig;
 import org.thoughtcrime.securesms.components.TypingStatusRepository;
 import org.thoughtcrime.securesms.components.TypingStatusSender;
@@ -14,6 +13,7 @@ import org.thoughtcrime.securesms.crypto.ReentrantSessionLock;
 import org.thoughtcrime.securesms.crypto.storage.SignalProtocolStoreImpl;
 import org.thoughtcrime.securesms.database.DatabaseObserver;
 import org.thoughtcrime.securesms.database.JobDatabase;
+import org.thoughtcrime.securesms.database.PendingRetryReceiptCache;
 import org.thoughtcrime.securesms.jobmanager.JobManager;
 import org.thoughtcrime.securesms.jobmanager.JobMigrator;
 import org.thoughtcrime.securesms.jobmanager.impl.FactoryJobPredicate;
@@ -33,8 +33,7 @@ import org.thoughtcrime.securesms.megaphone.MegaphoneRepository;
 import org.thoughtcrime.securesms.messages.BackgroundMessageRetriever;
 import org.thoughtcrime.securesms.messages.IncomingMessageObserver;
 import org.thoughtcrime.securesms.messages.IncomingMessageProcessor;
-import org.thoughtcrime.securesms.database.PendingRetryReceiptCache;
-import org.thoughtcrime.securesms.net.PipeConnectivityListener;
+import org.thoughtcrime.securesms.net.SignalWebSocketHealthMonitor;
 import org.thoughtcrime.securesms.notifications.MessageNotifier;
 import org.thoughtcrime.securesms.notifications.OptimizedMessageNotifier;
 import org.thoughtcrime.securesms.payments.MobileCoinConfig;
@@ -75,25 +74,16 @@ import java.util.UUID;
 */
 public class ApplicationDependencyProvider implements ApplicationDependencies.Provider {

-  private static final String TAG = Log.tag(ApplicationDependencyProvider.class);
-
-  private final Application              context;
-  private final PipeConnectivityListener pipeListener;
+  private final Application context;

  public ApplicationDependencyProvider(@NonNull Application context) {
-    this.context      = context;
-    this.pipeListener = new PipeConnectivityListener(context);
+    this.context = context;
  }

  private @NonNull ClientZkOperations provideClientZkOperations() {
    return ClientZkOperations.create(provideSignalServiceNetworkAccess().getConfiguration(context));
  }

-  @Override
-  public @NonNull PipeConnectivityListener providePipeListener() {
-    return pipeListener;
-  }
-
  @Override
  public @NonNull GroupsV2Operations provideGroupsV2Operations() {
    return new GroupsV2Operations(provideClientZkOperations());
@@ -126,13 +116,9 @@ public class ApplicationDependencyProvider implements ApplicationDependencies.Pr

  @Override
  public @NonNull SignalServiceMessageReceiver provideSignalServiceMessageReceiver() {
-    SleepTimer sleepTimer = TextSecurePreferences.isFcmDisabled(context) ? new AlarmSleepTimer(context)
-                                                                         : new UptimeSleepTimer();
    return new SignalServiceMessageReceiver(provideSignalServiceNetworkAccess().getConfiguration(context),
                                            new DynamicCredentialsProvider(context),
                                            BuildConfig.SIGNAL_AGENT,
-                                            pipeListener,
-                                            sleepTimer,
                                            provideClientZkOperations().getProfileOperations(),
                                            FeatureFlags.okHttpAutomaticRetry());
  }
@@ -265,35 +251,33 @@ public class ApplicationDependencyProvider implements ApplicationDependencies.Pr

  @Override
  public @NonNull SignalWebSocket provideSignalWebSocket() {
-    return new SignalWebSocket(provideWebSocketFactory());
+    SleepTimer                   sleepTimer      = TextSecurePreferences.isFcmDisabled(context) ? new AlarmSleepTimer(context) : new UptimeSleepTimer();
+    SignalWebSocketHealthMonitor healthMonitor   = new SignalWebSocketHealthMonitor(context, sleepTimer);
+    SignalWebSocket              signalWebSocket = new SignalWebSocket(provideWebSocketFactory(healthMonitor));
+
+    healthMonitor.monitor(signalWebSocket);
+
+    return signalWebSocket;
  }

-  private @NonNull WebSocketFactory provideWebSocketFactory() {
+  private @NonNull WebSocketFactory provideWebSocketFactory(@NonNull SignalWebSocketHealthMonitor healthMonitor) {
    return new WebSocketFactory() {
      @Override
      public WebSocketConnection createWebSocket() {
-        SleepTimer sleepTimer = TextSecurePreferences.isFcmDisabled(context) ? new AlarmSleepTimer(context)
-                                                                             : new UptimeSleepTimer();
-
        return new WebSocketConnection("normal",
                                       provideSignalServiceNetworkAccess().getConfiguration(context),
                                       Optional.of(new DynamicCredentialsProvider(context)),
                                       BuildConfig.SIGNAL_AGENT,
-                                       pipeListener,
-                                       sleepTimer);
+                                       healthMonitor);
      }

      @Override
      public WebSocketConnection createUnidentifiedWebSocket() {
-        SleepTimer sleepTimer = TextSecurePreferences.isFcmDisabled(context) ? new AlarmSleepTimer(context)
-                                                                             : new UptimeSleepTimer();
-
        return new WebSocketConnection("unidentified",
                                       provideSignalServiceNetworkAccess().getConfiguration(context),
                                       Optional.absent(),
                                       BuildConfig.SIGNAL_AGENT,
-                                       pipeListener,
-                                       sleepTimer);
+                                       healthMonitor);
      }
    };
  }
--- a/app/src/main/java/org/thoughtcrime/securesms/messages/IncomingMessageObserver.java
+++ b/app/src/main/java/org/thoughtcrime/securesms/messages/IncomingMessageObserver.java
@@ -14,10 +14,12 @@ import androidx.annotation.Nullable;
 import androidx.core.app.NotificationCompat;
 import androidx.core.content.ContextCompat;

+import org.signal.core.util.ThreadUtil;
 import org.signal.core.util.concurrent.SignalExecutors;
 import org.signal.core.util.logging.Log;
 import org.thoughtcrime.securesms.R;
 import org.thoughtcrime.securesms.dependencies.ApplicationDependencies;
+import org.thoughtcrime.securesms.jobmanager.impl.BackoffUtil;
 import org.thoughtcrime.securesms.jobmanager.impl.NetworkConstraint;
 import org.thoughtcrime.securesms.jobs.PushDecryptDrainedJob;
 import org.thoughtcrime.securesms.keyvalue.SignalStore;
@@ -50,6 +52,7 @@ public class IncomingMessageObserver {
  private final Application                context;
  private final SignalServiceNetworkAccess networkAccess;
  private final List<Runnable>             decryptionDrainedListeners;
+  private final BroadcastReceiver          connectionReceiver;

  private boolean appVisible;

@@ -84,7 +87,7 @@ public class IncomingMessageObserver {
      }
    });

-    context.registerReceiver(new BroadcastReceiver() {
+    connectionReceiver = new BroadcastReceiver() {
      @Override
      public void onReceive(Context context, Intent intent) {
        synchronized (IncomingMessageObserver.this) {
@@ -92,12 +95,14 @@ public class IncomingMessageObserver {
            Log.w(TAG, "Lost network connection. Shutting down our websocket connections and resetting the drained state.");
            networkDrained    = false;
            decryptionDrained = false;
-            shutdown();
+            disconnect();
          }
          IncomingMessageObserver.this.notifyAll();
        }
      }
-    }, new IntentFilter(ConnectivityManager.CONNECTIVITY_ACTION));
+    };
+
+    context.registerReceiver(connectionReceiver, new IntentFilter(ConnectivityManager.CONNECTIVITY_ACTION));
  }

  public synchronized void notifyRegistrationChanged() {
@@ -169,14 +174,16 @@ public class IncomingMessageObserver {
  public void terminateAsync() {
    INSTANCE_COUNT.decrementAndGet();

+    context.unregisterReceiver(connectionReceiver);
+
    SignalExecutors.BOUNDED.execute(() -> {
      Log.w(TAG, "Beginning termination.");
      terminated = true;
-      shutdown();
+      disconnect();
    });
  }

-  private void shutdown() {
+  private void disconnect() {
    ApplicationDependencies.getSignalWebSocket().disconnect();
  }

@@ -190,8 +197,15 @@ public class IncomingMessageObserver {

    @Override
    public void run() {
+      int attempts = 0;
+
      while (!terminated) {
        Log.i(TAG, "Waiting for websocket state change....");
+        if (attempts > 1) {
+          long backoff = BackoffUtil.exponentialBackoff(attempts, TimeUnit.SECONDS.toMillis(30));
+          Log.w(TAG, "Too many failed connection attempts,  attempts: " + attempts + " backing off: " + backoff);
+          ThreadUtil.sleep(backoff);
+        }
        waitForConnectionNecessary();

        Log.i(TAG, "Making websocket connection....");
@@ -208,6 +222,7 @@ public class IncomingMessageObserver {
                  processor.processEnvelope(envelope);
                }
              });
+              attempts = 0;

              if (!result.isPresent() && !networkDrained) {
                Log.i(TAG, "Network was newly-drained. Enqueuing a job to listen for decryption draining.");
@@ -219,13 +234,15 @@ public class IncomingMessageObserver {
              signalWebSocket.connect();
            } catch (TimeoutException e) {
              Log.w(TAG, "Application level read timeout...");
+              attempts = 0;
            }
          }
        } catch (Throwable e) {
+          attempts++;
          Log.w(TAG, e);
        } finally {
          Log.w(TAG, "Shutting down pipe...");
-          shutdown();
+          disconnect();
        }

        Log.i(TAG, "Looping...");
--- a/app/src/main/java/org/thoughtcrime/securesms/net/HttpErrorTracker.java
+++ b/app/src/main/java/org/thoughtcrime/securesms/net/HttpErrorTracker.java
@@ -0,0 +1,44 @@
+package org.thoughtcrime.securesms.net;
+
+import java.util.Arrays;
+
+/**
+ * Track error occurrences by time and indicate if too many occur within the
+ * time limit.
+ */
+public final class HttpErrorTracker {
+
+  private final long[] timestamps;
+  private final long   errorTimeRange;
+
+  public HttpErrorTracker(int samples, long errorTimeRange) {
+    this.timestamps     = new long[samples];
+    this.errorTimeRange = errorTimeRange;
+  }
+
+  public synchronized boolean addSample(long now) {
+    long errorsMustBeAfter = now - errorTimeRange;
+    int  count             = 1;
+    int  minIndex          = 0;
+
+    for (int i = 0; i < timestamps.length; i++) {
+      if (timestamps[i] < errorsMustBeAfter) {
+        timestamps[i] = 0;
+      } else if (timestamps[i] != 0) {
+        count++;
+      }
+
+      if (timestamps[i] < timestamps[minIndex]) {
+        minIndex = i;
+      }
+    }
+
+    timestamps[minIndex] = now;
+
+    if (count >= timestamps.length) {
+      Arrays.fill(timestamps, 0);
+      return true;
+    }
+    return false;
+  }
+}
--- a/app/src/main/java/org/thoughtcrime/securesms/net/PipeConnectivityListener.java
+++ b/app/src/main/java/org/thoughtcrime/securesms/net/PipeConnectivityListener.java
@@ -1,89 +0,0 @@
-package org.thoughtcrime.securesms.net;
-
-import android.app.Application;
-
-import androidx.annotation.NonNull;
-
-import org.greenrobot.eventbus.EventBus;
-import org.signal.core.util.logging.Log;
-import org.thoughtcrime.securesms.dependencies.ApplicationDependencies;
-import org.thoughtcrime.securesms.events.ReminderUpdateEvent;
-import org.thoughtcrime.securesms.keyvalue.SignalStore;
-import org.thoughtcrime.securesms.util.DefaultValueLiveData;
-import org.thoughtcrime.securesms.util.TextSecurePreferences;
-import org.whispersystems.signalservice.api.websocket.ConnectivityListener;
-
-import okhttp3.Response;
-
-/**
- * Our standard listener for reacting to the state of the websocket. Translates the state into a
- * LiveData for observation.
- */
-public class PipeConnectivityListener implements ConnectivityListener {
-
-  private static final String TAG = Log.tag(PipeConnectivityListener.class);
-
-  private final Application                 application;
-  private final DefaultValueLiveData<State> state;
-
-  public PipeConnectivityListener(@NonNull Application application) {
-    this.application = application;
-    this.state       = new DefaultValueLiveData<>(State.DISCONNECTED);
-  }
-
-  @Override
-  public void onConnected() {
-    Log.i(TAG, "onConnected()");
-    TextSecurePreferences.setUnauthorizedReceived(application, false);
-    state.postValue(State.CONNECTED);
-  }
-
-  @Override
-  public void onConnecting() {
-    Log.i(TAG, "onConnecting()");
-    state.postValue(State.CONNECTING);
-  }
-
-  @Override
-  public void onDisconnected() {
-    Log.w(TAG, "onDisconnected()");
-
-    if (state.getValue() != State.FAILURE) {
-      state.postValue(State.DISCONNECTED);
-    }
-  }
-
-  @Override
-  public void onAuthenticationFailure() {
-    Log.w(TAG, "onAuthenticationFailure()");
-    TextSecurePreferences.setUnauthorizedReceived(application, true);
-    EventBus.getDefault().post(new ReminderUpdateEvent());
-    state.postValue(State.FAILURE);
-  }
-
-  @Override
-  public boolean onGenericFailure(Response response, Throwable throwable) {
-    Log.w(TAG, "onGenericFailure() Response: " + response, throwable);
-    state.postValue(State.FAILURE);
-
-    if (SignalStore.proxy().isProxyEnabled()) {
-      Log.w(TAG, "Encountered an error while we had a proxy set! Terminating the connection to prevent retry spam.");
-      ApplicationDependencies.closeConnections();
-      return false;
-    } else {
-      return true;
-    }
-  }
-
-  public void reset() {
-    state.postValue(State.DISCONNECTED);
-  }
-
-  public @NonNull DefaultValueLiveData<State> getState() {
-    return state;
-  }
-
-  public enum State {
-    DISCONNECTED, CONNECTING, CONNECTED, FAILURE
-  }
-}
--- a/app/src/main/java/org/thoughtcrime/securesms/net/SignalWebSocketHealthMonitor.java
+++ b/app/src/main/java/org/thoughtcrime/securesms/net/SignalWebSocketHealthMonitor.java
@@ -0,0 +1,171 @@
+package org.thoughtcrime.securesms.net;
+
+import android.app.Application;
+
+import androidx.annotation.NonNull;
+
+import org.greenrobot.eventbus.EventBus;
+import org.signal.core.util.logging.Log;
+import org.thoughtcrime.securesms.dependencies.ApplicationDependencies;
+import org.thoughtcrime.securesms.events.ReminderUpdateEvent;
+import org.thoughtcrime.securesms.keyvalue.SignalStore;
+import org.thoughtcrime.securesms.util.TextSecurePreferences;
+import org.whispersystems.libsignal.util.guava.Preconditions;
+import org.whispersystems.signalservice.api.SignalWebSocket;
+import org.whispersystems.signalservice.api.util.SleepTimer;
+import org.whispersystems.signalservice.api.websocket.HealthMonitor;
+import org.whispersystems.signalservice.api.websocket.WebSocketConnectionState;
+import org.whispersystems.signalservice.internal.websocket.WebSocketConnection;
+
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import io.reactivex.rxjava3.schedulers.Schedulers;
+
+/**
+ * Monitors the health of the identified and unidentified WebSockets. If either one appears to be
+ * unhealthy, will trigger restarting both.
+ * <p>
+ * The monitor is also responsible for sending heartbeats/keep-alive messages to prevent
+ * timeouts.
+ */
+public final class SignalWebSocketHealthMonitor implements HealthMonitor {
+
+  private static final String TAG = Log.tag(SignalWebSocketHealthMonitor.class);
+
+  private static final long KEEP_ALIVE_SEND_CADENCE              = TimeUnit.SECONDS.toMillis(WebSocketConnection.KEEPALIVE_TIMEOUT_SECONDS);
+  private static final long MAX_TIME_SINCE_SUCCESSFUL_KEEP_ALIVE = KEEP_ALIVE_SEND_CADENCE * 3;
+
+  private final Application     context;
+  private       SignalWebSocket signalWebSocket;
+  private final SleepTimer      sleepTimer;
+
+  private volatile KeepAliveSender keepAliveSender;
+
+  private final HealthState identified   = new HealthState();
+  private final HealthState unidentified = new HealthState();
+
+  public SignalWebSocketHealthMonitor(@NonNull Application context, @NonNull SleepTimer sleepTimer) {
+    this.context    = context;
+    this.sleepTimer = sleepTimer;
+  }
+
+  public void monitor(@NonNull SignalWebSocket signalWebSocket) {
+    Preconditions.checkNotNull(signalWebSocket);
+    Preconditions.checkArgument(this.signalWebSocket == null, "monitor can only be called once");
+
+    this.signalWebSocket = signalWebSocket;
+
+    //noinspection ResultOfMethodCallIgnored
+    signalWebSocket.getWebSocketState()
+                   .subscribeOn(Schedulers.computation())
+                   .observeOn(Schedulers.computation())
+                   .distinctUntilChanged()
+                   .subscribe(s -> onStateChange(s, identified));
+
+    //noinspection ResultOfMethodCallIgnored
+    signalWebSocket.getUnidentifiedWebSocketState()
+                   .subscribeOn(Schedulers.computation())
+                   .observeOn(Schedulers.computation())
+                   .distinctUntilChanged()
+                   .subscribe(s -> onStateChange(s, unidentified));
+  }
+
+  private synchronized void onStateChange(WebSocketConnectionState connectionState, HealthState healthState) {
+    switch (connectionState) {
+      case CONNECTED:
+        TextSecurePreferences.setUnauthorizedReceived(context, false);
+        break;
+      case AUTHENTICATION_FAILED:
+        TextSecurePreferences.setUnauthorizedReceived(context, true);
+        EventBus.getDefault().post(new ReminderUpdateEvent());
+        break;
+      case FAILED:
+        if (SignalStore.proxy().isProxyEnabled()) {
+          Log.w(TAG, "Encountered an error while we had a proxy set! Terminating the connection to prevent retry spam.");
+          ApplicationDependencies.closeConnections();
+        }
+        break;
+    }
+
+    healthState.needsKeepAlive = connectionState == WebSocketConnectionState.CONNECTED;
+
+    if (keepAliveSender == null && isKeepAliveNecessary()) {
+      keepAliveSender = new KeepAliveSender();
+      keepAliveSender.start();
+    } else if (keepAliveSender != null && !isKeepAliveNecessary()) {
+      keepAliveSender.shutdown();
+      keepAliveSender = null;
+    }
+  }
+
+  @Override
+  public void onKeepAliveResponse(long sentTimestamp, boolean isIdentifiedWebSocket) {
+    if (isIdentifiedWebSocket) {
+      identified.lastKeepAliveReceived = System.currentTimeMillis();
+    } else {
+      unidentified.lastKeepAliveReceived = System.currentTimeMillis();
+    }
+  }
+
+  @Override
+  public void onMessageError(int status, boolean isIdentifiedWebSocket) {
+    if (status == 409) {
+      HealthState healthState = (isIdentifiedWebSocket ? identified : unidentified);
+      if (healthState.mismatchErrorTracker.addSample(System.currentTimeMillis())) {
+        Log.w(TAG, "Received too many mismatch device errors, forcing new websockets.");
+        signalWebSocket.forceNewWebSockets();
+      }
+    }
+  }
+
+  private boolean isKeepAliveNecessary() {
+    return identified.needsKeepAlive || unidentified.needsKeepAlive;
+  }
+
+  private static class HealthState {
+    private final HttpErrorTracker mismatchErrorTracker = new HttpErrorTracker(5, TimeUnit.MINUTES.toMillis(1));
+
+    private volatile boolean needsKeepAlive;
+    private volatile long    lastKeepAliveReceived;
+  }
+
+  /**
+   * Sends periodic heartbeats/keep-alives over both WebSockets to prevent connection timeouts. If
+   * either WebSocket fails 3 times to get a return heartbeat both are forced to be recreated.
+   */
+  private class KeepAliveSender extends Thread {
+
+    private volatile boolean shouldKeepRunning = true;
+
+    public void run() {
+      identified.lastKeepAliveReceived   = System.currentTimeMillis();
+      unidentified.lastKeepAliveReceived = System.currentTimeMillis();
+
+      while (shouldKeepRunning && isKeepAliveNecessary()) {
+        try {
+          sleepTimer.sleep(KEEP_ALIVE_SEND_CADENCE);
+
+          if (shouldKeepRunning && isKeepAliveNecessary()) {
+            long keepAliveRequiredSinceTime = System.currentTimeMillis() - MAX_TIME_SINCE_SUCCESSFUL_KEEP_ALIVE;
+
+            if (identified.lastKeepAliveReceived < keepAliveRequiredSinceTime || unidentified.lastKeepAliveReceived < keepAliveRequiredSinceTime) {
+              Log.w(TAG, "Missed keep alives, identified last: " + identified.lastKeepAliveReceived +
+                         " unidentified last: " + unidentified.lastKeepAliveReceived +
+                         " needed by: " + keepAliveRequiredSinceTime);
+              signalWebSocket.forceNewWebSockets();
+            } else {
+              signalWebSocket.sendKeepAlive();
+            }
+          }
+        } catch (Throwable e) {
+          Log.w(TAG, e);
+        }
+      }
+    }
+
+    public void shutdown() {
+      shouldKeepRunning = false;
+    }
+  }
+}
--- a/app/src/main/java/org/thoughtcrime/securesms/preferences/EditProxyFragment.java
+++ b/app/src/main/java/org/thoughtcrime/securesms/preferences/EditProxyFragment.java
@@ -21,13 +21,13 @@ import com.dd.CircularProgressButton;
 import org.thoughtcrime.securesms.R;
 import org.thoughtcrime.securesms.contactshare.SimpleTextWatcher;
 import org.thoughtcrime.securesms.keyvalue.SignalStore;
-import org.thoughtcrime.securesms.net.PipeConnectivityListener;
 import org.thoughtcrime.securesms.util.CommunicationActions;
 import org.thoughtcrime.securesms.util.SignalProxyUtil;
 import org.thoughtcrime.securesms.util.Util;
 import org.thoughtcrime.securesms.util.ViewUtil;
 import org.thoughtcrime.securesms.util.views.LearnMoreTextView;
 import org.whispersystems.libsignal.util.guava.Optional;
+import org.whispersystems.signalservice.api.websocket.WebSocketConnectionState;
 import org.whispersystems.signalservice.internal.configuration.SignalProxy;

 public class EditProxyFragment extends Fragment {
@@ -118,10 +118,11 @@ public class EditProxyFragment extends Fragment {
    }
  }

-  private void presentProxyState(@NonNull PipeConnectivityListener.State proxyState) {
+  private void presentProxyState(@NonNull WebSocketConnectionState proxyState) {
    if (SignalStore.proxy().getProxy() != null) {
      switch (proxyState) {
        case DISCONNECTED:
+        case DISCONNECTING:
        case CONNECTING:
          proxyStatus.setText(R.string.preferences_connecting_to_proxy);
          proxyStatus.setTextColor(getResources().getColor(R.color.signal_text_secondary));
@@ -130,7 +131,8 @@ public class EditProxyFragment extends Fragment {
          proxyStatus.setText(R.string.preferences_connected_to_proxy);
          proxyStatus.setTextColor(getResources().getColor(R.color.signal_accent_green));
          break;
-        case FAILURE:
+        case AUTHENTICATION_FAILED:
+        case FAILED:
          proxyStatus.setText(R.string.preferences_connection_failed);
          proxyStatus.setTextColor(getResources().getColor(R.color.signal_alert_primary));
          break;
--- a/app/src/main/java/org/thoughtcrime/securesms/preferences/EditProxyViewModel.java
+++ b/app/src/main/java/org/thoughtcrime/securesms/preferences/EditProxyViewModel.java
@@ -8,28 +8,34 @@ import androidx.lifecycle.ViewModel;
 import org.signal.core.util.concurrent.SignalExecutors;
 import org.thoughtcrime.securesms.dependencies.ApplicationDependencies;
 import org.thoughtcrime.securesms.keyvalue.SignalStore;
-import org.thoughtcrime.securesms.net.PipeConnectivityListener;
 import org.thoughtcrime.securesms.util.SignalProxyUtil;
 import org.thoughtcrime.securesms.util.SingleLiveEvent;
 import org.thoughtcrime.securesms.util.TextSecurePreferences;
 import org.thoughtcrime.securesms.util.Util;
+import org.whispersystems.signalservice.api.websocket.WebSocketConnectionState;
 import org.whispersystems.signalservice.internal.configuration.SignalProxy;

 import java.util.concurrent.TimeUnit;

+import io.reactivex.rxjava3.core.BackpressureStrategy;
+
+import static androidx.lifecycle.LiveDataReactiveStreams.fromPublisher;
+
 public class EditProxyViewModel extends ViewModel {

-  private final SingleLiveEvent<Event>                   events;
-  private final MutableLiveData<UiState>                 uiState;
-  private final MutableLiveData<SaveState>               saveState;
-  private final LiveData<PipeConnectivityListener.State> pipeState;
+  private final SingleLiveEvent<Event>             events;
+  private final MutableLiveData<UiState>           uiState;
+  private final MutableLiveData<SaveState>         saveState;
+  private final LiveData<WebSocketConnectionState> pipeState;

  public EditProxyViewModel() {
    this.events    = new SingleLiveEvent<>();
    this.uiState   = new MutableLiveData<>();
    this.saveState = new MutableLiveData<>(SaveState.IDLE);
    this.pipeState = TextSecurePreferences.getLocalNumber(ApplicationDependencies.getApplication()) == null ? new MutableLiveData<>()
-                                                                                                            : ApplicationDependencies.getPipeListener().getState();
+                                                                                                            : fromPublisher(ApplicationDependencies.getSignalWebSocket()
+                                                                                                                                                   .getWebSocketState()
+                                                                                                                                                   .toFlowable(BackpressureStrategy.LATEST));

    if (SignalStore.proxy().isProxyEnabled()) {
      uiState.setValue(UiState.ALL_ENABLED);
@@ -81,7 +87,7 @@ public class EditProxyViewModel extends ViewModel {
    return events;
  }

-  @NonNull LiveData<PipeConnectivityListener.State> getProxyState() {
+  @NonNull LiveData<WebSocketConnectionState> getProxyState() {
    return pipeState;
  }

--- a/app/src/main/java/org/thoughtcrime/securesms/util/SignalProxyUtil.java
+++ b/app/src/main/java/org/thoughtcrime/securesms/util/SignalProxyUtil.java
@@ -3,17 +3,15 @@ package org.thoughtcrime.securesms.util;
 import androidx.annotation.NonNull;
 import androidx.annotation.Nullable;
 import androidx.annotation.WorkerThread;
-import androidx.lifecycle.Observer;

 import org.conscrypt.Conscrypt;
-import org.signal.core.util.ThreadUtil;
 import org.signal.core.util.concurrent.SignalExecutors;
 import org.signal.core.util.logging.Log;
 import org.thoughtcrime.securesms.dependencies.ApplicationDependencies;
 import org.thoughtcrime.securesms.keyvalue.SignalStore;
-import org.thoughtcrime.securesms.net.PipeConnectivityListener;
 import org.thoughtcrime.securesms.push.AccountManagerFactory;
 import org.whispersystems.signalservice.api.SignalServiceAccountManager;
+import org.whispersystems.signalservice.api.websocket.WebSocketConnectionState;
 import org.whispersystems.signalservice.internal.configuration.SignalProxy;

 import java.io.IOException;
@@ -23,6 +21,9 @@ import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;

+import io.reactivex.rxjava3.core.Single;
+import io.reactivex.rxjava3.schedulers.Schedulers;
+
 public final class SignalProxyUtil {

  private static final String TAG = Log.tag(SignalProxyUtil.class);
@@ -35,7 +36,7 @@ public final class SignalProxyUtil {
  private SignalProxyUtil() {}

  public static void startListeningToWebsocket() {
-    if (SignalStore.proxy().isProxyEnabled() && ApplicationDependencies.getPipeListener().getState().getValue() == PipeConnectivityListener.State.FAILURE) {
+    if (SignalStore.proxy().isProxyEnabled() && ApplicationDependencies.getSignalWebSocket().getWebSocketState().firstOrError().blockingGet().isFailure()) {
      Log.w(TAG, "Proxy is in a failed state. Restarting.");
      ApplicationDependencies.closeConnections();
    }
@@ -81,30 +82,16 @@ public final class SignalProxyUtil {
      return testWebsocketConnectionUnregistered(timeout);
    }

-    CountDownLatch latch   = new CountDownLatch(1);
-    AtomicBoolean  success = new AtomicBoolean(false);
-
-    Observer<PipeConnectivityListener.State> observer = state -> {
-      if (state == PipeConnectivityListener.State.CONNECTED) {
-        success.set(true);
-        latch.countDown();
-      } else if (state == PipeConnectivityListener.State.FAILURE) {
-        success.set(false);
-        latch.countDown();
-      }
-    };
-
-    ThreadUtil.runOnMainSync(() -> ApplicationDependencies.getPipeListener().getState().observeForever(observer));
-
-    try {
-      latch.await(timeout, TimeUnit.MILLISECONDS);
-    } catch (InterruptedException e) {
-      Log.w(TAG, "Interrupted!", e);
-    } finally {
-      ThreadUtil.runOnMainSync(() -> ApplicationDependencies.getPipeListener().getState().removeObserver(observer));
-    }
-
-    return success.get();
+    return ApplicationDependencies.getSignalWebSocket()
+                                  .getWebSocketState()
+                                  .subscribeOn(Schedulers.trampoline())
+                                  .observeOn(Schedulers.trampoline())
+                                  .timeout(timeout, TimeUnit.MILLISECONDS)
+                                  .skipWhile(state -> state != WebSocketConnectionState.CONNECTED && !state.isFailure())
+                                  .firstOrError()
+                                  .flatMap(state -> Single.just(state == WebSocketConnectionState.CONNECTED))
+                                  .onErrorReturn(t -> false)
+                                  .blockingGet();
  }

  /**