From 489b58ad6700276fdcb3a644a2030ce35e128700 Mon Sep 17 00:00:00 2001
From: Nicholas Tinsley <nicholas@signal.org>
Date: Tue, 11 Jun 2024 13:05:32 -0400
Subject: [PATCH] Abort transcoding if frame processing gets stuck.

---
 .../videoconverter/AudioTrackConverter.java   | 20 +++++++------------
 .../video/videoconverter/MediaConverter.java  | 20 +++++++++++++++----
 .../videoconverter/MediaConverterState.kt     | 12 +++++++++++
 .../videoconverter/VideoTrackConverter.java   | 17 ++++++----------
 4 files changed, 41 insertions(+), 28 deletions(-)
 create mode 100644 video/lib/src/main/java/org/thoughtcrime/securesms/video/videoconverter/MediaConverterState.kt

diff --git a/video/lib/src/main/java/org/thoughtcrime/securesms/video/videoconverter/AudioTrackConverter.java b/video/lib/src/main/java/org/thoughtcrime/securesms/video/videoconverter/AudioTrackConverter.java
index a597702d83..d5d2380d32 100644
--- a/video/lib/src/main/java/org/thoughtcrime/securesms/video/videoconverter/AudioTrackConverter.java
+++ b/video/lib/src/main/java/org/thoughtcrime/securesms/video/videoconverter/AudioTrackConverter.java
@@ -398,19 +398,13 @@ final class AudioTrackConverter {
         }
     }
 
-    String dumpState() {
-        return String.format(Locale.US,
-                "A{"
-                        + "extracted:%d(done:%b) "
-                        + "decoded:%d(done:%b) "
-                        + "encoded:%d(done:%b) "
-                        + "pending:%d "
-                        + "muxing:%b(track:%d} )",
-                mAudioExtractedFrameCount, mAudioExtractorDone,
-                mAudioDecodedFrameCount, mAudioDecoderDone,
-                mAudioEncodedFrameCount, mAudioEncoderDone,
-                mPendingAudioDecoderOutputBufferIndex,
-                mMuxer != null, mOutputAudioTrack);
+    AudioTrackConverterState dumpState() {
+        return new AudioTrackConverterState(
+            mAudioExtractedFrameCount, mAudioExtractorDone,
+            mAudioDecodedFrameCount, mAudioDecoderDone,
+            mAudioEncodedFrameCount, mAudioEncoderDone,
+            mPendingAudioDecoderOutputBufferIndex,
+            mMuxer != null, mOutputAudioTrack);
     }
 
     void verifyEndState() {
diff --git a/video/lib/src/main/java/org/thoughtcrime/securesms/video/videoconverter/MediaConverter.java b/video/lib/src/main/java/org/thoughtcrime/securesms/video/videoconverter/MediaConverter.java
index 7d780d50ea..c7ad42ee2a 100644
--- a/video/lib/src/main/java/org/thoughtcrime/securesms/video/videoconverter/MediaConverter.java
+++ b/video/lib/src/main/java/org/thoughtcrime/securesms/video/videoconverter/MediaConverter.java
@@ -47,6 +47,8 @@ public final class MediaConverter {
     private static final String TAG = "media-converter";
     private static final boolean VERBOSE = false; // lots of logging
 
+    private static final int STUCK_FRAME_THRESHOLD = 100;
+
     // Describes when the annotation will be discarded
     @Retention(RetentionPolicy.SOURCE)
     @StringDef({VIDEO_CODEC_H264, VIDEO_CODEC_H265})
@@ -217,6 +219,8 @@ public final class MediaConverter {
             final @Nullable AudioTrackConverter audioTrackConverter,
             final @NonNull Muxer muxer) throws IOException, TranscodingException {
 
+        MediaConverterState oldState = null;
+        int stuckFrames = 0;
         boolean muxing = false;
         int percentProcessed = 0;
         long inputDuration = Math.max(
@@ -227,11 +231,19 @@ public final class MediaConverter {
                 ((videoTrackConverter != null && !videoTrackConverter.mVideoEncoderDone) ||
                  (audioTrackConverter != null &&!audioTrackConverter.mAudioEncoderDone))) {
 
+            final MediaConverterState currentState = new MediaConverterState(videoTrackConverter != null ? videoTrackConverter.dumpState() : null, audioTrackConverter != null ? audioTrackConverter.dumpState() : null, muxing);
+
             if (VERBOSE) {
-                Log.d(TAG, "loop: " +
-                        (videoTrackConverter == null ? "" : videoTrackConverter.dumpState()) +
-                        (audioTrackConverter == null ? "" : audioTrackConverter.dumpState()) +
-                        " muxing:" + muxing);
+                Log.d(TAG, "loop: " + currentState);
+            }
+
+            if (currentState.equals(oldState)) {
+                if (++stuckFrames >= STUCK_FRAME_THRESHOLD) {
+                    mCancelled = true;
+                }
+            } else {
+                oldState = currentState;
+                stuckFrames = 0;
             }
 
             if (videoTrackConverter != null && (audioTrackConverter == null || audioTrackConverter.mAudioExtractorDone || videoTrackConverter.mMuxingVideoPresentationTime <= audioTrackConverter.mMuxingAudioPresentationTime)) {
diff --git a/video/lib/src/main/java/org/thoughtcrime/securesms/video/videoconverter/MediaConverterState.kt b/video/lib/src/main/java/org/thoughtcrime/securesms/video/videoconverter/MediaConverterState.kt
new file mode 100644
index 0000000000..b59bde8763
--- /dev/null
+++ b/video/lib/src/main/java/org/thoughtcrime/securesms/video/videoconverter/MediaConverterState.kt
@@ -0,0 +1,12 @@
+/*
+ * Copyright 2024 Signal Messenger, LLC
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+package org.thoughtcrime.securesms.video.videoconverter
+
+data class MediaConverterState(val videoTrack: VideoTrackConverterState?, val audioTrack: AudioTrackConverterState?, val muxing: Boolean)
+
+data class VideoTrackConverterState(val extractedCount: Long, val extractedDone: Boolean, val decodedCount: Long, val decodedDone: Boolean, val encodedCount: Long, val encodedDone: Boolean, val muxing: Boolean, val trackIndex: Int)
+
+data class AudioTrackConverterState(val extractedCount: Long, val extractedDone: Boolean, val decodedCount: Long, val decodedDone: Boolean, val encodedCount: Long, val encodedDone: Boolean, val pendingBufferIndex: Int, val muxing: Boolean, val trackIndex: Int)
diff --git a/video/lib/src/main/java/org/thoughtcrime/securesms/video/videoconverter/VideoTrackConverter.java b/video/lib/src/main/java/org/thoughtcrime/securesms/video/videoconverter/VideoTrackConverter.java
index 961dfed6bb..9fd19564e1 100644
--- a/video/lib/src/main/java/org/thoughtcrime/securesms/video/videoconverter/VideoTrackConverter.java
+++ b/video/lib/src/main/java/org/thoughtcrime/securesms/video/videoconverter/VideoTrackConverter.java
@@ -429,17 +429,12 @@ final class VideoTrackConverter {
         }
     }
 
-    String dumpState() {
-        return String.format(Locale.US,
-                "V{"
-                        + "extracted:%d(done:%b) "
-                        + "decoded:%d(done:%b) "
-                        + "encoded:%d(done:%b) "
-                        + "muxing:%b(track:%d)} ",
-                mVideoExtractedFrameCount, mVideoExtractorDone,
-                mVideoDecodedFrameCount, mVideoDecoderDone,
-                mVideoEncodedFrameCount, mVideoEncoderDone,
-                mMuxer != null, mOutputVideoTrack);
+    VideoTrackConverterState dumpState() {
+        return new VideoTrackConverterState(
+            mVideoExtractedFrameCount, mVideoExtractorDone,
+            mVideoDecodedFrameCount, mVideoDecoderDone,
+            mVideoEncodedFrameCount, mVideoEncoderDone,
+            mMuxer != null, mOutputVideoTrack);
     }
 
     void verifyEndState() {