Improve video encoder/decoder fallback logic.

demo/video/src/androidTest/java/org/thoughtcrime/video/app/transcode/VideoTranscodeInstrumentationTest.kt

@@ -19,13 +19,13 @@ import org.junit.runner.RunWith
 import org.thoughtcrime.securesms.video.StreamingTranscoder
 import org.thoughtcrime.securesms.video.TranscodingPreset
 import org.thoughtcrime.securesms.video.exceptions.VideoSourceException
+import org.thoughtcrime.securesms.video.videoconverter.exceptions.CodecUnavailableException
 import org.thoughtcrime.securesms.video.videoconverter.exceptions.EncodingException
 import org.thoughtcrime.securesms.video.videoconverter.exceptions.HdrDecoderUnavailableException
 import org.thoughtcrime.securesms.video.videoconverter.mediadatasource.InputStreamMediaDataSource
 import java.io.File
 import java.io.FileInputStream
 import java.io.FileNotFoundException
-import java.io.IOException
 import java.io.InputStream
 
 /**
@@ -105,10 +105,9 @@ class VideoTranscodeInstrumentationTest {
       } catch (e: FileNotFoundException) {
         Log.w(TAG, "Skipping '$videoFileName' with preset ${preset.name}: encoder not available on this device", e)
       } catch (e: EncodingException) {
-        val isHdrDeviceLimitation = e.isHdrInput && (!e.toneMapApplied || isMediaCodecRuntimeFailure(e))
+        val isHdrDeviceLimitation = e.isHdrInput && !e.toneMapApplied
-        val isCodecRuntimeFailure = isMediaCodecRuntimeFailure(e)
+        if (isHdrDeviceLimitation) {
-        if (isHdrDeviceLimitation || isCodecRuntimeFailure) {
+          Log.w(TAG, "Video '$videoFileName' failed with preset ${preset.name} (HDR device limitation, toneMap=${e.toneMapApplied}, decoder=${e.decoderName}, encoder=${e.encoderName})", e)
-          Log.w(TAG, "Video '$videoFileName' failed with preset ${preset.name} (device limitation, hdr=${e.isHdrInput}, toneMap=${e.toneMapApplied}, decoder=${e.decoderName}, encoder=${e.encoderName})", e)
           deviceWarnings.add("$videoFileName [${preset.name}]: ${e::class.simpleName}: ${e.message}")
         } else {
           Log.e(TAG, "Failed to transcode '$videoFileName' with preset ${preset.name} (hdr=${e.isHdrInput}, toneMap=${e.toneMapApplied}, decoder=${e.decoderName}, encoder=${e.encoderName})", e)
@@ -117,8 +116,8 @@ class VideoTranscodeInstrumentationTest {
       } catch (e: VideoSourceException) {
         Log.w(TAG, "Device cannot read video source '$videoFileName' with preset ${preset.name} (device limitation)", e)
         deviceWarnings.add("$videoFileName [${preset.name}]: ${e::class.simpleName}: ${e.message}")
-      } catch (e: IOException) {
+      } catch (e: CodecUnavailableException) {
-        Log.w(TAG, "Device cannot decode/encode '$videoFileName' with preset ${preset.name} (device limitation)", e)
+        Log.w(TAG, "All codecs exhausted for '$videoFileName' with preset ${preset.name} (device limitation)", e)
         deviceWarnings.add("$videoFileName [${preset.name}]: ${e::class.simpleName}: ${e.message}")
       } catch (e: Exception) {
         Log.e(TAG, "Failed to transcode '$videoFileName' with preset ${preset.name}", e)
@@ -138,29 +137,6 @@ class VideoTranscodeInstrumentationTest {
     }
   }
 
-  /**
-   * Checks if the root cause of an exception indicates a device/content limitation
-   * rather than a bug in the transcoding code. This covers:
-   * - [IllegalStateException] from [android.media.MediaCodec] native methods (hardware codec crash)
-   * - Frame count mismatches from [VideoTrackConverter.verifyEndState] (unusual video formats
-   *   like spatial video that some decoders handle differently)
-   */
-  private fun isMediaCodecRuntimeFailure(e: Exception): Boolean {
-    var cause: Throwable? = e.cause
-    while (cause != null) {
-      if (cause is IllegalStateException) {
-        if (cause.stackTrace.any { it.className == "android.media.MediaCodec" }) {
-          return true
-        }
-        if (cause.message?.contains("frame counts should match") == true) {
-          return true
-        }
-      }
-      cause = cause.cause
-    }
-    return false
-  }
-
   private fun transcodeVideo(videoFileName: String, preset: TranscodingPreset) {
     val inputFile = createTempFile("input-", "-$videoFileName")
     val outputFile = createTempFile("output-${preset.name}-", "-$videoFileName")

lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/MediaConverter.java

@@ -149,6 +149,56 @@ public final class MediaConverter {
      */
     @WorkerThread
     public long convert() throws EncodingException, IOException {
+        final Set<String> excludedDecoders = new HashSet<>();
+
+        while (true) {
+            mCancelled = false;
+            try {
+                return doConvert(excludedDecoders);
+            } catch (EncodingException e) {
+                if (e.decoderName != null
+                        && isRetryableMidStreamFailure(e)
+                        && excludedDecoders.add(e.decoderName)) {
+                    Log.w(TAG, "Mid-stream codec failure with decoder " + e.decoderName
+                            + ", retrying with it excluded (" + excludedDecoders.size()
+                            + " decoder(s) excluded)");
+                    continue;
+                }
+                throw e;
+            }
+        }
+    }
+
+    /**
+     * Checks whether the given {@link EncodingException} represents a mid-stream codec failure
+     * that may succeed with a different decoder. This covers:
+     * <ul>
+     *     <li>{@link IllegalStateException} from {@link android.media.MediaCodec} native methods
+     *         (hardware codec crash during encoding/decoding)</li>
+     *     <li>Frame count mismatches from {@link VideoTrackConverter#verifyEndState()} (unusual
+     *         video formats like spatial video that some decoders handle differently)</li>
+     * </ul>
+     */
+    private static boolean isRetryableMidStreamFailure(final @NonNull EncodingException e) {
+        Throwable cause = e.getCause();
+        while (cause != null) {
+            if (cause instanceof IllegalStateException) {
+                for (StackTraceElement frame : cause.getStackTrace()) {
+                    if ("android.media.MediaCodec".equals(frame.getClassName())) {
+                        return true;
+                    }
+                }
+                String message = cause.getMessage();
+                if (message != null && message.contains("frame counts should match")) {
+                    return true;
+                }
+            }
+            cause = cause.getCause();
+        }
+        return false;
+    }
+
+    private long doConvert(final @NonNull Set<String> excludedDecoders) throws EncodingException, IOException {
         // Exception that may be thrown during release.
         Exception           exception           = null;
         Muxer               muxer               = null;
@@ -162,7 +212,7 @@ public final class MediaConverter {
         try {
             muxer = mOutput.createMuxer();
 
-            videoTrackConverter = VideoTrackConverter.create(mInput, mTimeFrom, mTimeTo, mVideoResolution, mVideoBitrate, mVideoCodec);
+            videoTrackConverter = VideoTrackConverter.create(mInput, mTimeFrom, mTimeTo, mVideoResolution, mVideoBitrate, mVideoCodec, excludedDecoders);
             audioTrackConverter = AudioTrackConverter.create(mInput, mTimeFrom, mTimeTo, mAudioBitrate, mAllowAudioRemux && muxer.supportsAudioRemux());
 
             if (videoTrackConverter == null && audioTrackConverter == null) {

lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/VideoTrackConverter.java

@@ -13,6 +13,7 @@ import androidx.annotation.Nullable;
 import org.signal.core.util.logging.Log;
 import org.thoughtcrime.securesms.video.interfaces.MediaInput;
 import org.thoughtcrime.securesms.video.interfaces.Muxer;
+import org.thoughtcrime.securesms.video.videoconverter.exceptions.CodecUnavailableException;
 import org.thoughtcrime.securesms.video.videoconverter.exceptions.HdrDecoderUnavailableException;
 import org.thoughtcrime.securesms.video.videoconverter.utils.Extensions;
 import org.thoughtcrime.securesms.video.videoconverter.utils.MediaCodecCompat;
@@ -21,7 +22,9 @@ import org.thoughtcrime.securesms.video.videoconverter.utils.Preconditions;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.nio.ByteBuffer;
+import java.util.ArrayList;
 import java.util.List;
+import java.util.Set;
 import kotlin.Pair;
 
 final class VideoTrackConverter {
@@ -51,9 +54,9 @@ final class VideoTrackConverter {
 
     private final MediaExtractor mVideoExtractor;
     private final MediaCodec mVideoDecoder;
-    private final MediaCodec mVideoEncoder;
+    private MediaCodec mVideoEncoder;
 
-    private final InputSurface mInputSurface;
+    private InputSurface mInputSurface;
     private final OutputSurface mOutputSurface;
 
     private final ByteBuffer[] mVideoDecoderInputBuffers;
@@ -83,7 +86,8 @@ final class VideoTrackConverter {
             final long timeTo,
             final int videoResolution,
             final int videoBitrate,
-            final @NonNull String videoCodec) throws IOException, TranscodingException {
+            final @NonNull String videoCodec,
+            final @NonNull Set<String> excludedDecoders) throws IOException, TranscodingException {
 
         final MediaExtractor videoExtractor = input.createExtractor();
         final int videoInputTrack = getAndSelectVideoTrackIndex(videoExtractor);
@@ -91,7 +95,7 @@ final class VideoTrackConverter {
             videoExtractor.release();
             return null;
         }
-        return new VideoTrackConverter(videoExtractor, videoInputTrack, timeFrom, timeTo, videoResolution, videoBitrate, videoCodec);
+        return new VideoTrackConverter(videoExtractor, videoInputTrack, timeFrom, timeTo, videoResolution, videoBitrate, videoCodec, excludedDecoders);
     }
 
 
@@ -102,7 +106,8 @@ final class VideoTrackConverter {
             final long timeTo,
             final int videoResolution,
             final int videoBitrate,
-            final @NonNull String videoCodec) throws IOException, TranscodingException {
+            final @NonNull String videoCodec,
+            final @NonNull Set<String> excludedDecoders) throws IOException, TranscodingException {
 
         mTimeFrom = timeFrom;
         mTimeTo = timeTo;
@@ -167,16 +172,19 @@ final class VideoTrackConverter {
                 inputVideoFormat.getInteger(MediaFormat.KEY_WIDTH), inputVideoFormat.getInteger(MediaFormat.KEY_HEIGHT),
                 outputWidth, outputHeight);
 
-        // Configure the encoder but do NOT start it yet. The encoder's start() is
+        // Create encoder, decoder, and surfaces. The encoder's start() is deferred
-        // deferred until after the decoder is created, so that the decoder gets first
+        // until after the decoder is created, so that the decoder gets first access to
-        // access to hardware codec resources on memory-constrained devices.
+        // hardware codec resources on memory-constrained devices. If start() fails
+        // (e.g. NO_MEMORY on a resource-constrained device), we try the next encoder
+        // candidate while keeping the same decoder and OutputSurface.
         mVideoEncoder = createVideoEncoder(videoCodecCandidates, outputVideoFormat);
         mInputSurface = new InputSurface(mVideoEncoder.createInputSurface());
         mInputSurface.makeCurrent();
         mOutputSurface = new OutputSurface();
         mOutputSurface.changeFragmentShader(fragmentShader);
-        mVideoDecoder = createVideoDecoder(inputVideoFormat, mOutputSurface.getSurface());
+        mVideoDecoder = createVideoDecoder(inputVideoFormat, mOutputSurface.getSurface(), excludedDecoders);
-        mVideoEncoder.start();
+        startEncoderWithFallback(videoCodecCandidates, outputVideoFormat);
+
         mVideoDecoderInputBuffers = mVideoDecoder.getInputBuffers();
         mVideoEncoderOutputBuffers = mVideoEncoder.getOutputBuffers();
         mVideoDecoderOutputBufferInfo = new MediaCodec.BufferInfo();
@@ -483,10 +491,17 @@ final class VideoTrackConverter {
     private @NonNull
     MediaCodec createVideoDecoder(
             final @NonNull MediaFormat inputFormat,
-            final @NonNull Surface surface) throws IOException {
+            final @NonNull Surface surface,
+            final @NonNull Set<String> excludedDecoders) throws IOException {
         final boolean               isHdr              = MediaCodecCompat.isHdrVideo(inputFormat);
         final boolean               requestToneMapping = Build.VERSION.SDK_INT >= 31 && isHdr;
-        final List<Pair<String, MediaFormat>> candidates = MediaCodecCompat.findDecoderCandidates(inputFormat);
+        final List<Pair<String, MediaFormat>> allCandidates = MediaCodecCompat.findDecoderCandidates(inputFormat);
+        final List<Pair<String, MediaFormat>> candidates    = new ArrayList<>();
+        for (Pair<String, MediaFormat> c : allCandidates) {
+            if (!excludedDecoders.contains(c.getFirst())) {
+                candidates.add(c);
+            }
+        }
 
         mIsHdrInput = isHdr;
         Exception lastException = null;
@@ -545,7 +560,7 @@ final class VideoTrackConverter {
         if (mIsHdrInput) {
             throw new HdrDecoderUnavailableException("All video decoder codecs failed for HDR video", lastException);
         }
-        throw new IOException("All video decoder codecs failed", lastException);
+        throw new CodecUnavailableException("All video decoder codecs failed", lastException);
     }
 
     /**
@@ -580,7 +595,59 @@ final class VideoTrackConverter {
             }
         }
 
-        throw new IOException("All video encoder codecs failed", lastException);
+        throw new CodecUnavailableException("All video encoder codecs failed", lastException);
+    }
+
+    /**
+     * Attempts to start the current encoder ({@link #mVideoEncoder}). If start() fails,
+     * iterates through the remaining encoder candidates from {@code codecCandidates},
+     * replacing the encoder and its {@link InputSurface} on each attempt. The decoder
+     * and {@link OutputSurface} are independent of the encoder and remain unchanged.
+     */
+    private void startEncoderWithFallback(
+            final @NonNull List<MediaCodecInfo> codecCandidates,
+            final @NonNull MediaFormat format) throws IOException {
+        Exception lastException = null;
+
+        for (int i = 0; i < codecCandidates.size(); i++) {
+            final MediaCodecInfo codecInfo = codecCandidates.get(i);
+
+            if (i > 0) {
+                // Replace the encoder with the next candidate.
+                mVideoEncoder.release();
+                mInputSurface.release();
+
+                try {
+                    mVideoEncoder = MediaCodec.createByCodecName(codecInfo.getName());
+                    mVideoEncoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
+                    mInputSurface = new InputSurface(mVideoEncoder.createInputSurface());
+                    mInputSurface.makeCurrent();
+                    mEncoderName = codecInfo.getName();
+                } catch (IllegalArgumentException | IllegalStateException | TranscodingException e) {
+                    Log.w(TAG, "Video encoder: codec " + codecInfo.getName() + " failed to configure (attempt " + (i + 1) + " of " + codecCandidates.size() + ")", e);
+                    lastException = e;
+                    continue;
+                }
+            } else if (!codecInfo.getName().equals(mEncoderName)) {
+                // First iteration but createVideoEncoder selected a different codec
+                // (i.e. the first candidate failed to configure). Skip until we reach
+                // the one that was actually configured.
+                continue;
+            }
+
+            try {
+                mVideoEncoder.start();
+                if (i > 0) {
+                    Log.w(TAG, "Video encoder: succeeded with fallback codec " + codecInfo.getName() + " (attempt " + (i + 1) + " of " + codecCandidates.size() + ")");
+                }
+                return;
+            } catch (IllegalStateException e) {
+                Log.w(TAG, "Video encoder: codec " + codecInfo.getName() + " failed to start (attempt " + (i + 1) + " of " + codecCandidates.size() + ")", e);
+                lastException = e;
+            }
+        }
+
+        throw new CodecUnavailableException("All video encoder codecs failed to start", lastException);
     }
 
     private static int getAndSelectVideoTrackIndex(@NonNull MediaExtractor extractor) {

lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/exceptions/CodecUnavailableException.kt

@@ -0,0 +1,14 @@
+/*
+ * Copyright 2026 Signal Messenger, LLC
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+package org.thoughtcrime.securesms.video.videoconverter.exceptions
+
+import java.io.IOException
+
+/**
+ * Thrown when all available codec candidates have been exhausted (either they all
+ * failed to configure/start, or they were all excluded due to mid-stream failures).
+ * This indicates a device limitation, not a bug in the transcoding code.
+ */
+open class CodecUnavailableException(message: String, cause: Throwable? = null) : IOException(message, cause)

lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/exceptions/HdrDecoderUnavailableException.kt

@@ -4,10 +4,8 @@
  */
 package org.thoughtcrime.securesms.video.videoconverter.exceptions
 
-import java.io.IOException
-
 /**
  * Thrown when no decoder on the device can properly decode HDR video content.
  * This is typically a device limitation, not a bug.
  */
-class HdrDecoderUnavailableException(message: String, cause: Throwable?) : IOException(message, cause)
+class HdrDecoderUnavailableException(message: String, cause: Throwable?) : CodecUnavailableException(message, cause)