diff --git a/demo/video/src/androidTest/java/org/thoughtcrime/video/app/transcode/VideoTranscodeInstrumentationTest.kt b/demo/video/src/androidTest/java/org/thoughtcrime/video/app/transcode/VideoTranscodeInstrumentationTest.kt index 1d89deb616..b104b695ee 100644 --- a/demo/video/src/androidTest/java/org/thoughtcrime/video/app/transcode/VideoTranscodeInstrumentationTest.kt +++ b/demo/video/src/androidTest/java/org/thoughtcrime/video/app/transcode/VideoTranscodeInstrumentationTest.kt @@ -19,13 +19,13 @@ import org.junit.runner.RunWith import org.thoughtcrime.securesms.video.StreamingTranscoder import org.thoughtcrime.securesms.video.TranscodingPreset import org.thoughtcrime.securesms.video.exceptions.VideoSourceException +import org.thoughtcrime.securesms.video.videoconverter.exceptions.CodecUnavailableException import org.thoughtcrime.securesms.video.videoconverter.exceptions.EncodingException import org.thoughtcrime.securesms.video.videoconverter.exceptions.HdrDecoderUnavailableException import org.thoughtcrime.securesms.video.videoconverter.mediadatasource.InputStreamMediaDataSource import java.io.File import java.io.FileInputStream import java.io.FileNotFoundException -import java.io.IOException import java.io.InputStream /** @@ -105,10 +105,9 @@ class VideoTranscodeInstrumentationTest { } catch (e: FileNotFoundException) { Log.w(TAG, "Skipping '$videoFileName' with preset ${preset.name}: encoder not available on this device", e) } catch (e: EncodingException) { - val isHdrDeviceLimitation = e.isHdrInput && (!e.toneMapApplied || isMediaCodecRuntimeFailure(e)) - val isCodecRuntimeFailure = isMediaCodecRuntimeFailure(e) - if (isHdrDeviceLimitation || isCodecRuntimeFailure) { - Log.w(TAG, "Video '$videoFileName' failed with preset ${preset.name} (device limitation, hdr=${e.isHdrInput}, toneMap=${e.toneMapApplied}, decoder=${e.decoderName}, encoder=${e.encoderName})", e) + val isHdrDeviceLimitation = e.isHdrInput && !e.toneMapApplied + if (isHdrDeviceLimitation) { + Log.w(TAG, "Video '$videoFileName' failed with preset ${preset.name} (HDR device limitation, toneMap=${e.toneMapApplied}, decoder=${e.decoderName}, encoder=${e.encoderName})", e) deviceWarnings.add("$videoFileName [${preset.name}]: ${e::class.simpleName}: ${e.message}") } else { Log.e(TAG, "Failed to transcode '$videoFileName' with preset ${preset.name} (hdr=${e.isHdrInput}, toneMap=${e.toneMapApplied}, decoder=${e.decoderName}, encoder=${e.encoderName})", e) @@ -117,8 +116,8 @@ class VideoTranscodeInstrumentationTest { } catch (e: VideoSourceException) { Log.w(TAG, "Device cannot read video source '$videoFileName' with preset ${preset.name} (device limitation)", e) deviceWarnings.add("$videoFileName [${preset.name}]: ${e::class.simpleName}: ${e.message}") - } catch (e: IOException) { - Log.w(TAG, "Device cannot decode/encode '$videoFileName' with preset ${preset.name} (device limitation)", e) + } catch (e: CodecUnavailableException) { + Log.w(TAG, "All codecs exhausted for '$videoFileName' with preset ${preset.name} (device limitation)", e) deviceWarnings.add("$videoFileName [${preset.name}]: ${e::class.simpleName}: ${e.message}") } catch (e: Exception) { Log.e(TAG, "Failed to transcode '$videoFileName' with preset ${preset.name}", e) @@ -138,29 +137,6 @@ class VideoTranscodeInstrumentationTest { } } - /** - * Checks if the root cause of an exception indicates a device/content limitation - * rather than a bug in the transcoding code. This covers: - * - [IllegalStateException] from [android.media.MediaCodec] native methods (hardware codec crash) - * - Frame count mismatches from [VideoTrackConverter.verifyEndState] (unusual video formats - * like spatial video that some decoders handle differently) - */ - private fun isMediaCodecRuntimeFailure(e: Exception): Boolean { - var cause: Throwable? = e.cause - while (cause != null) { - if (cause is IllegalStateException) { - if (cause.stackTrace.any { it.className == "android.media.MediaCodec" }) { - return true - } - if (cause.message?.contains("frame counts should match") == true) { - return true - } - } - cause = cause.cause - } - return false - } - private fun transcodeVideo(videoFileName: String, preset: TranscodingPreset) { val inputFile = createTempFile("input-", "-$videoFileName") val outputFile = createTempFile("output-${preset.name}-", "-$videoFileName") diff --git a/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/MediaConverter.java b/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/MediaConverter.java index 719cdf4744..b7c7461dbd 100644 --- a/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/MediaConverter.java +++ b/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/MediaConverter.java @@ -149,6 +149,56 @@ public final class MediaConverter { */ @WorkerThread public long convert() throws EncodingException, IOException { + final Set excludedDecoders = new HashSet<>(); + + while (true) { + mCancelled = false; + try { + return doConvert(excludedDecoders); + } catch (EncodingException e) { + if (e.decoderName != null + && isRetryableMidStreamFailure(e) + && excludedDecoders.add(e.decoderName)) { + Log.w(TAG, "Mid-stream codec failure with decoder " + e.decoderName + + ", retrying with it excluded (" + excludedDecoders.size() + + " decoder(s) excluded)"); + continue; + } + throw e; + } + } + } + + /** + * Checks whether the given {@link EncodingException} represents a mid-stream codec failure + * that may succeed with a different decoder. This covers: + * + */ + private static boolean isRetryableMidStreamFailure(final @NonNull EncodingException e) { + Throwable cause = e.getCause(); + while (cause != null) { + if (cause instanceof IllegalStateException) { + for (StackTraceElement frame : cause.getStackTrace()) { + if ("android.media.MediaCodec".equals(frame.getClassName())) { + return true; + } + } + String message = cause.getMessage(); + if (message != null && message.contains("frame counts should match")) { + return true; + } + } + cause = cause.getCause(); + } + return false; + } + + private long doConvert(final @NonNull Set excludedDecoders) throws EncodingException, IOException { // Exception that may be thrown during release. Exception exception = null; Muxer muxer = null; @@ -162,7 +212,7 @@ public final class MediaConverter { try { muxer = mOutput.createMuxer(); - videoTrackConverter = VideoTrackConverter.create(mInput, mTimeFrom, mTimeTo, mVideoResolution, mVideoBitrate, mVideoCodec); + videoTrackConverter = VideoTrackConverter.create(mInput, mTimeFrom, mTimeTo, mVideoResolution, mVideoBitrate, mVideoCodec, excludedDecoders); audioTrackConverter = AudioTrackConverter.create(mInput, mTimeFrom, mTimeTo, mAudioBitrate, mAllowAudioRemux && muxer.supportsAudioRemux()); if (videoTrackConverter == null && audioTrackConverter == null) { diff --git a/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/VideoTrackConverter.java b/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/VideoTrackConverter.java index 69bbc0fa42..18624c97e6 100644 --- a/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/VideoTrackConverter.java +++ b/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/VideoTrackConverter.java @@ -13,6 +13,7 @@ import androidx.annotation.Nullable; import org.signal.core.util.logging.Log; import org.thoughtcrime.securesms.video.interfaces.MediaInput; import org.thoughtcrime.securesms.video.interfaces.Muxer; +import org.thoughtcrime.securesms.video.videoconverter.exceptions.CodecUnavailableException; import org.thoughtcrime.securesms.video.videoconverter.exceptions.HdrDecoderUnavailableException; import org.thoughtcrime.securesms.video.videoconverter.utils.Extensions; import org.thoughtcrime.securesms.video.videoconverter.utils.MediaCodecCompat; @@ -21,7 +22,9 @@ import org.thoughtcrime.securesms.video.videoconverter.utils.Preconditions; import java.io.FileNotFoundException; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.ArrayList; import java.util.List; +import java.util.Set; import kotlin.Pair; final class VideoTrackConverter { @@ -51,9 +54,9 @@ final class VideoTrackConverter { private final MediaExtractor mVideoExtractor; private final MediaCodec mVideoDecoder; - private final MediaCodec mVideoEncoder; + private MediaCodec mVideoEncoder; - private final InputSurface mInputSurface; + private InputSurface mInputSurface; private final OutputSurface mOutputSurface; private final ByteBuffer[] mVideoDecoderInputBuffers; @@ -83,7 +86,8 @@ final class VideoTrackConverter { final long timeTo, final int videoResolution, final int videoBitrate, - final @NonNull String videoCodec) throws IOException, TranscodingException { + final @NonNull String videoCodec, + final @NonNull Set excludedDecoders) throws IOException, TranscodingException { final MediaExtractor videoExtractor = input.createExtractor(); final int videoInputTrack = getAndSelectVideoTrackIndex(videoExtractor); @@ -91,7 +95,7 @@ final class VideoTrackConverter { videoExtractor.release(); return null; } - return new VideoTrackConverter(videoExtractor, videoInputTrack, timeFrom, timeTo, videoResolution, videoBitrate, videoCodec); + return new VideoTrackConverter(videoExtractor, videoInputTrack, timeFrom, timeTo, videoResolution, videoBitrate, videoCodec, excludedDecoders); } @@ -102,7 +106,8 @@ final class VideoTrackConverter { final long timeTo, final int videoResolution, final int videoBitrate, - final @NonNull String videoCodec) throws IOException, TranscodingException { + final @NonNull String videoCodec, + final @NonNull Set excludedDecoders) throws IOException, TranscodingException { mTimeFrom = timeFrom; mTimeTo = timeTo; @@ -167,16 +172,19 @@ final class VideoTrackConverter { inputVideoFormat.getInteger(MediaFormat.KEY_WIDTH), inputVideoFormat.getInteger(MediaFormat.KEY_HEIGHT), outputWidth, outputHeight); - // Configure the encoder but do NOT start it yet. The encoder's start() is - // deferred until after the decoder is created, so that the decoder gets first - // access to hardware codec resources on memory-constrained devices. + // Create encoder, decoder, and surfaces. The encoder's start() is deferred + // until after the decoder is created, so that the decoder gets first access to + // hardware codec resources on memory-constrained devices. If start() fails + // (e.g. NO_MEMORY on a resource-constrained device), we try the next encoder + // candidate while keeping the same decoder and OutputSurface. mVideoEncoder = createVideoEncoder(videoCodecCandidates, outputVideoFormat); mInputSurface = new InputSurface(mVideoEncoder.createInputSurface()); mInputSurface.makeCurrent(); mOutputSurface = new OutputSurface(); mOutputSurface.changeFragmentShader(fragmentShader); - mVideoDecoder = createVideoDecoder(inputVideoFormat, mOutputSurface.getSurface()); - mVideoEncoder.start(); + mVideoDecoder = createVideoDecoder(inputVideoFormat, mOutputSurface.getSurface(), excludedDecoders); + startEncoderWithFallback(videoCodecCandidates, outputVideoFormat); + mVideoDecoderInputBuffers = mVideoDecoder.getInputBuffers(); mVideoEncoderOutputBuffers = mVideoEncoder.getOutputBuffers(); mVideoDecoderOutputBufferInfo = new MediaCodec.BufferInfo(); @@ -483,10 +491,17 @@ final class VideoTrackConverter { private @NonNull MediaCodec createVideoDecoder( final @NonNull MediaFormat inputFormat, - final @NonNull Surface surface) throws IOException { + final @NonNull Surface surface, + final @NonNull Set excludedDecoders) throws IOException { final boolean isHdr = MediaCodecCompat.isHdrVideo(inputFormat); final boolean requestToneMapping = Build.VERSION.SDK_INT >= 31 && isHdr; - final List> candidates = MediaCodecCompat.findDecoderCandidates(inputFormat); + final List> allCandidates = MediaCodecCompat.findDecoderCandidates(inputFormat); + final List> candidates = new ArrayList<>(); + for (Pair c : allCandidates) { + if (!excludedDecoders.contains(c.getFirst())) { + candidates.add(c); + } + } mIsHdrInput = isHdr; Exception lastException = null; @@ -545,7 +560,7 @@ final class VideoTrackConverter { if (mIsHdrInput) { throw new HdrDecoderUnavailableException("All video decoder codecs failed for HDR video", lastException); } - throw new IOException("All video decoder codecs failed", lastException); + throw new CodecUnavailableException("All video decoder codecs failed", lastException); } /** @@ -580,7 +595,59 @@ final class VideoTrackConverter { } } - throw new IOException("All video encoder codecs failed", lastException); + throw new CodecUnavailableException("All video encoder codecs failed", lastException); + } + + /** + * Attempts to start the current encoder ({@link #mVideoEncoder}). If start() fails, + * iterates through the remaining encoder candidates from {@code codecCandidates}, + * replacing the encoder and its {@link InputSurface} on each attempt. The decoder + * and {@link OutputSurface} are independent of the encoder and remain unchanged. + */ + private void startEncoderWithFallback( + final @NonNull List codecCandidates, + final @NonNull MediaFormat format) throws IOException { + Exception lastException = null; + + for (int i = 0; i < codecCandidates.size(); i++) { + final MediaCodecInfo codecInfo = codecCandidates.get(i); + + if (i > 0) { + // Replace the encoder with the next candidate. + mVideoEncoder.release(); + mInputSurface.release(); + + try { + mVideoEncoder = MediaCodec.createByCodecName(codecInfo.getName()); + mVideoEncoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); + mInputSurface = new InputSurface(mVideoEncoder.createInputSurface()); + mInputSurface.makeCurrent(); + mEncoderName = codecInfo.getName(); + } catch (IllegalArgumentException | IllegalStateException | TranscodingException e) { + Log.w(TAG, "Video encoder: codec " + codecInfo.getName() + " failed to configure (attempt " + (i + 1) + " of " + codecCandidates.size() + ")", e); + lastException = e; + continue; + } + } else if (!codecInfo.getName().equals(mEncoderName)) { + // First iteration but createVideoEncoder selected a different codec + // (i.e. the first candidate failed to configure). Skip until we reach + // the one that was actually configured. + continue; + } + + try { + mVideoEncoder.start(); + if (i > 0) { + Log.w(TAG, "Video encoder: succeeded with fallback codec " + codecInfo.getName() + " (attempt " + (i + 1) + " of " + codecCandidates.size() + ")"); + } + return; + } catch (IllegalStateException e) { + Log.w(TAG, "Video encoder: codec " + codecInfo.getName() + " failed to start (attempt " + (i + 1) + " of " + codecCandidates.size() + ")", e); + lastException = e; + } + } + + throw new CodecUnavailableException("All video encoder codecs failed to start", lastException); } private static int getAndSelectVideoTrackIndex(@NonNull MediaExtractor extractor) { diff --git a/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/exceptions/CodecUnavailableException.kt b/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/exceptions/CodecUnavailableException.kt new file mode 100644 index 0000000000..a0a16bfa15 --- /dev/null +++ b/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/exceptions/CodecUnavailableException.kt @@ -0,0 +1,14 @@ +/* + * Copyright 2026 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ +package org.thoughtcrime.securesms.video.videoconverter.exceptions + +import java.io.IOException + +/** + * Thrown when all available codec candidates have been exhausted (either they all + * failed to configure/start, or they were all excluded due to mid-stream failures). + * This indicates a device limitation, not a bug in the transcoding code. + */ +open class CodecUnavailableException(message: String, cause: Throwable? = null) : IOException(message, cause) diff --git a/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/exceptions/HdrDecoderUnavailableException.kt b/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/exceptions/HdrDecoderUnavailableException.kt index 585edb5aa2..5857f72587 100644 --- a/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/exceptions/HdrDecoderUnavailableException.kt +++ b/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/exceptions/HdrDecoderUnavailableException.kt @@ -4,10 +4,8 @@ */ package org.thoughtcrime.securesms.video.videoconverter.exceptions -import java.io.IOException - /** * Thrown when no decoder on the device can properly decode HDR video content. * This is typically a device limitation, not a bug. */ -class HdrDecoderUnavailableException(message: String, cause: Throwable?) : IOException(message, cause) +class HdrDecoderUnavailableException(message: String, cause: Throwable?) : CodecUnavailableException(message, cause)