Signal/Android
1
0
Fork 0
mirror of https://github.com/signalapp/Signal-Android.git synced 2026-04-02 00:17:41 +01:00
Code Issues Packages Projects Releases Wiki Activity

Improve video encoder/decoder fallback logic.

Browse Source
This commit is contained in:
Greyson Parrelli
2026-02-20 14:07:38 -05:00
committed by Cody Henthorne
parent 093a79045d
commit 7b31383b88
5 changed files with 153 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
demo/video/src/androidTest/java/org/thoughtcrime/video/app/transcode/VideoTranscodeInstrumentationTest.kt
View File

@@ -19,13 +19,13 @@ import org.junit.runner.RunWith
import org.thoughtcrime.securesms.video.StreamingTranscoder
import org.thoughtcrime.securesms.video.TranscodingPreset
import org.thoughtcrime.securesms.video.exceptions.VideoSourceException
import org.thoughtcrime.securesms.video.videoconverter.exceptions.CodecUnavailableException
import org.thoughtcrime.securesms.video.videoconverter.exceptions.EncodingException
import org.thoughtcrime.securesms.video.videoconverter.exceptions.HdrDecoderUnavailableException
import org.thoughtcrime.securesms.video.videoconverter.mediadatasource.InputStreamMediaDataSource
import java.io.File
import java.io.FileInputStream
import java.io.FileNotFoundException
import java.io.IOException
import java.io.InputStream
/**
@@ -105,10 +105,9 @@ class VideoTranscodeInstrumentationTest {
} catch (e: FileNotFoundException) {
Log.w(TAG, "Skipping '$videoFileName' with preset ${preset.name}: encoder not available on this device", e)
} catch (e: EncodingException) {
val isHdrDeviceLimitation = e.isHdrInput && (!e.toneMapApplied || isMediaCodecRuntimeFailure(e))
val isCodecRuntimeFailure = isMediaCodecRuntimeFailure(e)
if (isHdrDeviceLimitation || isCodecRuntimeFailure) {
Log.w(TAG, "Video '$videoFileName' failed with preset ${preset.name} (device limitation, hdr=${e.isHdrInput}, toneMap=${e.toneMapApplied}, decoder=${e.decoderName}, encoder=${e.encoderName})", e)
val isHdrDeviceLimitation = e.isHdrInput && !e.toneMapApplied
if (isHdrDeviceLimitation) {
Log.w(TAG, "Video '$videoFileName' failed with preset ${preset.name} (HDR device limitation, toneMap=${e.toneMapApplied}, decoder=${e.decoderName}, encoder=${e.encoderName})", e)
deviceWarnings.add("$videoFileName [${preset.name}]: ${e::class.simpleName}: ${e.message}")
} else {
Log.e(TAG, "Failed to transcode '$videoFileName' with preset ${preset.name} (hdr=${e.isHdrInput}, toneMap=${e.toneMapApplied}, decoder=${e.decoderName}, encoder=${e.encoderName})", e)
@@ -117,8 +116,8 @@ class VideoTranscodeInstrumentationTest {
} catch (e: VideoSourceException) {
Log.w(TAG, "Device cannot read video source '$videoFileName' with preset ${preset.name} (device limitation)", e)
deviceWarnings.add("$videoFileName [${preset.name}]: ${e::class.simpleName}: ${e.message}")
} catch (e: IOException) {
Log.w(TAG, "Device cannot decode/encode '$videoFileName' with preset ${preset.name} (device limitation)", e)
} catch (e: CodecUnavailableException) {
Log.w(TAG, "All codecs exhausted for '$videoFileName' with preset ${preset.name} (device limitation)", e)
deviceWarnings.add("$videoFileName [${preset.name}]: ${e::class.simpleName}: ${e.message}")
} catch (e: Exception) {
Log.e(TAG, "Failed to transcode '$videoFileName' with preset ${preset.name}", e)
@@ -138,29 +137,6 @@ class VideoTranscodeInstrumentationTest {
}
}
/**
* Checks if the root cause of an exception indicates a device/content limitation
* rather than a bug in the transcoding code. This covers:
* - [IllegalStateException] from [android.media.MediaCodec] native methods (hardware codec crash)
* - Frame count mismatches from [VideoTrackConverter.verifyEndState] (unusual video formats
* like spatial video that some decoders handle differently)
*/
private fun isMediaCodecRuntimeFailure(e: Exception): Boolean {
var cause: Throwable? = e.cause
while (cause != null) {
if (cause is IllegalStateException) {
if (cause.stackTrace.any { it.className == "android.media.MediaCodec" }) {
return true
}
if (cause.message?.contains("frame counts should match") == true) {
return true
}
}
cause = cause.cause
}
return false
}
private fun transcodeVideo(videoFileName: String, preset: TranscodingPreset) {
val inputFile = createTempFile("input-", "-$videoFileName")
val outputFile = createTempFile("output-${preset.name}-", "-$videoFileName")

52
lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/MediaConverter.java
View File

@@ -149,6 +149,56 @@ public final class MediaConverter {
*/
@WorkerThread
public long convert() throws EncodingException, IOException {
final Set<String> excludedDecoders = new HashSet<>();
while (true) {
mCancelled = false;
try {
return doConvert(excludedDecoders);
} catch (EncodingException e) {
if (e.decoderName != null
&& isRetryableMidStreamFailure(e)
&& excludedDecoders.add(e.decoderName)) {
Log.w(TAG, "Mid-stream codec failure with decoder " + e.decoderName
+ ", retrying with it excluded (" + excludedDecoders.size()
+ " decoder(s) excluded)");
continue;
}
throw e;
}
}
}
/**
* Checks whether the given {@link EncodingException} represents a mid-stream codec failure
* that may succeed with a different decoder. This covers:
* <ul>
* <li>{@link IllegalStateException} from {@link android.media.MediaCodec} native methods
* (hardware codec crash during encoding/decoding)</li>
* <li>Frame count mismatches from {@link VideoTrackConverter#verifyEndState()} (unusual
* video formats like spatial video that some decoders handle differently)</li>
* </ul>
*/
private static boolean isRetryableMidStreamFailure(final @NonNull EncodingException e) {
Throwable cause = e.getCause();
while (cause != null) {
if (cause instanceof IllegalStateException) {
for (StackTraceElement frame : cause.getStackTrace()) {
if ("android.media.MediaCodec".equals(frame.getClassName())) {
return true;
}
}
String message = cause.getMessage();
if (message != null && message.contains("frame counts should match")) {
return true;
}
}
cause = cause.getCause();
}
return false;
}
private long doConvert(final @NonNull Set<String> excludedDecoders) throws EncodingException, IOException {
// Exception that may be thrown during release.
Exception exception = null;
Muxer muxer = null;
@@ -162,7 +212,7 @@ public final class MediaConverter {
try {
muxer = mOutput.createMuxer();
videoTrackConverter = VideoTrackConverter.create(mInput, mTimeFrom, mTimeTo, mVideoResolution, mVideoBitrate, mVideoCodec);
videoTrackConverter = VideoTrackConverter.create(mInput, mTimeFrom, mTimeTo, mVideoResolution, mVideoBitrate, mVideoCodec, excludedDecoders);
audioTrackConverter = AudioTrackConverter.create(mInput, mTimeFrom, mTimeTo, mAudioBitrate, mAllowAudioRemux && muxer.supportsAudioRemux());
if (videoTrackConverter == null && audioTrackConverter == null) {

95
lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/VideoTrackConverter.java
View File

@@ -13,6 +13,7 @@ import androidx.annotation.Nullable;
import org.signal.core.util.logging.Log;
import org.thoughtcrime.securesms.video.interfaces.MediaInput;
import org.thoughtcrime.securesms.video.interfaces.Muxer;
import org.thoughtcrime.securesms.video.videoconverter.exceptions.CodecUnavailableException;
import org.thoughtcrime.securesms.video.videoconverter.exceptions.HdrDecoderUnavailableException;
import org.thoughtcrime.securesms.video.videoconverter.utils.Extensions;
import org.thoughtcrime.securesms.video.videoconverter.utils.MediaCodecCompat;
@@ -21,7 +22,9 @@ import org.thoughtcrime.securesms.video.videoconverter.utils.Preconditions;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import kotlin.Pair;
final class VideoTrackConverter {
@@ -51,9 +54,9 @@ final class VideoTrackConverter {
private final MediaExtractor mVideoExtractor;
private final MediaCodec mVideoDecoder;
private final MediaCodec mVideoEncoder;
private MediaCodec mVideoEncoder;
private final InputSurface mInputSurface;
private InputSurface mInputSurface;
private final OutputSurface mOutputSurface;
private final ByteBuffer[] mVideoDecoderInputBuffers;
@@ -83,7 +86,8 @@ final class VideoTrackConverter {
final long timeTo,
final int videoResolution,
final int videoBitrate,
final @NonNull String videoCodec) throws IOException, TranscodingException {
final @NonNull String videoCodec,
final @NonNull Set<String> excludedDecoders) throws IOException, TranscodingException {
final MediaExtractor videoExtractor = input.createExtractor();
final int videoInputTrack = getAndSelectVideoTrackIndex(videoExtractor);
@@ -91,7 +95,7 @@ final class VideoTrackConverter {
videoExtractor.release();
return null;
}
return new VideoTrackConverter(videoExtractor, videoInputTrack, timeFrom, timeTo, videoResolution, videoBitrate, videoCodec);
return new VideoTrackConverter(videoExtractor, videoInputTrack, timeFrom, timeTo, videoResolution, videoBitrate, videoCodec, excludedDecoders);
}
@@ -102,7 +106,8 @@ final class VideoTrackConverter {
final long timeTo,
final int videoResolution,
final int videoBitrate,
final @NonNull String videoCodec) throws IOException, TranscodingException {
final @NonNull String videoCodec,
final @NonNull Set<String> excludedDecoders) throws IOException, TranscodingException {
mTimeFrom = timeFrom;
mTimeTo = timeTo;
@@ -167,16 +172,19 @@ final class VideoTrackConverter {
inputVideoFormat.getInteger(MediaFormat.KEY_WIDTH), inputVideoFormat.getInteger(MediaFormat.KEY_HEIGHT),
outputWidth, outputHeight);
// Configure the encoder but do NOT start it yet. The encoder's start() is
// deferred until after the decoder is created, so that the decoder gets first
// access to hardware codec resources on memory-constrained devices.
// Create encoder, decoder, and surfaces. The encoder's start() is deferred
// until after the decoder is created, so that the decoder gets first access to
// hardware codec resources on memory-constrained devices. If start() fails
// (e.g. NO_MEMORY on a resource-constrained device), we try the next encoder
// candidate while keeping the same decoder and OutputSurface.
mVideoEncoder = createVideoEncoder(videoCodecCandidates, outputVideoFormat);
mInputSurface = new InputSurface(mVideoEncoder.createInputSurface());
mInputSurface.makeCurrent();
mOutputSurface = new OutputSurface();
mOutputSurface.changeFragmentShader(fragmentShader);
mVideoDecoder = createVideoDecoder(inputVideoFormat, mOutputSurface.getSurface());
mVideoEncoder.start();
mVideoDecoder = createVideoDecoder(inputVideoFormat, mOutputSurface.getSurface(), excludedDecoders);
startEncoderWithFallback(videoCodecCandidates, outputVideoFormat);
mVideoDecoderInputBuffers = mVideoDecoder.getInputBuffers();
mVideoEncoderOutputBuffers = mVideoEncoder.getOutputBuffers();
mVideoDecoderOutputBufferInfo = new MediaCodec.BufferInfo();
@@ -483,10 +491,17 @@ final class VideoTrackConverter {
private @NonNull
MediaCodec createVideoDecoder(
final @NonNull MediaFormat inputFormat,
final @NonNull Surface surface) throws IOException {
final @NonNull Surface surface,
final @NonNull Set<String> excludedDecoders) throws IOException {
final boolean isHdr = MediaCodecCompat.isHdrVideo(inputFormat);
final boolean requestToneMapping = Build.VERSION.SDK_INT >= 31 && isHdr;
final List<Pair<String, MediaFormat>> candidates = MediaCodecCompat.findDecoderCandidates(inputFormat);
final List<Pair<String, MediaFormat>> allCandidates = MediaCodecCompat.findDecoderCandidates(inputFormat);
final List<Pair<String, MediaFormat>> candidates = new ArrayList<>();
for (Pair<String, MediaFormat> c : allCandidates) {
if (!excludedDecoders.contains(c.getFirst())) {
candidates.add(c);
}
}
mIsHdrInput = isHdr;
Exception lastException = null;
@@ -545,7 +560,7 @@ final class VideoTrackConverter {
if (mIsHdrInput) {
throw new HdrDecoderUnavailableException("All video decoder codecs failed for HDR video", lastException);
}
throw new IOException("All video decoder codecs failed", lastException);
throw new CodecUnavailableException("All video decoder codecs failed", lastException);
}
/**
@@ -580,7 +595,59 @@ final class VideoTrackConverter {
}
}
throw new IOException("All video encoder codecs failed", lastException);
throw new CodecUnavailableException("All video encoder codecs failed", lastException);
}
/**
* Attempts to start the current encoder ({@link #mVideoEncoder}). If start() fails,
* iterates through the remaining encoder candidates from {@code codecCandidates},
* replacing the encoder and its {@link InputSurface} on each attempt. The decoder
* and {@link OutputSurface} are independent of the encoder and remain unchanged.
*/
private void startEncoderWithFallback(
final @NonNull List<MediaCodecInfo> codecCandidates,
final @NonNull MediaFormat format) throws IOException {
Exception lastException = null;
for (int i = 0; i < codecCandidates.size(); i++) {
final MediaCodecInfo codecInfo = codecCandidates.get(i);
if (i > 0) {
// Replace the encoder with the next candidate.
mVideoEncoder.release();
mInputSurface.release();
try {
mVideoEncoder = MediaCodec.createByCodecName(codecInfo.getName());
mVideoEncoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
mInputSurface = new InputSurface(mVideoEncoder.createInputSurface());
mInputSurface.makeCurrent();
mEncoderName = codecInfo.getName();
} catch (IllegalArgumentException | IllegalStateException | TranscodingException e) {
Log.w(TAG, "Video encoder: codec " + codecInfo.getName() + " failed to configure (attempt " + (i + 1) + " of " + codecCandidates.size() + ")", e);
lastException = e;
continue;
}
} else if (!codecInfo.getName().equals(mEncoderName)) {
// First iteration but createVideoEncoder selected a different codec
// (i.e. the first candidate failed to configure). Skip until we reach
// the one that was actually configured.
continue;
}
try {
mVideoEncoder.start();
if (i > 0) {
Log.w(TAG, "Video encoder: succeeded with fallback codec " + codecInfo.getName() + " (attempt " + (i + 1) + " of " + codecCandidates.size() + ")");
}
return;
} catch (IllegalStateException e) {
Log.w(TAG, "Video encoder: codec " + codecInfo.getName() + " failed to start (attempt " + (i + 1) + " of " + codecCandidates.size() + ")", e);
lastException = e;
}
}
throw new CodecUnavailableException("All video encoder codecs failed to start", lastException);
}
private static int getAndSelectVideoTrackIndex(@NonNull MediaExtractor extractor) {

14
lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/exceptions/CodecUnavailableException.kt Normal file
View File

@@ -0,0 +1,14 @@
/*
* Copyright 2026 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.thoughtcrime.securesms.video.videoconverter.exceptions
import java.io.IOException
/**
* Thrown when all available codec candidates have been exhausted (either they all
* failed to configure/start, or they were all excluded due to mid-stream failures).
* This indicates a device limitation, not a bug in the transcoding code.
*/
open class CodecUnavailableException(message: String, cause: Throwable? = null) : IOException(message, cause)

4
lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/exceptions/HdrDecoderUnavailableException.kt
View File

@@ -4,10 +4,8 @@
*/
package org.thoughtcrime.securesms.video.videoconverter.exceptions
import java.io.IOException
/**
* Thrown when no decoder on the device can properly decode HDR video content.
* This is typically a device limitation, not a bug.
*/
class HdrDecoderUnavailableException(message: String, cause: Throwable?) : IOException(message, cause)
class HdrDecoderUnavailableException(message: String, cause: Throwable?) : CodecUnavailableException(message, cause)