mirror of
https://github.com/signalapp/Signal-Android.git
synced 2026-04-02 08:23:00 +01:00
Fix more HDR transcoding errors.
This commit is contained in:
Greyson Parrelli
committed by
Cody Henthorne
Cody Henthorne
parent
32dc36d937
commit
caf2e555dd
@@ -18,9 +18,12 @@ import org.junit.Test
|
||||
import org.junit.runner.RunWith
|
||||
import org.thoughtcrime.securesms.video.StreamingTranscoder
|
||||
import org.thoughtcrime.securesms.video.TranscodingPreset
|
||||
import org.thoughtcrime.securesms.video.videoconverter.exceptions.EncodingException
|
||||
import org.thoughtcrime.securesms.video.videoconverter.exceptions.HdrDecoderUnavailableException
|
||||
import org.thoughtcrime.securesms.video.videoconverter.mediadatasource.InputStreamMediaDataSource
|
||||
import java.io.File
|
||||
import java.io.FileInputStream
|
||||
import java.io.FileNotFoundException
|
||||
import java.io.InputStream
|
||||
|
||||
/**
|
||||
@@ -87,17 +90,37 @@ class VideoTranscodeInstrumentationTest {
|
||||
|
||||
val failures = mutableListOf<String>()
|
||||
|
||||
val hdrWarnings = mutableListOf<String>()
|
||||
|
||||
for (videoFileName in videoFiles) {
|
||||
Log.i(TAG, "Transcoding '$videoFileName' with preset ${preset.name}...")
|
||||
try {
|
||||
transcodeVideo(videoFileName, preset)
|
||||
Log.i(TAG, "Successfully transcoded '$videoFileName' with preset ${preset.name}")
|
||||
} catch (e: HdrDecoderUnavailableException) {
|
||||
Log.w(TAG, "No decoder available for HDR video '$videoFileName' with preset ${preset.name} (device limitation)", e)
|
||||
hdrWarnings.add("$videoFileName [${preset.name}]: ${e::class.simpleName}: ${e.message}")
|
||||
} catch (e: FileNotFoundException) {
|
||||
Log.w(TAG, "Skipping '$videoFileName' with preset ${preset.name}: encoder not available on this device", e)
|
||||
} catch (e: EncodingException) {
|
||||
val isHdrDeviceLimitation = e.isHdrInput && (!e.toneMapApplied || isMediaCodecRuntimeFailure(e))
|
||||
if (isHdrDeviceLimitation) {
|
||||
Log.w(TAG, "HDR video '$videoFileName' failed with preset ${preset.name} (device limitation, toneMap=${e.toneMapApplied}, decoder=${e.decoderName}, encoder=${e.encoderName})", e)
|
||||
hdrWarnings.add("$videoFileName [${preset.name}]: ${e::class.simpleName}: ${e.message}")
|
||||
} else {
|
||||
Log.e(TAG, "Failed to transcode '$videoFileName' with preset ${preset.name} (hdr=${e.isHdrInput}, toneMap=${e.toneMapApplied}, decoder=${e.decoderName}, encoder=${e.encoderName})", e)
|
||||
failures.add("$videoFileName: ${e::class.simpleName}: ${e.message}")
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
Log.e(TAG, "Failed to transcode '$videoFileName' with preset ${preset.name}", e)
|
||||
failures.add("$videoFileName: ${e::class.simpleName}: ${e.message}")
|
||||
}
|
||||
}
|
||||
|
||||
if (hdrWarnings.isNotEmpty()) {
|
||||
Log.w(TAG, "${hdrWarnings.size} HDR video(s) could not be transcoded (device limitation, not a bug):\n${hdrWarnings.joinToString("\n")}")
|
||||
}
|
||||
|
||||
if (failures.isNotEmpty()) {
|
||||
Assert.fail(
|
||||
"${failures.size}/${videoFiles.size} video(s) failed transcoding with ${preset.name}:\n" +
|
||||
@@ -106,6 +129,24 @@ class VideoTranscodeInstrumentationTest {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if the root cause of an exception is an [IllegalStateException] from
|
||||
* [android.media.MediaCodec] native methods. This indicates the hardware codec crashed
|
||||
* during processing, which for HDR content is a device limitation (not a Signal bug).
|
||||
*/
|
||||
private fun isMediaCodecRuntimeFailure(e: Exception): Boolean {
|
||||
var cause: Throwable? = e.cause
|
||||
while (cause != null) {
|
||||
if (cause is IllegalStateException &&
|
||||
cause.stackTrace.any { it.className == "android.media.MediaCodec" }
|
||||
) {
|
||||
return true
|
||||
}
|
||||
cause = cause.cause
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
private fun transcodeVideo(videoFileName: String, preset: TranscodingPreset) {
|
||||
val inputFile = createTempFile("input-", "-$videoFileName")
|
||||
val outputFile = createTempFile("output-${preset.name}-", "-$videoFileName")
|
||||
|
||||
@@ -20,6 +20,7 @@ package org.thoughtcrime.securesms.video.videoconverter;
|
||||
|
||||
import android.media.MediaCodecInfo;
|
||||
import android.media.MediaCodecList;
|
||||
import android.media.MediaExtractor;
|
||||
import android.media.MediaFormat;
|
||||
|
||||
import androidx.annotation.NonNull;
|
||||
@@ -33,6 +34,7 @@ import org.thoughtcrime.securesms.video.interfaces.MediaInput;
|
||||
import org.thoughtcrime.securesms.video.interfaces.Muxer;
|
||||
import org.thoughtcrime.securesms.video.videoconverter.exceptions.EncodingException;
|
||||
import org.thoughtcrime.securesms.video.videoconverter.muxer.StreamingMuxer;
|
||||
import org.thoughtcrime.securesms.video.videoconverter.utils.MediaCodecCompat;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileDescriptor;
|
||||
@@ -226,7 +228,33 @@ public final class MediaConverter {
|
||||
}
|
||||
}
|
||||
if (exception != null) {
|
||||
throw new EncodingException("Transcode failed", exception);
|
||||
EncodingException encodingException = new EncodingException("Transcode failed", exception);
|
||||
if (videoTrackConverter != null) {
|
||||
encodingException.isHdrInput = videoTrackConverter.isHdrInput();
|
||||
encodingException.toneMapApplied = videoTrackConverter.isToneMapApplied();
|
||||
encodingException.decoderName = videoTrackConverter.getDecoderName();
|
||||
encodingException.encoderName = videoTrackConverter.getEncoderName();
|
||||
} else {
|
||||
// VideoTrackConverter failed during creation. Try to determine HDR
|
||||
// status independently so the caller can classify the failure.
|
||||
try {
|
||||
final MediaExtractor extractor = mInput.createExtractor();
|
||||
try {
|
||||
for (int i = 0; i < extractor.getTrackCount(); i++) {
|
||||
final MediaFormat format = extractor.getTrackFormat(i);
|
||||
if (getMimeTypeFor(format).startsWith("video/")) {
|
||||
encodingException.isHdrInput = MediaCodecCompat.isHdrVideo(format);
|
||||
break;
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
extractor.release();
|
||||
}
|
||||
} catch (Exception e) {
|
||||
Log.w(TAG, "Could not determine HDR status for failed transcode", e);
|
||||
}
|
||||
}
|
||||
throw encodingException;
|
||||
}
|
||||
|
||||
return mdatContentLength;
|
||||
|
||||
@@ -13,6 +13,7 @@ import androidx.annotation.Nullable;
|
||||
import org.signal.core.util.logging.Log;
|
||||
import org.thoughtcrime.securesms.video.interfaces.MediaInput;
|
||||
import org.thoughtcrime.securesms.video.interfaces.Muxer;
|
||||
import org.thoughtcrime.securesms.video.videoconverter.exceptions.HdrDecoderUnavailableException;
|
||||
import org.thoughtcrime.securesms.video.videoconverter.utils.Extensions;
|
||||
import org.thoughtcrime.securesms.video.videoconverter.utils.MediaCodecCompat;
|
||||
import org.thoughtcrime.securesms.video.videoconverter.utils.Preconditions;
|
||||
@@ -38,6 +39,11 @@ final class VideoTrackConverter {
|
||||
|
||||
private static final float FRAME_RATE_TOLERANCE = 0.05f; // tolerance for transcoding VFR -> CFR
|
||||
|
||||
private boolean mIsHdrInput;
|
||||
private boolean mToneMapApplied;
|
||||
private String mDecoderName;
|
||||
private String mEncoderName;
|
||||
|
||||
private final long mTimeFrom;
|
||||
private final long mTimeTo;
|
||||
|
||||
@@ -421,6 +427,11 @@ final class VideoTrackConverter {
|
||||
Preconditions.checkState("decoded frame count should be less than extracted frame count", mVideoDecodedFrameCount <= mVideoExtractedFrameCount);
|
||||
}
|
||||
|
||||
boolean isHdrInput() { return mIsHdrInput; }
|
||||
boolean isToneMapApplied() { return mToneMapApplied; }
|
||||
String getDecoderName() { return mDecoderName; }
|
||||
String getEncoderName() { return mEncoderName; }
|
||||
|
||||
private static String createFragmentShader(
|
||||
final int srcWidth,
|
||||
final int srcHeight,
|
||||
@@ -473,31 +484,53 @@ final class VideoTrackConverter {
|
||||
MediaCodec createVideoDecoder(
|
||||
final @NonNull MediaFormat inputFormat,
|
||||
final @NonNull Surface surface) throws IOException {
|
||||
final boolean isHdr = MediaCodecCompat.isHdrVideo(inputFormat);
|
||||
final boolean isHdr = MediaCodecCompat.isHdrVideo(inputFormat);
|
||||
final boolean requestToneMapping = Build.VERSION.SDK_INT >= 31 && isHdr;
|
||||
final List<Pair<String, MediaFormat>> candidates = MediaCodecCompat.findDecoderCandidates(inputFormat);
|
||||
|
||||
mIsHdrInput = isHdr;
|
||||
Exception lastException = null;
|
||||
|
||||
for (int i = 0; i < candidates.size(); i++) {
|
||||
final Pair<String, MediaFormat> candidate = candidates.get(i);
|
||||
final String codecName = candidate.getFirst();
|
||||
final MediaFormat decoderFormat = candidate.getSecond();
|
||||
final String codecName = candidate.getFirst();
|
||||
final MediaFormat baseFormat = candidate.getSecond();
|
||||
MediaCodec decoder = null;
|
||||
|
||||
try {
|
||||
decoder = MediaCodec.createByCodecName(codecName);
|
||||
|
||||
// For HDR video, request SDR tone-mapping from the decoder (API 31+).
|
||||
if (Build.VERSION.SDK_INT >= 31 && isHdr) {
|
||||
decoderFormat.setInteger(MediaFormat.KEY_COLOR_TRANSFER_REQUEST, MediaFormat.COLOR_TRANSFER_SDR_VIDEO);
|
||||
// For HDR video on API 31+, try requesting SDR tone-mapping.
|
||||
// Some codecs reject this key, so we catch the error and retry without it.
|
||||
if (requestToneMapping) {
|
||||
try {
|
||||
final MediaFormat toneMapFormat = new MediaFormat(baseFormat);
|
||||
toneMapFormat.setInteger(MediaFormat.KEY_COLOR_TRANSFER_REQUEST, MediaFormat.COLOR_TRANSFER_SDR_VIDEO);
|
||||
decoder.configure(toneMapFormat, surface, null, 0);
|
||||
decoder.start();
|
||||
|
||||
mToneMapApplied = isToneMapEffective(decoder, codecName);
|
||||
mDecoderName = codecName;
|
||||
if (i > 0) {
|
||||
Log.w(TAG, "Video decoder: succeeded with fallback codec " + codecName + " (attempt " + (i + 1) + " of " + candidates.size() + ")");
|
||||
}
|
||||
return decoder;
|
||||
} catch (IllegalArgumentException | IllegalStateException e) {
|
||||
Log.w(TAG, "Video decoder: codec " + codecName + " rejected tone-mapping request, retrying without (attempt " + (i + 1) + " of " + candidates.size() + ")", e);
|
||||
decoder.release();
|
||||
decoder = MediaCodec.createByCodecName(codecName);
|
||||
}
|
||||
}
|
||||
decoder.configure(decoderFormat, surface, null, 0);
|
||||
|
||||
decoder.configure(baseFormat, surface, null, 0);
|
||||
decoder.start();
|
||||
|
||||
if (i > 0) {
|
||||
Log.w(TAG, "Video decoder: succeeded with fallback codec " + codecName + " (attempt " + (i + 1) + " of " + candidates.size() + ")");
|
||||
mDecoderName = codecName;
|
||||
if (i > 0 || requestToneMapping) {
|
||||
Log.w(TAG, "Video decoder: succeeded with codec " + codecName + (requestToneMapping ? " (no tone-mapping)" : "") + " (attempt " + (i + 1) + " of " + candidates.size() + ")");
|
||||
}
|
||||
return decoder;
|
||||
} catch (IllegalStateException e) {
|
||||
} catch (IllegalArgumentException | IllegalStateException e) {
|
||||
Log.w(TAG, "Video decoder: codec " + codecName + " failed (attempt " + (i + 1) + " of " + candidates.size() + ")", e);
|
||||
lastException = e;
|
||||
if (decoder != null) {
|
||||
@@ -509,6 +542,9 @@ final class VideoTrackConverter {
|
||||
}
|
||||
}
|
||||
|
||||
if (mIsHdrInput) {
|
||||
throw new HdrDecoderUnavailableException("All video decoder codecs failed for HDR video", lastException);
|
||||
}
|
||||
throw new IOException("All video decoder codecs failed", lastException);
|
||||
}
|
||||
|
||||
@@ -517,7 +553,7 @@ final class VideoTrackConverter {
|
||||
* {@link MediaCodec#createInputSurface()} (between configure and start) and then
|
||||
* {@link MediaCodec#start()} after the decoder has been created.
|
||||
*/
|
||||
private static @NonNull
|
||||
private @NonNull
|
||||
MediaCodec createVideoEncoder(
|
||||
final @NonNull List<MediaCodecInfo> codecCandidates,
|
||||
final @NonNull MediaFormat format) throws IOException {
|
||||
@@ -530,11 +566,12 @@ final class VideoTrackConverter {
|
||||
try {
|
||||
encoder = MediaCodec.createByCodecName(codecInfo.getName());
|
||||
encoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
|
||||
mEncoderName = codecInfo.getName();
|
||||
if (i > 0) {
|
||||
Log.w(TAG, "Video encoder: succeeded with fallback codec " + codecInfo.getName() + " (attempt " + (i + 1) + " of " + codecCandidates.size() + ")");
|
||||
}
|
||||
return encoder;
|
||||
} catch (IllegalStateException e) {
|
||||
} catch (IllegalArgumentException | IllegalStateException e) {
|
||||
Log.w(TAG, "Video encoder: codec " + codecInfo.getName() + " failed (attempt " + (i + 1) + " of " + codecCandidates.size() + ")", e);
|
||||
lastException = e;
|
||||
if (encoder != null) {
|
||||
@@ -563,5 +600,37 @@ final class VideoTrackConverter {
|
||||
return MediaConverter.getMimeTypeFor(format).startsWith("video/");
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks whether HDR-to-SDR tone-mapping is effective after the decoder has been configured
|
||||
* and started with {@link MediaFormat#KEY_COLOR_TRANSFER_REQUEST}. Some codecs (especially
|
||||
* software decoders and some hardware decoders) accept the tone-mapping key without error
|
||||
* but don't actually perform the conversion.
|
||||
*/
|
||||
private static boolean isToneMapEffective(final @NonNull MediaCodec decoder, final @NonNull String codecName) {
|
||||
// Software codecs never perform HDR→SDR tone-mapping.
|
||||
String lower = codecName.toLowerCase(java.util.Locale.ROOT);
|
||||
if (lower.startsWith("omx.google.") || lower.startsWith("c2.android.")) {
|
||||
Log.w(TAG, "Video decoder: software codec " + codecName + " cannot perform HDR tone-mapping");
|
||||
return false;
|
||||
}
|
||||
|
||||
// For hardware codecs, verify the output format. If the output transfer function
|
||||
// is still HDR (ST2084 or HLG), the decoder accepted the request but isn't honoring it.
|
||||
try {
|
||||
MediaFormat outputFormat = decoder.getOutputFormat();
|
||||
if (outputFormat.containsKey(MediaFormat.KEY_COLOR_TRANSFER)) {
|
||||
int transfer = outputFormat.getInteger(MediaFormat.KEY_COLOR_TRANSFER);
|
||||
if (transfer == MediaFormat.COLOR_TRANSFER_ST2084 || transfer == MediaFormat.COLOR_TRANSFER_HLG) {
|
||||
Log.w(TAG, "Video decoder: codec " + codecName + " accepted tone-mapping but output transfer is " + transfer + " (still HDR)");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
Log.w(TAG, "Video decoder: could not verify tone-mapping for codec " + codecName, e);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
@@ -5,6 +5,15 @@
|
||||
package org.thoughtcrime.securesms.video.videoconverter.exceptions
|
||||
|
||||
class EncodingException : Exception {
|
||||
/** Whether the input video was HDR content. */
|
||||
@JvmField var isHdrInput: Boolean = false
|
||||
/** Whether HDR-to-SDR tone-mapping was successfully applied to the decoder. */
|
||||
@JvmField var toneMapApplied: Boolean = false
|
||||
/** The name of the video decoder codec that was selected, or null if decoder creation failed. */
|
||||
@JvmField var decoderName: String? = null
|
||||
/** The name of the video encoder codec that was selected, or null if encoder creation failed. */
|
||||
@JvmField var encoderName: String? = null
|
||||
|
||||
constructor(message: String?) : super(message)
|
||||
constructor(message: String?, inner: Exception?) : super(message, inner)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
/*
|
||||
* Copyright 2026 Signal Messenger, LLC
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
package org.thoughtcrime.securesms.video.videoconverter.exceptions
|
||||
|
||||
import java.io.IOException
|
||||
|
||||
/**
|
||||
* Thrown when no decoder on the device can properly decode HDR video content.
|
||||
* This is typically a device limitation, not a bug.
|
||||
*/
|
||||
class HdrDecoderUnavailableException(message: String, cause: Throwable?) : IOException(message, cause)
|
||||
@@ -266,6 +266,26 @@ object MediaCodecCompat {
|
||||
return true
|
||||
}
|
||||
|
||||
// On older APIs, the extractor may not populate KEY_COLOR_TRANSFER or HDR metadata keys.
|
||||
// Check the HEVC profile as a fallback: Main 10 and its HDR variants indicate 10-bit
|
||||
// content that typically requires HDR-capable decoders.
|
||||
val mime = try { format.getString(MediaFormat.KEY_MIME) } catch (_: Exception) { null }
|
||||
if (mime.equals("video/hevc", ignoreCase = true)) {
|
||||
try {
|
||||
val profile = format.getInteger(MediaFormat.KEY_PROFILE)
|
||||
if (profile == CodecProfileLevel.HEVCProfileMain10 ||
|
||||
profile == CodecProfileLevel.HEVCProfileMain10HDR10 ||
|
||||
profile == CodecProfileLevel.HEVCProfileMain10HDR10Plus
|
||||
) {
|
||||
return true
|
||||
}
|
||||
} catch (_: NullPointerException) {
|
||||
// key doesn't exist
|
||||
} catch (_: ClassCastException) {
|
||||
// key exists but wrong type
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user