diff --git a/demo/video/src/androidTest/java/org/thoughtcrime/video/app/transcode/VideoTranscodeInstrumentationTest.kt b/demo/video/src/androidTest/java/org/thoughtcrime/video/app/transcode/VideoTranscodeInstrumentationTest.kt index be6b31c413..b7fe217534 100644 --- a/demo/video/src/androidTest/java/org/thoughtcrime/video/app/transcode/VideoTranscodeInstrumentationTest.kt +++ b/demo/video/src/androidTest/java/org/thoughtcrime/video/app/transcode/VideoTranscodeInstrumentationTest.kt @@ -18,9 +18,12 @@ import org.junit.Test import org.junit.runner.RunWith import org.thoughtcrime.securesms.video.StreamingTranscoder import org.thoughtcrime.securesms.video.TranscodingPreset +import org.thoughtcrime.securesms.video.videoconverter.exceptions.EncodingException +import org.thoughtcrime.securesms.video.videoconverter.exceptions.HdrDecoderUnavailableException import org.thoughtcrime.securesms.video.videoconverter.mediadatasource.InputStreamMediaDataSource import java.io.File import java.io.FileInputStream +import java.io.FileNotFoundException import java.io.InputStream /** @@ -87,17 +90,37 @@ class VideoTranscodeInstrumentationTest { val failures = mutableListOf() + val hdrWarnings = mutableListOf() + for (videoFileName in videoFiles) { Log.i(TAG, "Transcoding '$videoFileName' with preset ${preset.name}...") try { transcodeVideo(videoFileName, preset) Log.i(TAG, "Successfully transcoded '$videoFileName' with preset ${preset.name}") + } catch (e: HdrDecoderUnavailableException) { + Log.w(TAG, "No decoder available for HDR video '$videoFileName' with preset ${preset.name} (device limitation)", e) + hdrWarnings.add("$videoFileName [${preset.name}]: ${e::class.simpleName}: ${e.message}") + } catch (e: FileNotFoundException) { + Log.w(TAG, "Skipping '$videoFileName' with preset ${preset.name}: encoder not available on this device", e) + } catch (e: EncodingException) { + val isHdrDeviceLimitation = e.isHdrInput && (!e.toneMapApplied || isMediaCodecRuntimeFailure(e)) + if (isHdrDeviceLimitation) { + Log.w(TAG, "HDR video '$videoFileName' failed with preset ${preset.name} (device limitation, toneMap=${e.toneMapApplied}, decoder=${e.decoderName}, encoder=${e.encoderName})", e) + hdrWarnings.add("$videoFileName [${preset.name}]: ${e::class.simpleName}: ${e.message}") + } else { + Log.e(TAG, "Failed to transcode '$videoFileName' with preset ${preset.name} (hdr=${e.isHdrInput}, toneMap=${e.toneMapApplied}, decoder=${e.decoderName}, encoder=${e.encoderName})", e) + failures.add("$videoFileName: ${e::class.simpleName}: ${e.message}") + } } catch (e: Exception) { Log.e(TAG, "Failed to transcode '$videoFileName' with preset ${preset.name}", e) failures.add("$videoFileName: ${e::class.simpleName}: ${e.message}") } } + if (hdrWarnings.isNotEmpty()) { + Log.w(TAG, "${hdrWarnings.size} HDR video(s) could not be transcoded (device limitation, not a bug):\n${hdrWarnings.joinToString("\n")}") + } + if (failures.isNotEmpty()) { Assert.fail( "${failures.size}/${videoFiles.size} video(s) failed transcoding with ${preset.name}:\n" + @@ -106,6 +129,24 @@ class VideoTranscodeInstrumentationTest { } } + /** + * Checks if the root cause of an exception is an [IllegalStateException] from + * [android.media.MediaCodec] native methods. This indicates the hardware codec crashed + * during processing, which for HDR content is a device limitation (not a Signal bug). + */ + private fun isMediaCodecRuntimeFailure(e: Exception): Boolean { + var cause: Throwable? = e.cause + while (cause != null) { + if (cause is IllegalStateException && + cause.stackTrace.any { it.className == "android.media.MediaCodec" } + ) { + return true + } + cause = cause.cause + } + return false + } + private fun transcodeVideo(videoFileName: String, preset: TranscodingPreset) { val inputFile = createTempFile("input-", "-$videoFileName") val outputFile = createTempFile("output-${preset.name}-", "-$videoFileName") diff --git a/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/MediaConverter.java b/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/MediaConverter.java index f217d3b7f3..719cdf4744 100644 --- a/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/MediaConverter.java +++ b/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/MediaConverter.java @@ -20,6 +20,7 @@ package org.thoughtcrime.securesms.video.videoconverter; import android.media.MediaCodecInfo; import android.media.MediaCodecList; +import android.media.MediaExtractor; import android.media.MediaFormat; import androidx.annotation.NonNull; @@ -33,6 +34,7 @@ import org.thoughtcrime.securesms.video.interfaces.MediaInput; import org.thoughtcrime.securesms.video.interfaces.Muxer; import org.thoughtcrime.securesms.video.videoconverter.exceptions.EncodingException; import org.thoughtcrime.securesms.video.videoconverter.muxer.StreamingMuxer; +import org.thoughtcrime.securesms.video.videoconverter.utils.MediaCodecCompat; import java.io.File; import java.io.FileDescriptor; @@ -226,7 +228,33 @@ public final class MediaConverter { } } if (exception != null) { - throw new EncodingException("Transcode failed", exception); + EncodingException encodingException = new EncodingException("Transcode failed", exception); + if (videoTrackConverter != null) { + encodingException.isHdrInput = videoTrackConverter.isHdrInput(); + encodingException.toneMapApplied = videoTrackConverter.isToneMapApplied(); + encodingException.decoderName = videoTrackConverter.getDecoderName(); + encodingException.encoderName = videoTrackConverter.getEncoderName(); + } else { + // VideoTrackConverter failed during creation. Try to determine HDR + // status independently so the caller can classify the failure. + try { + final MediaExtractor extractor = mInput.createExtractor(); + try { + for (int i = 0; i < extractor.getTrackCount(); i++) { + final MediaFormat format = extractor.getTrackFormat(i); + if (getMimeTypeFor(format).startsWith("video/")) { + encodingException.isHdrInput = MediaCodecCompat.isHdrVideo(format); + break; + } + } + } finally { + extractor.release(); + } + } catch (Exception e) { + Log.w(TAG, "Could not determine HDR status for failed transcode", e); + } + } + throw encodingException; } return mdatContentLength; diff --git a/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/VideoTrackConverter.java b/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/VideoTrackConverter.java index 0a5d9eceff..69bbc0fa42 100644 --- a/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/VideoTrackConverter.java +++ b/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/VideoTrackConverter.java @@ -13,6 +13,7 @@ import androidx.annotation.Nullable; import org.signal.core.util.logging.Log; import org.thoughtcrime.securesms.video.interfaces.MediaInput; import org.thoughtcrime.securesms.video.interfaces.Muxer; +import org.thoughtcrime.securesms.video.videoconverter.exceptions.HdrDecoderUnavailableException; import org.thoughtcrime.securesms.video.videoconverter.utils.Extensions; import org.thoughtcrime.securesms.video.videoconverter.utils.MediaCodecCompat; import org.thoughtcrime.securesms.video.videoconverter.utils.Preconditions; @@ -38,6 +39,11 @@ final class VideoTrackConverter { private static final float FRAME_RATE_TOLERANCE = 0.05f; // tolerance for transcoding VFR -> CFR + private boolean mIsHdrInput; + private boolean mToneMapApplied; + private String mDecoderName; + private String mEncoderName; + private final long mTimeFrom; private final long mTimeTo; @@ -421,6 +427,11 @@ final class VideoTrackConverter { Preconditions.checkState("decoded frame count should be less than extracted frame count", mVideoDecodedFrameCount <= mVideoExtractedFrameCount); } + boolean isHdrInput() { return mIsHdrInput; } + boolean isToneMapApplied() { return mToneMapApplied; } + String getDecoderName() { return mDecoderName; } + String getEncoderName() { return mEncoderName; } + private static String createFragmentShader( final int srcWidth, final int srcHeight, @@ -473,31 +484,53 @@ final class VideoTrackConverter { MediaCodec createVideoDecoder( final @NonNull MediaFormat inputFormat, final @NonNull Surface surface) throws IOException { - final boolean isHdr = MediaCodecCompat.isHdrVideo(inputFormat); + final boolean isHdr = MediaCodecCompat.isHdrVideo(inputFormat); + final boolean requestToneMapping = Build.VERSION.SDK_INT >= 31 && isHdr; final List> candidates = MediaCodecCompat.findDecoderCandidates(inputFormat); + + mIsHdrInput = isHdr; Exception lastException = null; for (int i = 0; i < candidates.size(); i++) { final Pair candidate = candidates.get(i); - final String codecName = candidate.getFirst(); - final MediaFormat decoderFormat = candidate.getSecond(); + final String codecName = candidate.getFirst(); + final MediaFormat baseFormat = candidate.getSecond(); MediaCodec decoder = null; try { decoder = MediaCodec.createByCodecName(codecName); - // For HDR video, request SDR tone-mapping from the decoder (API 31+). - if (Build.VERSION.SDK_INT >= 31 && isHdr) { - decoderFormat.setInteger(MediaFormat.KEY_COLOR_TRANSFER_REQUEST, MediaFormat.COLOR_TRANSFER_SDR_VIDEO); + // For HDR video on API 31+, try requesting SDR tone-mapping. + // Some codecs reject this key, so we catch the error and retry without it. + if (requestToneMapping) { + try { + final MediaFormat toneMapFormat = new MediaFormat(baseFormat); + toneMapFormat.setInteger(MediaFormat.KEY_COLOR_TRANSFER_REQUEST, MediaFormat.COLOR_TRANSFER_SDR_VIDEO); + decoder.configure(toneMapFormat, surface, null, 0); + decoder.start(); + + mToneMapApplied = isToneMapEffective(decoder, codecName); + mDecoderName = codecName; + if (i > 0) { + Log.w(TAG, "Video decoder: succeeded with fallback codec " + codecName + " (attempt " + (i + 1) + " of " + candidates.size() + ")"); + } + return decoder; + } catch (IllegalArgumentException | IllegalStateException e) { + Log.w(TAG, "Video decoder: codec " + codecName + " rejected tone-mapping request, retrying without (attempt " + (i + 1) + " of " + candidates.size() + ")", e); + decoder.release(); + decoder = MediaCodec.createByCodecName(codecName); + } } - decoder.configure(decoderFormat, surface, null, 0); + + decoder.configure(baseFormat, surface, null, 0); decoder.start(); - if (i > 0) { - Log.w(TAG, "Video decoder: succeeded with fallback codec " + codecName + " (attempt " + (i + 1) + " of " + candidates.size() + ")"); + mDecoderName = codecName; + if (i > 0 || requestToneMapping) { + Log.w(TAG, "Video decoder: succeeded with codec " + codecName + (requestToneMapping ? " (no tone-mapping)" : "") + " (attempt " + (i + 1) + " of " + candidates.size() + ")"); } return decoder; - } catch (IllegalStateException e) { + } catch (IllegalArgumentException | IllegalStateException e) { Log.w(TAG, "Video decoder: codec " + codecName + " failed (attempt " + (i + 1) + " of " + candidates.size() + ")", e); lastException = e; if (decoder != null) { @@ -509,6 +542,9 @@ final class VideoTrackConverter { } } + if (mIsHdrInput) { + throw new HdrDecoderUnavailableException("All video decoder codecs failed for HDR video", lastException); + } throw new IOException("All video decoder codecs failed", lastException); } @@ -517,7 +553,7 @@ final class VideoTrackConverter { * {@link MediaCodec#createInputSurface()} (between configure and start) and then * {@link MediaCodec#start()} after the decoder has been created. */ - private static @NonNull + private @NonNull MediaCodec createVideoEncoder( final @NonNull List codecCandidates, final @NonNull MediaFormat format) throws IOException { @@ -530,11 +566,12 @@ final class VideoTrackConverter { try { encoder = MediaCodec.createByCodecName(codecInfo.getName()); encoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); + mEncoderName = codecInfo.getName(); if (i > 0) { Log.w(TAG, "Video encoder: succeeded with fallback codec " + codecInfo.getName() + " (attempt " + (i + 1) + " of " + codecCandidates.size() + ")"); } return encoder; - } catch (IllegalStateException e) { + } catch (IllegalArgumentException | IllegalStateException e) { Log.w(TAG, "Video encoder: codec " + codecInfo.getName() + " failed (attempt " + (i + 1) + " of " + codecCandidates.size() + ")", e); lastException = e; if (encoder != null) { @@ -563,5 +600,37 @@ final class VideoTrackConverter { return MediaConverter.getMimeTypeFor(format).startsWith("video/"); } + /** + * Checks whether HDR-to-SDR tone-mapping is effective after the decoder has been configured + * and started with {@link MediaFormat#KEY_COLOR_TRANSFER_REQUEST}. Some codecs (especially + * software decoders and some hardware decoders) accept the tone-mapping key without error + * but don't actually perform the conversion. + */ + private static boolean isToneMapEffective(final @NonNull MediaCodec decoder, final @NonNull String codecName) { + // Software codecs never perform HDR→SDR tone-mapping. + String lower = codecName.toLowerCase(java.util.Locale.ROOT); + if (lower.startsWith("omx.google.") || lower.startsWith("c2.android.")) { + Log.w(TAG, "Video decoder: software codec " + codecName + " cannot perform HDR tone-mapping"); + return false; + } + + // For hardware codecs, verify the output format. If the output transfer function + // is still HDR (ST2084 or HLG), the decoder accepted the request but isn't honoring it. + try { + MediaFormat outputFormat = decoder.getOutputFormat(); + if (outputFormat.containsKey(MediaFormat.KEY_COLOR_TRANSFER)) { + int transfer = outputFormat.getInteger(MediaFormat.KEY_COLOR_TRANSFER); + if (transfer == MediaFormat.COLOR_TRANSFER_ST2084 || transfer == MediaFormat.COLOR_TRANSFER_HLG) { + Log.w(TAG, "Video decoder: codec " + codecName + " accepted tone-mapping but output transfer is " + transfer + " (still HDR)"); + return false; + } + } + } catch (Exception e) { + Log.w(TAG, "Video decoder: could not verify tone-mapping for codec " + codecName, e); + } + + return true; + } + } diff --git a/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/exceptions/EncodingException.kt b/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/exceptions/EncodingException.kt index 91ba42096d..1f5138adea 100644 --- a/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/exceptions/EncodingException.kt +++ b/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/exceptions/EncodingException.kt @@ -5,6 +5,15 @@ package org.thoughtcrime.securesms.video.videoconverter.exceptions class EncodingException : Exception { + /** Whether the input video was HDR content. */ + @JvmField var isHdrInput: Boolean = false + /** Whether HDR-to-SDR tone-mapping was successfully applied to the decoder. */ + @JvmField var toneMapApplied: Boolean = false + /** The name of the video decoder codec that was selected, or null if decoder creation failed. */ + @JvmField var decoderName: String? = null + /** The name of the video encoder codec that was selected, or null if encoder creation failed. */ + @JvmField var encoderName: String? = null + constructor(message: String?) : super(message) constructor(message: String?, inner: Exception?) : super(message, inner) } diff --git a/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/exceptions/HdrDecoderUnavailableException.kt b/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/exceptions/HdrDecoderUnavailableException.kt new file mode 100644 index 0000000000..585edb5aa2 --- /dev/null +++ b/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/exceptions/HdrDecoderUnavailableException.kt @@ -0,0 +1,13 @@ +/* + * Copyright 2026 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ +package org.thoughtcrime.securesms.video.videoconverter.exceptions + +import java.io.IOException + +/** + * Thrown when no decoder on the device can properly decode HDR video content. + * This is typically a device limitation, not a bug. + */ +class HdrDecoderUnavailableException(message: String, cause: Throwable?) : IOException(message, cause) diff --git a/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/utils/MediaCodecCompat.kt b/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/utils/MediaCodecCompat.kt index b7f5f3a827..e51b3b5336 100644 --- a/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/utils/MediaCodecCompat.kt +++ b/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/utils/MediaCodecCompat.kt @@ -266,6 +266,26 @@ object MediaCodecCompat { return true } + // On older APIs, the extractor may not populate KEY_COLOR_TRANSFER or HDR metadata keys. + // Check the HEVC profile as a fallback: Main 10 and its HDR variants indicate 10-bit + // content that typically requires HDR-capable decoders. + val mime = try { format.getString(MediaFormat.KEY_MIME) } catch (_: Exception) { null } + if (mime.equals("video/hevc", ignoreCase = true)) { + try { + val profile = format.getInteger(MediaFormat.KEY_PROFILE) + if (profile == CodecProfileLevel.HEVCProfileMain10 || + profile == CodecProfileLevel.HEVCProfileMain10HDR10 || + profile == CodecProfileLevel.HEVCProfileMain10HDR10Plus + ) { + return true + } + } catch (_: NullPointerException) { + // key doesn't exist + } catch (_: ClassCastException) { + // key exists but wrong type + } + } + return false } }