From 32dc36d937387aa422beba3678a5143568d6112d Mon Sep 17 00:00:00 2001 From: Greyson Parrelli Date: Thu, 19 Feb 2026 15:45:46 -0500 Subject: [PATCH] Fix various transcoding issues on samsung devices. --- demo/video/build.gradle.kts | 8 +- .../video/app/ExampleInstrumentedTest.kt | 27 ---- .../video/videoconverter/MediaConverter.java | 42 +++-- .../videoconverter/VideoTrackConverter.java | 147 ++++++++++-------- .../videoconverter/utils/MediaCodecCompat.kt | 119 +++++++++++++- 5 files changed, 228 insertions(+), 115 deletions(-) delete mode 100644 demo/video/src/androidTest/java/org/thoughtcrime/video/app/ExampleInstrumentedTest.kt diff --git a/demo/video/build.gradle.kts b/demo/video/build.gradle.kts index fd9d08bb6d..af38c2c925 100644 --- a/demo/video/build.gradle.kts +++ b/demo/video/build.gradle.kts @@ -29,11 +29,16 @@ android { versionName = "1.0" testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner" + testInstrumentationRunnerArguments["clearPackageData"] = "true" vectorDrawables { useSupportLibrary = true } } + testOptions { + execution = "ANDROIDX_TEST_ORCHESTRATOR" + } + buildTypes { release { isMinifyEnabled = false @@ -62,7 +67,7 @@ android { sourceSets { val sampleVideosPath = localProperties?.getProperty("sample.videos.dir") if (sampleVideosPath != null) { - val sampleVideosDir = File(rootProject.projectDir, sampleVideosPath) + val sampleVideosDir = File(sampleVideosPath) if (sampleVideosDir.isDirectory) { getByName("androidTest").assets.srcDir(sampleVideosDir) } @@ -85,4 +90,5 @@ dependencies { androidTestImplementation(testLibs.junit.junit) androidTestImplementation(testLibs.androidx.test.runner) androidTestImplementation(testLibs.androidx.test.ext.junit.ktx) + androidTestUtil(testLibs.androidx.test.orchestrator) } diff --git a/demo/video/src/androidTest/java/org/thoughtcrime/video/app/ExampleInstrumentedTest.kt b/demo/video/src/androidTest/java/org/thoughtcrime/video/app/ExampleInstrumentedTest.kt deleted file mode 100644 index be3f148b04..0000000000 --- a/demo/video/src/androidTest/java/org/thoughtcrime/video/app/ExampleInstrumentedTest.kt +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright 2023 Signal Messenger, LLC - * SPDX-License-Identifier: AGPL-3.0-only - */ - -package org.thoughtcrime.video.app - -import androidx.test.ext.junit.runners.AndroidJUnit4 -import androidx.test.platform.app.InstrumentationRegistry -import org.junit.Assert.assertEquals -import org.junit.Test -import org.junit.runner.RunWith - -/** - * Instrumented test, which will execute on an Android device. - * - * See [testing documentation](http://d.android.com/tools/testing). - */ -@RunWith(AndroidJUnit4::class) -class ExampleInstrumentedTest { - @Test - fun useAppContext() { - // Context of the app under test. - val appContext = InstrumentationRegistry.getInstrumentation().targetContext - assertEquals("org.thoughtcrime.video.app", appContext.packageName) - } -} diff --git a/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/MediaConverter.java b/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/MediaConverter.java index 80182de7f3..f217d3b7f3 100644 --- a/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/MediaConverter.java +++ b/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/MediaConverter.java @@ -41,6 +41,10 @@ import java.io.IOException; import java.io.OutputStream; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; @SuppressWarnings("WeakerAccess") public final class MediaConverter { @@ -325,22 +329,38 @@ public final class MediaConverter { * found. */ static MediaCodecInfo selectCodec(final String mimeType) { - final int numCodecs = MediaCodecList.getCodecCount(); - for (int i = 0; i < numCodecs; i++) { - final MediaCodecInfo codecInfo = MediaCodecList.getCodecInfoAt(i); + final List codecs = selectCodecs(mimeType); + return codecs.isEmpty() ? null : codecs.get(0); + } - if (!codecInfo.isEncoder()) { - continue; - } + /** + * Returns all codecs capable of encoding the specified MIME type, ordered with hardware-preferred + * codecs ({@link MediaCodecList#REGULAR_CODECS}) first, then additional codecs from + * {@link MediaCodecList#ALL_CODECS} (includes software), deduplicated by name. + */ + static List selectCodecs(final String mimeType) { + final List candidates = new ArrayList<>(); + final Set seen = new HashSet<>(); - final String[] types = codecInfo.getSupportedTypes(); - for (String type : types) { - if (type.equalsIgnoreCase(mimeType)) { - return codecInfo; + for (MediaCodecInfo codecInfo : new MediaCodecList(MediaCodecList.REGULAR_CODECS).getCodecInfos()) { + if (!codecInfo.isEncoder()) continue; + for (String type : codecInfo.getSupportedTypes()) { + if (type.equalsIgnoreCase(mimeType) && seen.add(codecInfo.getName())) { + candidates.add(codecInfo); } } } - return null; + + for (MediaCodecInfo codecInfo : new MediaCodecList(MediaCodecList.ALL_CODECS).getCodecInfos()) { + if (!codecInfo.isEncoder()) continue; + for (String type : codecInfo.getSupportedTypes()) { + if (type.equalsIgnoreCase(mimeType) && seen.add(codecInfo.getName())) { + candidates.add(codecInfo); + } + } + } + + return candidates; } interface Output { diff --git a/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/VideoTrackConverter.java b/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/VideoTrackConverter.java index 105965e33f..0a5d9eceff 100644 --- a/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/VideoTrackConverter.java +++ b/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/VideoTrackConverter.java @@ -5,7 +5,6 @@ import android.media.MediaCodecInfo; import android.media.MediaExtractor; import android.media.MediaFormat; import android.os.Build; -import android.os.Bundle; import android.view.Surface; import androidx.annotation.NonNull; @@ -21,8 +20,7 @@ import org.thoughtcrime.securesms.video.videoconverter.utils.Preconditions; import java.io.FileNotFoundException; import java.io.IOException; import java.nio.ByteBuffer; -import java.util.concurrent.atomic.AtomicReference; - +import java.util.List; import kotlin.Pair; final class VideoTrackConverter { @@ -40,8 +38,6 @@ final class VideoTrackConverter { private static final float FRAME_RATE_TOLERANCE = 0.05f; // tolerance for transcoding VFR -> CFR - private static final String VENDOR_DOLBY_CODEC_TRANSFER_PARAMKEY = "vendor.dolby.codec.transfer.value"; - private final long mTimeFrom; private final long mTimeTo; @@ -106,13 +102,13 @@ final class VideoTrackConverter { mTimeTo = timeTo; mVideoExtractor = videoExtractor; - final MediaCodecInfo videoCodecInfo = MediaConverter.selectCodec(videoCodec); - if (videoCodecInfo == null) { + final List videoCodecCandidates = MediaConverter.selectCodecs(videoCodec); + if (videoCodecCandidates.isEmpty()) { // Don't fail CTS if they don't have an AVC codec (not here, anyway). Log.e(TAG, "Unable to find an appropriate codec for " + videoCodec); throw new FileNotFoundException(); } - if (VERBOSE) Log.d(TAG, "video found codec: " + videoCodecInfo.getName()); + if (VERBOSE) Log.d(TAG, "video found codecs: " + videoCodecCandidates.size()); final MediaFormat inputVideoFormat = mVideoExtractor.getTrackFormat(videoInputTrack); @@ -161,21 +157,20 @@ final class VideoTrackConverter { outputVideoFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, OUTPUT_VIDEO_IFRAME_INTERVAL); if (VERBOSE) Log.d(TAG, "video format: " + outputVideoFormat); - // Create a MediaCodec for the desired codec, then configure it as an encoder with - // our desired properties. Request a Surface to use for input. - final AtomicReference inputSurfaceReference = new AtomicReference<>(); - mVideoEncoder = createVideoEncoder(videoCodecInfo, outputVideoFormat, inputSurfaceReference); - mInputSurface = new InputSurface(inputSurfaceReference.get()); - mInputSurface.makeCurrent(); - // Create a MediaCodec for the decoder, based on the extractor's format. - mOutputSurface = new OutputSurface(); - - mOutputSurface.changeFragmentShader(createFragmentShader( + final String fragmentShader = createFragmentShader( inputVideoFormat.getInteger(MediaFormat.KEY_WIDTH), inputVideoFormat.getInteger(MediaFormat.KEY_HEIGHT), - outputWidth, outputHeight)); + outputWidth, outputHeight); + // Configure the encoder but do NOT start it yet. The encoder's start() is + // deferred until after the decoder is created, so that the decoder gets first + // access to hardware codec resources on memory-constrained devices. + mVideoEncoder = createVideoEncoder(videoCodecCandidates, outputVideoFormat); + mInputSurface = new InputSurface(mVideoEncoder.createInputSurface()); + mInputSurface.makeCurrent(); + mOutputSurface = new OutputSurface(); + mOutputSurface.changeFragmentShader(fragmentShader); mVideoDecoder = createVideoDecoder(inputVideoFormat, mOutputSurface.getSurface()); - + mVideoEncoder.start(); mVideoDecoderInputBuffers = mVideoDecoder.getInputBuffers(); mVideoEncoderOutputBuffers = mVideoEncoder.getOutputBuffers(); mVideoDecoderOutputBufferInfo = new MediaCodec.BufferInfo(); @@ -477,62 +472,78 @@ final class VideoTrackConverter { private @NonNull MediaCodec createVideoDecoder( final @NonNull MediaFormat inputFormat, - final @NonNull Surface surface) { - final Pair decoderPair = MediaCodecCompat.findDecoder(inputFormat); - final MediaCodec decoder = decoderPair.getFirst(); + final @NonNull Surface surface) throws IOException { + final boolean isHdr = MediaCodecCompat.isHdrVideo(inputFormat); + final List> candidates = MediaCodecCompat.findDecoderCandidates(inputFormat); + Exception lastException = null; - // For HDR video, request SDR tone-mapping from the decoder. Only do this for HDR content - // (PQ or HLG transfer), as some hardware decoders (e.g. Qualcomm HEVC) crash when this is - // set on non-HDR video. - final boolean isHdr = MediaCodecCompat.isHdrVideo(decoderPair.getSecond()); - if (Build.VERSION.SDK_INT >= 31 && isHdr) { - decoderPair.getSecond().setInteger(MediaFormat.KEY_COLOR_TRANSFER_REQUEST, MediaFormat.COLOR_TRANSFER_SDR_VIDEO); - } - decoder.configure(decoderPair.getSecond(), surface, null, 0); - decoder.start(); - if (Build.VERSION.SDK_INT >= 31 && isHdr) { - try { - MediaCodec.ParameterDescriptor descriptor = decoder.getParameterDescriptor(VENDOR_DOLBY_CODEC_TRANSFER_PARAMKEY); - if (descriptor != null) { - Bundle transferBundle = new Bundle(); - transferBundle.putString(VENDOR_DOLBY_CODEC_TRANSFER_PARAMKEY, "transfer.sdr.normal"); - decoder.setParameters(transferBundle); + for (int i = 0; i < candidates.size(); i++) { + final Pair candidate = candidates.get(i); + final String codecName = candidate.getFirst(); + final MediaFormat decoderFormat = candidate.getSecond(); + MediaCodec decoder = null; + + try { + decoder = MediaCodec.createByCodecName(codecName); + + // For HDR video, request SDR tone-mapping from the decoder (API 31+). + if (Build.VERSION.SDK_INT >= 31 && isHdr) { + decoderFormat.setInteger(MediaFormat.KEY_COLOR_TRANSFER_REQUEST, MediaFormat.COLOR_TRANSFER_SDR_VIDEO); + } + decoder.configure(decoderFormat, surface, null, 0); + decoder.start(); + + if (i > 0) { + Log.w(TAG, "Video decoder: succeeded with fallback codec " + codecName + " (attempt " + (i + 1) + " of " + candidates.size() + ")"); + } + return decoder; + } catch (IllegalStateException e) { + Log.w(TAG, "Video decoder: codec " + codecName + " failed (attempt " + (i + 1) + " of " + candidates.size() + ")", e); + lastException = e; + if (decoder != null) { + decoder.release(); + } + } catch (IOException e) { + Log.w(TAG, "Video decoder: codec " + codecName + " failed to create (attempt " + (i + 1) + " of " + candidates.size() + ")", e); + lastException = e; } - } catch (IllegalStateException e) { - Log.w(TAG, "Failed to set Dolby Vision transfer parameter", e); - } } - return decoder; + + throw new IOException("All video decoder codecs failed", lastException); } - private @NonNull + /** + * Creates and configures a video encoder but does NOT start it. The caller must call + * {@link MediaCodec#createInputSurface()} (between configure and start) and then + * {@link MediaCodec#start()} after the decoder has been created. + */ + private static @NonNull MediaCodec createVideoEncoder( - final @NonNull MediaCodecInfo codecInfo, - final @NonNull MediaFormat format, - final @NonNull AtomicReference surfaceReference) throws IOException { - boolean tonemapRequested = isTonemapEnabled(format); - final MediaCodec encoder = MediaCodec.createByCodecName(codecInfo.getName()); - encoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); - if (tonemapRequested && !isTonemapEnabled(format)) { - Log.d(TAG, "HDR tone-mapping requested but not supported by the decoder."); - } - // Must be called before start() - surfaceReference.set(encoder.createInputSurface()); - encoder.start(); - return encoder; - } + final @NonNull List codecCandidates, + final @NonNull MediaFormat format) throws IOException { + Exception lastException = null; - private static boolean isTonemapEnabled(@NonNull MediaFormat format) { - if (Build.VERSION.SDK_INT < 31) { - return false; - } - try { - int request = format.getInteger(MediaFormat.KEY_COLOR_TRANSFER_REQUEST); - return request == MediaFormat.COLOR_TRANSFER_SDR_VIDEO; - } catch (NullPointerException npe) { - // transfer request key does not exist, tone mapping not requested - return false; + for (int i = 0; i < codecCandidates.size(); i++) { + final MediaCodecInfo codecInfo = codecCandidates.get(i); + MediaCodec encoder = null; + + try { + encoder = MediaCodec.createByCodecName(codecInfo.getName()); + encoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); + if (i > 0) { + Log.w(TAG, "Video encoder: succeeded with fallback codec " + codecInfo.getName() + " (attempt " + (i + 1) + " of " + codecCandidates.size() + ")"); + } + return encoder; + } catch (IllegalStateException e) { + Log.w(TAG, "Video encoder: codec " + codecInfo.getName() + " failed (attempt " + (i + 1) + " of " + codecCandidates.size() + ")", e); + lastException = e; + if (encoder != null) { + encoder.release(); + } + } } + + throw new IOException("All video encoder codecs failed", lastException); } private static int getAndSelectVideoTrackIndex(@NonNull MediaExtractor extractor) { diff --git a/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/utils/MediaCodecCompat.kt b/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/utils/MediaCodecCompat.kt index 334dcc7281..b7f5f3a827 100644 --- a/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/utils/MediaCodecCompat.kt +++ b/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/utils/MediaCodecCompat.kt @@ -23,6 +23,57 @@ object MediaCodecCompat { const val MEDIA_FORMAT_KEY_CODEC_SPECIFIC_DATA_1 = "csd-1" const val MEDIA_FORMAT_KEY_CODEC_SPECIFIC_DATA_2 = "csd-2" + /** + * Returns all matching decoders for the given [inputFormat], ordered with hardware-preferred + * codecs ([MediaCodecList.REGULAR_CODECS]) first, then additional codecs from + * [MediaCodecList.ALL_CODECS] (includes software), deduplicated by name. + * + * Each entry is a codec name paired with the format to use (which may differ from [inputFormat] + * for Dolby Vision content, where it is resolved to the base-layer format). + */ + @JvmStatic + @Throws(IOException::class) + fun findDecoderCandidates(inputFormat: MediaFormat): List> { + val mimeType = inputFormat.getString(MediaFormat.KEY_MIME) + + val resolvedFormat: MediaFormat = if (MediaFormat.MIMETYPE_VIDEO_DOLBY_VISION == mimeType) { + val dvFormat = if (Build.VERSION.SDK_INT >= 29) MediaFormat(inputFormat) else inputFormat + resolveDolbyVisionFormat(dvFormat) ?: throw IOException("Can't resolve Dolby Vision format for decoder candidates!") + } else { + inputFormat + } + + val resolvedMime = resolvedFormat.getString(MediaFormat.KEY_MIME) + ?: throw IOException("No MIME type in resolved format!") + + val candidates = mutableListOf>() + val seen = mutableSetOf() + + for (codecInfo in MediaCodecList(MediaCodecList.REGULAR_CODECS).codecInfos) { + if (codecInfo.isEncoder) continue + if (codecInfo.supportedTypes.any { it.equals(resolvedMime, ignoreCase = true) }) { + if (seen.add(codecInfo.name)) { + candidates.add(Pair(codecInfo.name, resolvedFormat)) + } + } + } + + for (codecInfo in MediaCodecList(MediaCodecList.ALL_CODECS).codecInfos) { + if (codecInfo.isEncoder) continue + if (codecInfo.supportedTypes.any { it.equals(resolvedMime, ignoreCase = true) }) { + if (seen.add(codecInfo.name)) { + candidates.add(Pair(codecInfo.name, resolvedFormat)) + } + } + } + + if (candidates.isEmpty()) { + throw IOException("Can't find decoder for $resolvedMime!") + } + + return candidates + } + @JvmStatic fun findDecoder(inputFormat: MediaFormat): Pair { val codecs = MediaCodecList(MediaCodecList.REGULAR_CODECS) @@ -50,6 +101,39 @@ object MediaCodecCompat { throw IOException("Can't create decoder for $mimeType!") } + /** + * Resolves a Dolby Vision [MediaFormat] to its base-layer format by mutating the MIME type, + * profile, and level. Returns the mutated format, or null if the DV profile is unsupported. + */ + private fun resolveDolbyVisionFormat(mediaFormat: MediaFormat): MediaFormat? { + if (MediaFormat.MIMETYPE_VIDEO_DOLBY_VISION != mediaFormat.getString(MediaFormat.KEY_MIME)) { + throw IllegalStateException("Must supply Dolby Vision MediaFormat!") + } + + return try { + when (mediaFormat.getInteger(MediaFormat.KEY_PROFILE)) { + CodecProfileLevel.DolbyVisionProfileDvheDtr, + CodecProfileLevel.DolbyVisionProfileDvheSt -> { + mediaFormat.setString(MediaFormat.KEY_MIME, MediaFormat.MIMETYPE_VIDEO_HEVC) + mediaFormat.setInteger(MediaFormat.KEY_PROFILE, CodecProfileLevel.HEVCProfileMain10) + mediaFormat.setBaseCodecLevelFromDolbyVisionLevel() + mediaFormat + } + + CodecProfileLevel.DolbyVisionProfileDvavSe -> { + mediaFormat.setString(MediaFormat.KEY_MIME, MediaFormat.MIMETYPE_VIDEO_AVC) + mediaFormat.setInteger(MediaFormat.KEY_PROFILE, CodecProfileLevel.AVCProfileHigh) + mediaFormat.setBaseCodecLevelFromDolbyVisionLevel() + mediaFormat + } + + else -> null + } + } catch (npe: NullPointerException) { + null + } + } + /** * Find backup decoder for a [MediaFormat] object with a MIME type of Dolby Vision. * @@ -151,18 +235,37 @@ object MediaCodecCompat { } /** - * Returns true if the given [MediaFormat] describes an HDR video (PQ or HLG color transfer). - * Some hardware decoders crash when tone-mapping parameters are set on non-HDR video. + * Returns true if the given [MediaFormat] describes an HDR video. + * + * Checks three signals: + * 1. PQ or HLG color transfer function (standard indicator) + * 2. HDR static metadata (mastering display info, present in HDR10 content) + * 3. HDR10+ dynamic metadata (present in HDR10+ content, API 29+) + * + * Some devices report non-standard color transfer values (e.g. 65791) for HDR content, + * so checking metadata keys provides a more reliable detection. */ @JvmStatic fun isHdrVideo(format: MediaFormat): Boolean { - return try { + try { val colorTransfer = format.getInteger(MediaFormat.KEY_COLOR_TRANSFER) - colorTransfer == MediaFormat.COLOR_TRANSFER_ST2084 || colorTransfer == MediaFormat.COLOR_TRANSFER_HLG - } catch (e: NullPointerException) { - false - } catch (e: ClassCastException) { - false + if (colorTransfer == MediaFormat.COLOR_TRANSFER_ST2084 || colorTransfer == MediaFormat.COLOR_TRANSFER_HLG) { + return true + } + } catch (_: NullPointerException) { + // key doesn't exist + } catch (_: ClassCastException) { + // key exists but wrong type } + + if (format.containsKey(MediaFormat.KEY_HDR_STATIC_INFO)) { + return true + } + + if (Build.VERSION.SDK_INT >= 29 && format.containsKey(MediaFormat.KEY_HDR10_PLUS_INFO)) { + return true + } + + return false } }