Signal/Android
1
0
Fork 0
mirror of https://github.com/signalapp/Signal-Android.git synced 2026-04-02 00:17:41 +01:00
Code Issues Packages Projects Releases Wiki Activity

Fix various transcoding issues on samsung devices.

Browse Source
This commit is contained in:
Greyson Parrelli
2026-02-19 15:45:46 -05:00
committed by Cody Henthorne
parent 771d49bfa8
commit 32dc36d937
5 changed files with 228 additions and 115 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
demo/video/build.gradle.kts
View File

@@ -29,11 +29,16 @@ android {
versionName = "1.0"
testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner"
testInstrumentationRunnerArguments["clearPackageData"] = "true"
vectorDrawables {
useSupportLibrary = true
}
}
testOptions {
execution = "ANDROIDX_TEST_ORCHESTRATOR"
}
buildTypes {
release {
isMinifyEnabled = false
@@ -62,7 +67,7 @@ android {
sourceSets {
val sampleVideosPath = localProperties?.getProperty("sample.videos.dir")
if (sampleVideosPath != null) {
val sampleVideosDir = File(rootProject.projectDir, sampleVideosPath)
val sampleVideosDir = File(sampleVideosPath)
if (sampleVideosDir.isDirectory) {
getByName("androidTest").assets.srcDir(sampleVideosDir)
}
@@ -85,4 +90,5 @@ dependencies {
androidTestImplementation(testLibs.junit.junit)
androidTestImplementation(testLibs.androidx.test.runner)
androidTestImplementation(testLibs.androidx.test.ext.junit.ktx)
androidTestUtil(testLibs.androidx.test.orchestrator)
}

27
demo/video/src/androidTest/java/org/thoughtcrime/video/app/ExampleInstrumentedTest.kt
View File

@@ -1,27 +0,0 @@
/*
* Copyright 2023 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.thoughtcrime.video.app
import androidx.test.ext.junit.runners.AndroidJUnit4
import androidx.test.platform.app.InstrumentationRegistry
import org.junit.Assert.assertEquals
import org.junit.Test
import org.junit.runner.RunWith
/**
* Instrumented test, which will execute on an Android device.
*
* See [testing documentation](http://d.android.com/tools/testing).
*/
@RunWith(AndroidJUnit4::class)
class ExampleInstrumentedTest {
@Test
fun useAppContext() {
// Context of the app under test.
val appContext = InstrumentationRegistry.getInstrumentation().targetContext
assertEquals("org.thoughtcrime.video.app", appContext.packageName)
}
}

42
lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/MediaConverter.java
View File

@@ -41,6 +41,10 @@ import java.io.IOException;
import java.io.OutputStream;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
@SuppressWarnings("WeakerAccess")
public final class MediaConverter {
@@ -325,22 +329,38 @@ public final class MediaConverter {
* found.
*/
static MediaCodecInfo selectCodec(final String mimeType) {
final int numCodecs = MediaCodecList.getCodecCount();
for (int i = 0; i < numCodecs; i++) {
final MediaCodecInfo codecInfo = MediaCodecList.getCodecInfoAt(i);
final List<MediaCodecInfo> codecs = selectCodecs(mimeType);
return codecs.isEmpty() ? null : codecs.get(0);
}
if (!codecInfo.isEncoder()) {
continue;
}
/**
* Returns all codecs capable of encoding the specified MIME type, ordered with hardware-preferred
* codecs ({@link MediaCodecList#REGULAR_CODECS}) first, then additional codecs from
* {@link MediaCodecList#ALL_CODECS} (includes software), deduplicated by name.
*/
static List<MediaCodecInfo> selectCodecs(final String mimeType) {
final List<MediaCodecInfo> candidates = new ArrayList<>();
final Set<String> seen = new HashSet<>();
final String[] types = codecInfo.getSupportedTypes();
for (String type : types) {
if (type.equalsIgnoreCase(mimeType)) {
return codecInfo;
for (MediaCodecInfo codecInfo : new MediaCodecList(MediaCodecList.REGULAR_CODECS).getCodecInfos()) {
if (!codecInfo.isEncoder()) continue;
for (String type : codecInfo.getSupportedTypes()) {
if (type.equalsIgnoreCase(mimeType) && seen.add(codecInfo.getName())) {
candidates.add(codecInfo);
}
}
}
return null;
for (MediaCodecInfo codecInfo : new MediaCodecList(MediaCodecList.ALL_CODECS).getCodecInfos()) {
if (!codecInfo.isEncoder()) continue;
for (String type : codecInfo.getSupportedTypes()) {
if (type.equalsIgnoreCase(mimeType) && seen.add(codecInfo.getName())) {
candidates.add(codecInfo);
}
}
}
return candidates;
}
interface Output {

147
lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/VideoTrackConverter.java
View File

@@ -5,7 +5,6 @@ import android.media.MediaCodecInfo;
import android.media.MediaExtractor;
import android.media.MediaFormat;
import android.os.Build;
import android.os.Bundle;
import android.view.Surface;
import androidx.annotation.NonNull;
@@ -21,8 +20,7 @@ import org.thoughtcrime.securesms.video.videoconverter.utils.Preconditions;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.concurrent.atomic.AtomicReference;
import java.util.List;
import kotlin.Pair;
final class VideoTrackConverter {
@@ -40,8 +38,6 @@ final class VideoTrackConverter {
private static final float FRAME_RATE_TOLERANCE = 0.05f; // tolerance for transcoding VFR -> CFR
private static final String VENDOR_DOLBY_CODEC_TRANSFER_PARAMKEY = "vendor.dolby.codec.transfer.value";
private final long mTimeFrom;
private final long mTimeTo;
@@ -106,13 +102,13 @@ final class VideoTrackConverter {
mTimeTo = timeTo;
mVideoExtractor = videoExtractor;
final MediaCodecInfo videoCodecInfo = MediaConverter.selectCodec(videoCodec);
if (videoCodecInfo == null) {
final List<MediaCodecInfo> videoCodecCandidates = MediaConverter.selectCodecs(videoCodec);
if (videoCodecCandidates.isEmpty()) {
// Don't fail CTS if they don't have an AVC codec (not here, anyway).
Log.e(TAG, "Unable to find an appropriate codec for " + videoCodec);
throw new FileNotFoundException();
}
if (VERBOSE) Log.d(TAG, "video found codec: " + videoCodecInfo.getName());
if (VERBOSE) Log.d(TAG, "video found codecs: " + videoCodecCandidates.size());
final MediaFormat inputVideoFormat = mVideoExtractor.getTrackFormat(videoInputTrack);
@@ -161,21 +157,20 @@ final class VideoTrackConverter {
outputVideoFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, OUTPUT_VIDEO_IFRAME_INTERVAL);
if (VERBOSE) Log.d(TAG, "video format: " + outputVideoFormat);
// Create a MediaCodec for the desired codec, then configure it as an encoder with
// our desired properties. Request a Surface to use for input.
final AtomicReference<Surface> inputSurfaceReference = new AtomicReference<>();
mVideoEncoder = createVideoEncoder(videoCodecInfo, outputVideoFormat, inputSurfaceReference);
mInputSurface = new InputSurface(inputSurfaceReference.get());
mInputSurface.makeCurrent();
// Create a MediaCodec for the decoder, based on the extractor's format.
mOutputSurface = new OutputSurface();
mOutputSurface.changeFragmentShader(createFragmentShader(
final String fragmentShader = createFragmentShader(
inputVideoFormat.getInteger(MediaFormat.KEY_WIDTH), inputVideoFormat.getInteger(MediaFormat.KEY_HEIGHT),
outputWidth, outputHeight));
outputWidth, outputHeight);
// Configure the encoder but do NOT start it yet. The encoder's start() is
// deferred until after the decoder is created, so that the decoder gets first
// access to hardware codec resources on memory-constrained devices.
mVideoEncoder = createVideoEncoder(videoCodecCandidates, outputVideoFormat);
mInputSurface = new InputSurface(mVideoEncoder.createInputSurface());
mInputSurface.makeCurrent();
mOutputSurface = new OutputSurface();
mOutputSurface.changeFragmentShader(fragmentShader);
mVideoDecoder = createVideoDecoder(inputVideoFormat, mOutputSurface.getSurface());
mVideoEncoder.start();
mVideoDecoderInputBuffers = mVideoDecoder.getInputBuffers();
mVideoEncoderOutputBuffers = mVideoEncoder.getOutputBuffers();
mVideoDecoderOutputBufferInfo = new MediaCodec.BufferInfo();
@@ -477,62 +472,78 @@ final class VideoTrackConverter {
private @NonNull
MediaCodec createVideoDecoder(
final @NonNull MediaFormat inputFormat,
final @NonNull Surface surface) {
final Pair<MediaCodec, MediaFormat> decoderPair = MediaCodecCompat.findDecoder(inputFormat);
final MediaCodec decoder = decoderPair.getFirst();
final @NonNull Surface surface) throws IOException {
final boolean isHdr = MediaCodecCompat.isHdrVideo(inputFormat);
final List<Pair<String, MediaFormat>> candidates = MediaCodecCompat.findDecoderCandidates(inputFormat);
Exception lastException = null;
// For HDR video, request SDR tone-mapping from the decoder. Only do this for HDR content
// (PQ or HLG transfer), as some hardware decoders (e.g. Qualcomm HEVC) crash when this is
// set on non-HDR video.
final boolean isHdr = MediaCodecCompat.isHdrVideo(decoderPair.getSecond());
if (Build.VERSION.SDK_INT >= 31 && isHdr) {
decoderPair.getSecond().setInteger(MediaFormat.KEY_COLOR_TRANSFER_REQUEST, MediaFormat.COLOR_TRANSFER_SDR_VIDEO);
}
decoder.configure(decoderPair.getSecond(), surface, null, 0);
decoder.start();
if (Build.VERSION.SDK_INT >= 31 && isHdr) {
try {
MediaCodec.ParameterDescriptor descriptor = decoder.getParameterDescriptor(VENDOR_DOLBY_CODEC_TRANSFER_PARAMKEY);
if (descriptor != null) {
Bundle transferBundle = new Bundle();
transferBundle.putString(VENDOR_DOLBY_CODEC_TRANSFER_PARAMKEY, "transfer.sdr.normal");
decoder.setParameters(transferBundle);
for (int i = 0; i < candidates.size(); i++) {
final Pair<String, MediaFormat> candidate = candidates.get(i);
final String codecName = candidate.getFirst();
final MediaFormat decoderFormat = candidate.getSecond();
MediaCodec decoder = null;
try {
decoder = MediaCodec.createByCodecName(codecName);
// For HDR video, request SDR tone-mapping from the decoder (API 31+).
if (Build.VERSION.SDK_INT >= 31 && isHdr) {
decoderFormat.setInteger(MediaFormat.KEY_COLOR_TRANSFER_REQUEST, MediaFormat.COLOR_TRANSFER_SDR_VIDEO);
}
decoder.configure(decoderFormat, surface, null, 0);
decoder.start();
if (i > 0) {
Log.w(TAG, "Video decoder: succeeded with fallback codec " + codecName + " (attempt " + (i + 1) + " of " + candidates.size() + ")");
}
return decoder;
} catch (IllegalStateException e) {
Log.w(TAG, "Video decoder: codec " + codecName + " failed (attempt " + (i + 1) + " of " + candidates.size() + ")", e);
lastException = e;
if (decoder != null) {
decoder.release();
}
} catch (IOException e) {
Log.w(TAG, "Video decoder: codec " + codecName + " failed to create (attempt " + (i + 1) + " of " + candidates.size() + ")", e);
lastException = e;
}
} catch (IllegalStateException e) {
Log.w(TAG, "Failed to set Dolby Vision transfer parameter", e);
}
}
return decoder;
throw new IOException("All video decoder codecs failed", lastException);
}
private @NonNull
/**
* Creates and configures a video encoder but does NOT start it. The caller must call
* {@link MediaCodec#createInputSurface()} (between configure and start) and then
* {@link MediaCodec#start()} after the decoder has been created.
*/
private static @NonNull
MediaCodec createVideoEncoder(
final @NonNull MediaCodecInfo codecInfo,
final @NonNull MediaFormat format,
final @NonNull AtomicReference<Surface> surfaceReference) throws IOException {
boolean tonemapRequested = isTonemapEnabled(format);
final MediaCodec encoder = MediaCodec.createByCodecName(codecInfo.getName());
encoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
if (tonemapRequested && !isTonemapEnabled(format)) {
Log.d(TAG, "HDR tone-mapping requested but not supported by the decoder.");
}
// Must be called before start()
surfaceReference.set(encoder.createInputSurface());
encoder.start();
return encoder;
}
final @NonNull List<MediaCodecInfo> codecCandidates,
final @NonNull MediaFormat format) throws IOException {
Exception lastException = null;
private static boolean isTonemapEnabled(@NonNull MediaFormat format) {
if (Build.VERSION.SDK_INT < 31) {
return false;
}
try {
int request = format.getInteger(MediaFormat.KEY_COLOR_TRANSFER_REQUEST);
return request == MediaFormat.COLOR_TRANSFER_SDR_VIDEO;
} catch (NullPointerException npe) {
// transfer request key does not exist, tone mapping not requested
return false;
for (int i = 0; i < codecCandidates.size(); i++) {
final MediaCodecInfo codecInfo = codecCandidates.get(i);
MediaCodec encoder = null;
try {
encoder = MediaCodec.createByCodecName(codecInfo.getName());
encoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
if (i > 0) {
Log.w(TAG, "Video encoder: succeeded with fallback codec " + codecInfo.getName() + " (attempt " + (i + 1) + " of " + codecCandidates.size() + ")");
}
return encoder;
} catch (IllegalStateException e) {
Log.w(TAG, "Video encoder: codec " + codecInfo.getName() + " failed (attempt " + (i + 1) + " of " + codecCandidates.size() + ")", e);
lastException = e;
if (encoder != null) {
encoder.release();
}
}
}
throw new IOException("All video encoder codecs failed", lastException);
}
private static int getAndSelectVideoTrackIndex(@NonNull MediaExtractor extractor) {

119
lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/utils/MediaCodecCompat.kt
View File

@@ -23,6 +23,57 @@ object MediaCodecCompat {
const val MEDIA_FORMAT_KEY_CODEC_SPECIFIC_DATA_1 = "csd-1"
const val MEDIA_FORMAT_KEY_CODEC_SPECIFIC_DATA_2 = "csd-2"
/**
* Returns all matching decoders for the given [inputFormat], ordered with hardware-preferred
* codecs ([MediaCodecList.REGULAR_CODECS]) first, then additional codecs from
* [MediaCodecList.ALL_CODECS] (includes software), deduplicated by name.
*
* Each entry is a codec name paired with the format to use (which may differ from [inputFormat]
* for Dolby Vision content, where it is resolved to the base-layer format).
*/
@JvmStatic
@Throws(IOException::class)
fun findDecoderCandidates(inputFormat: MediaFormat): List<Pair<String, MediaFormat>> {
val mimeType = inputFormat.getString(MediaFormat.KEY_MIME)
val resolvedFormat: MediaFormat = if (MediaFormat.MIMETYPE_VIDEO_DOLBY_VISION == mimeType) {
val dvFormat = if (Build.VERSION.SDK_INT >= 29) MediaFormat(inputFormat) else inputFormat
resolveDolbyVisionFormat(dvFormat) ?: throw IOException("Can't resolve Dolby Vision format for decoder candidates!")
} else {
inputFormat
}
val resolvedMime = resolvedFormat.getString(MediaFormat.KEY_MIME)
?: throw IOException("No MIME type in resolved format!")
val candidates = mutableListOf<Pair<String, MediaFormat>>()
val seen = mutableSetOf<String>()
for (codecInfo in MediaCodecList(MediaCodecList.REGULAR_CODECS).codecInfos) {
if (codecInfo.isEncoder) continue
if (codecInfo.supportedTypes.any { it.equals(resolvedMime, ignoreCase = true) }) {
if (seen.add(codecInfo.name)) {
candidates.add(Pair(codecInfo.name, resolvedFormat))
}
}
}
for (codecInfo in MediaCodecList(MediaCodecList.ALL_CODECS).codecInfos) {
if (codecInfo.isEncoder) continue
if (codecInfo.supportedTypes.any { it.equals(resolvedMime, ignoreCase = true) }) {
if (seen.add(codecInfo.name)) {
candidates.add(Pair(codecInfo.name, resolvedFormat))
}
}
}
if (candidates.isEmpty()) {
throw IOException("Can't find decoder for $resolvedMime!")
}
return candidates
}
@JvmStatic
fun findDecoder(inputFormat: MediaFormat): Pair<MediaCodec, MediaFormat> {
val codecs = MediaCodecList(MediaCodecList.REGULAR_CODECS)
@@ -50,6 +101,39 @@ object MediaCodecCompat {
throw IOException("Can't create decoder for $mimeType!")
}
/**
* Resolves a Dolby Vision [MediaFormat] to its base-layer format by mutating the MIME type,
* profile, and level. Returns the mutated format, or null if the DV profile is unsupported.
*/
private fun resolveDolbyVisionFormat(mediaFormat: MediaFormat): MediaFormat? {
if (MediaFormat.MIMETYPE_VIDEO_DOLBY_VISION != mediaFormat.getString(MediaFormat.KEY_MIME)) {
throw IllegalStateException("Must supply Dolby Vision MediaFormat!")
}
return try {
when (mediaFormat.getInteger(MediaFormat.KEY_PROFILE)) {
CodecProfileLevel.DolbyVisionProfileDvheDtr,
CodecProfileLevel.DolbyVisionProfileDvheSt -> {
mediaFormat.setString(MediaFormat.KEY_MIME, MediaFormat.MIMETYPE_VIDEO_HEVC)
mediaFormat.setInteger(MediaFormat.KEY_PROFILE, CodecProfileLevel.HEVCProfileMain10)
mediaFormat.setBaseCodecLevelFromDolbyVisionLevel()
mediaFormat
}
CodecProfileLevel.DolbyVisionProfileDvavSe -> {
mediaFormat.setString(MediaFormat.KEY_MIME, MediaFormat.MIMETYPE_VIDEO_AVC)
mediaFormat.setInteger(MediaFormat.KEY_PROFILE, CodecProfileLevel.AVCProfileHigh)
mediaFormat.setBaseCodecLevelFromDolbyVisionLevel()
mediaFormat
}
else -> null
}
} catch (npe: NullPointerException) {
null
}
}
/**
* Find backup decoder for a [MediaFormat] object with a MIME type of Dolby Vision.
*
@@ -151,18 +235,37 @@ object MediaCodecCompat {
}
/**
* Returns true if the given [MediaFormat] describes an HDR video (PQ or HLG color transfer).
* Some hardware decoders crash when tone-mapping parameters are set on non-HDR video.
* Returns true if the given [MediaFormat] describes an HDR video.
*
* Checks three signals:
* 1. PQ or HLG color transfer function (standard indicator)
* 2. HDR static metadata (mastering display info, present in HDR10 content)
* 3. HDR10+ dynamic metadata (present in HDR10+ content, API 29+)
*
* Some devices report non-standard color transfer values (e.g. 65791) for HDR content,
* so checking metadata keys provides a more reliable detection.
*/
@JvmStatic
fun isHdrVideo(format: MediaFormat): Boolean {
return try {
try {
val colorTransfer = format.getInteger(MediaFormat.KEY_COLOR_TRANSFER)
colorTransfer == MediaFormat.COLOR_TRANSFER_ST2084 || colorTransfer == MediaFormat.COLOR_TRANSFER_HLG
} catch (e: NullPointerException) {
false
} catch (e: ClassCastException) {
false
if (colorTransfer == MediaFormat.COLOR_TRANSFER_ST2084 || colorTransfer == MediaFormat.COLOR_TRANSFER_HLG) {
return true
}
} catch (_: NullPointerException) {
// key doesn't exist
} catch (_: ClassCastException) {
// key exists but wrong type
}
if (format.containsKey(MediaFormat.KEY_HDR_STATIC_INFO)) {
return true
}
if (Build.VERSION.SDK_INT >= 29 && format.containsKey(MediaFormat.KEY_HDR10_PLUS_INFO)) {
return true
}
return false
}
}