Signal/Android
1
0
Fork 0
mirror of https://github.com/signalapp/Signal-Android.git synced 2026-04-22 01:40:07 +01:00
Code Issues Packages Projects Releases Wiki Activity

StreamingTranscoder sample app.

Browse Source
This commit is contained in:
Nicholas Tinsley
2024-01-12 18:18:01 -05:00
committed by Greyson Parrelli
parent 750fd4efe1
commit c7609f9a2a
46 changed files with 880 additions and 254 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
app/src/main/java/org/thoughtcrime/securesms/audio/AudioWaveFormGenerator.java
View File

@@ -12,7 +12,7 @@ import androidx.annotation.WorkerThread;
import org.signal.core.util.logging.Log;
import org.thoughtcrime.securesms.media.DecryptableUriMediaInput;
import org.thoughtcrime.securesms.media.MediaInput;
import org.thoughtcrime.securesms.video.videoconverter.MediaInput;
import java.io.IOException;
import java.nio.ByteBuffer;

2
app/src/main/java/org/thoughtcrime/securesms/jobs/AttachmentCompressionJob.java
View File

@@ -47,7 +47,7 @@ import org.thoughtcrime.securesms.video.InMemoryTranscoder;
import org.thoughtcrime.securesms.video.StreamingTranscoder;
import org.thoughtcrime.securesms.video.TranscoderCancelationSignal;
import org.thoughtcrime.securesms.video.TranscoderOptions;
import org.thoughtcrime.securesms.video.VideoSourceException;
import org.thoughtcrime.securesms.video.exceptions.VideoSourceException;
import org.thoughtcrime.securesms.video.videoconverter.EncodingException;
import java.io.ByteArrayInputStream;

52
app/src/main/java/org/thoughtcrime/securesms/media/DecryptableUriMediaInput.java
View File

@@ -1,52 +0,0 @@
package org.thoughtcrime.securesms.media;
import android.content.Context;
import android.media.MediaDataSource;
import android.net.Uri;
import androidx.annotation.NonNull;
import androidx.annotation.RequiresApi;
import org.thoughtcrime.securesms.attachments.AttachmentId;
import org.thoughtcrime.securesms.database.SignalDatabase;
import org.thoughtcrime.securesms.mms.PartAuthority;
import org.thoughtcrime.securesms.mms.PartUriParser;
import org.thoughtcrime.securesms.providers.BlobProvider;
import java.io.IOException;
@RequiresApi(api = 23)
public final class DecryptableUriMediaInput {
private DecryptableUriMediaInput() {
}
public static @NonNull MediaInput createForUri(@NonNull Context context, @NonNull Uri uri) throws IOException {
if (BlobProvider.isAuthority(uri)) {
return new MediaInput.MediaDataSourceMediaInput(BlobProvider.getInstance().getMediaDataSource(context, uri));
}
if (PartAuthority.isLocalUri(uri)) {
return createForAttachmentUri(context, uri);
}
return new MediaInput.UriMediaInput(context, uri);
}
private static @NonNull MediaInput createForAttachmentUri(@NonNull Context context, @NonNull Uri uri) {
AttachmentId partId = new PartUriParser(uri).getPartId();
if (!partId.isValid()) {
throw new AssertionError();
}
MediaDataSource mediaDataSource = SignalDatabase.attachments().mediaDataSourceFor(partId, true);
if (mediaDataSource == null) {
throw new AssertionError();
}
return new MediaInput.MediaDataSourceMediaInput(mediaDataSource);
}
}

40
app/src/main/java/org/thoughtcrime/securesms/media/DecryptableUriMediaInput.kt Normal file
View File

@@ -0,0 +1,40 @@
package org.thoughtcrime.securesms.media
import android.content.Context
import android.net.Uri
import androidx.annotation.RequiresApi
import org.thoughtcrime.securesms.database.SignalDatabase.Companion.attachments
import org.thoughtcrime.securesms.mms.PartAuthority
import org.thoughtcrime.securesms.mms.PartUriParser
import org.thoughtcrime.securesms.providers.BlobProvider
import org.thoughtcrime.securesms.video.videoconverter.MediaInput
import org.thoughtcrime.securesms.video.videoconverter.mediadatasource.MediaDataSourceMediaInput
import java.io.IOException
/**
* A media input source that is decrypted on the fly.
*/
@RequiresApi(api = 23)
object DecryptableUriMediaInput {
@JvmStatic
@Throws(IOException::class)
fun createForUri(context: Context, uri: Uri): MediaInput {
if (BlobProvider.isAuthority(uri)) {
return MediaDataSourceMediaInput(BlobProvider.getInstance().getMediaDataSource(context, uri))
}
return if (PartAuthority.isLocalUri(uri)) {
createForAttachmentUri(uri)
} else {
UriMediaInput(context, uri)
}
}
private fun createForAttachmentUri(uri: Uri): MediaInput {
val partId = PartUriParser(uri).partId
if (!partId.isValid) {
throw AssertionError()
}
val mediaDataSource = attachments.mediaDataSourceFor(partId, true) ?: throw AssertionError()
return MediaDataSourceMediaInput(mediaDataSource)
}
}

24
app/src/main/java/org/thoughtcrime/securesms/media/FileMediaInput.kt Normal file
View File

@@ -0,0 +1,24 @@
/*
* Copyright 2024 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.thoughtcrime.securesms.media
import android.media.MediaExtractor
import org.thoughtcrime.securesms.video.videoconverter.MediaInput
import java.io.File
import java.io.IOException
/**
* A media input source that the system reads directly from the file.
*/
class FileMediaInput(private val file: File) : MediaInput {
@Throws(IOException::class)
override fun createExtractor(): MediaExtractor {
val extractor = MediaExtractor()
extractor.setDataSource(file.absolutePath)
return extractor
}
override fun close() {}
}

83
app/src/main/java/org/thoughtcrime/securesms/media/MediaInput.java
View File

@@ -1,83 +0,0 @@
package org.thoughtcrime.securesms.media;
import android.content.Context;
import android.media.MediaDataSource;
import android.media.MediaExtractor;
import android.net.Uri;
import androidx.annotation.NonNull;
import androidx.annotation.RequiresApi;
import java.io.Closeable;
import java.io.File;
import java.io.IOException;
public abstract class MediaInput implements Closeable {
@NonNull
public abstract MediaExtractor createExtractor() throws IOException;
public static class FileMediaInput extends MediaInput {
private final File file;
public FileMediaInput(@NonNull File file) {
this.file = file;
}
@Override
public @NonNull MediaExtractor createExtractor() throws IOException {
final MediaExtractor extractor = new MediaExtractor();
extractor.setDataSource(file.getAbsolutePath());
return extractor;
}
@Override
public void close() {
}
}
public static class UriMediaInput extends MediaInput {
private final Uri uri;
private final Context context;
public UriMediaInput(@NonNull Context context, @NonNull Uri uri) {
this.uri = uri;
this.context = context;
}
@Override
public @NonNull MediaExtractor createExtractor() throws IOException {
final MediaExtractor extractor = new MediaExtractor();
extractor.setDataSource(context, uri, null);
return extractor;
}
@Override
public void close() {
}
}
@RequiresApi(23)
public static class MediaDataSourceMediaInput extends MediaInput {
private final MediaDataSource mediaDataSource;
public MediaDataSourceMediaInput(@NonNull MediaDataSource mediaDataSource) {
this.mediaDataSource = mediaDataSource;
}
@Override
public @NonNull MediaExtractor createExtractor() throws IOException {
final MediaExtractor extractor = new MediaExtractor();
extractor.setDataSource(mediaDataSource);
return extractor;
}
@Override
public void close() throws IOException {
mediaDataSource.close();
}
}
}

25
app/src/main/java/org/thoughtcrime/securesms/media/UriMediaInput.kt Normal file
View File

@@ -0,0 +1,25 @@
/*
* Copyright 2024 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.thoughtcrime.securesms.media
import android.content.Context
import android.media.MediaExtractor
import android.net.Uri
import org.thoughtcrime.securesms.video.videoconverter.MediaInput
import java.io.IOException
/**
* A media input source defined by a [Uri] that the system can parse and access.
*/
class UriMediaInput(private val context: Context, private val uri: Uri) : MediaInput {
@Throws(IOException::class)
override fun createExtractor(): MediaExtractor {
val extractor = MediaExtractor()
extractor.setDataSource(context, uri, null)
return extractor
}
override fun close() {}
}

4
app/src/main/java/org/thoughtcrime/securesms/mediasend/v2/capture/MediaCaptureRepository.kt
View File

@@ -15,7 +15,7 @@ import org.thoughtcrime.securesms.mediasend.MediaRepository
import org.thoughtcrime.securesms.providers.BlobProvider
import org.thoughtcrime.securesms.util.MediaUtil
import org.thoughtcrime.securesms.util.StorageUtil
import org.thoughtcrime.securesms.video.VideoUtil
import org.thoughtcrime.securesms.video.videoconverter.VideoConstants
import java.io.FileDescriptor
import java.io.FileInputStream
import java.io.IOException
@@ -66,7 +66,7 @@ class MediaCaptureRepository(context: Context) {
dataSupplier = { FileInputStream(fileDescriptor) },
getLength = { it.channel.size() },
createBlobBuilder = BlobProvider::forData,
mimeType = VideoUtil.RECORDED_VIDEO_CONTENT_TYPE,
mimeType = VideoConstants.RECORDED_VIDEO_CONTENT_TYPE,
width = 0,
height = 0
)

6
app/src/main/java/org/thoughtcrime/securesms/video/InMemoryTranscoder.java
View File

@@ -15,11 +15,13 @@ import org.signal.core.util.logging.Log;
import org.signal.libsignal.media.Mp4Sanitizer;
import org.signal.libsignal.media.ParseException;
import org.signal.libsignal.media.SanitizedMetadata;
import org.thoughtcrime.securesms.media.MediaInput;
import org.thoughtcrime.securesms.mms.MediaStream;
import org.thoughtcrime.securesms.util.MemoryFileDescriptor;
import org.thoughtcrime.securesms.video.exceptions.VideoSizeException;
import org.thoughtcrime.securesms.video.exceptions.VideoSourceException;
import org.thoughtcrime.securesms.video.videoconverter.EncodingException;
import org.thoughtcrime.securesms.video.videoconverter.MediaConverter;
import org.thoughtcrime.securesms.video.videoconverter.mediadatasource.MediaDataSourceMediaInput;
import java.io.ByteArrayInputStream;
import java.io.Closeable;
@@ -122,7 +124,7 @@ public final class InMemoryTranscoder implements Closeable {
final MediaConverter converter = new MediaConverter();
converter.setInput(new MediaInput.MediaDataSourceMediaInput(dataSource));
converter.setInput(new MediaDataSourceMediaInput(dataSource));
converter.setOutput(memoryFileFileDescriptor);
converter.setVideoResolution(targetQuality.getOutputResolution());
converter.setVideoBitrate(targetQuality.getTargetVideoBitRate());

38
app/src/main/java/org/thoughtcrime/securesms/video/ModernEncryptedMediaDataSource.java
View File

@@ -8,6 +8,7 @@ import androidx.annotation.RequiresApi;
import org.thoughtcrime.securesms.crypto.AttachmentSecret;
import org.thoughtcrime.securesms.crypto.ModernDecryptingPartInputStream;
import org.thoughtcrime.securesms.video.videoconverter.mediadatasource.InputStreamMediaDataSource;
import java.io.File;
import java.io.IOException;
@@ -22,7 +23,7 @@ import java.io.InputStream;
* the presence of a random part of the key supplied in the constructor.
*/
@RequiresApi(23)
final class ModernEncryptedMediaDataSource extends MediaDataSource {
final class ModernEncryptedMediaDataSource extends InputStreamMediaDataSource {
private final AttachmentSecret attachmentSecret;
private final File mediaFile;
@@ -37,44 +38,15 @@ final class ModernEncryptedMediaDataSource extends MediaDataSource {
}
@Override
public int readAt(long position, byte[] bytes, int offset, int length) throws IOException {
if (position >= this.length) {
return -1;
}
try (InputStream inputStream = createInputStream(position)) {
int totalRead = 0;
while (length > 0) {
int read = inputStream.read(bytes, offset, length);
if (read == -1) {
if (totalRead == 0) {
return -1;
} else {
return totalRead;
}
}
length -= read;
offset += read;
totalRead += read;
}
return totalRead;
}
}
public void close() {}
@Override
public long getSize() {
return length;
}
@Override
public void close() {
}
private InputStream createInputStream(long position) throws IOException {
@NonNull
public InputStream createInputStream(long position) throws IOException {
if (random == null) {
return ModernDecryptingPartInputStream.createFor(attachmentSecret, mediaFile, position);
} else {

210
app/src/main/java/org/thoughtcrime/securesms/video/StreamingTranscoder.java
View File

@@ -1,210 +0,0 @@
package org.thoughtcrime.securesms.video;
import android.media.MediaDataSource;
import android.media.MediaMetadataRetriever;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import androidx.annotation.RequiresApi;
import org.signal.core.util.logging.Log;
import org.thoughtcrime.securesms.media.MediaInput;
import org.thoughtcrime.securesms.video.videoconverter.EncodingException;
import org.thoughtcrime.securesms.video.videoconverter.MediaConverter;
import java.io.FilterOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.text.NumberFormat;
import java.util.Locale;
@RequiresApi(26)
public final class StreamingTranscoder {
private static final String TAG = Log.tag(StreamingTranscoder.class);
private final MediaDataSource dataSource;
private final long upperSizeLimit;
private final long inSize;
private final long duration;
private final int inputBitRate;
private final VideoBitRateCalculator.Quality targetQuality;
private final boolean transcodeRequired;
private final long fileSizeEstimate;
private final @Nullable TranscoderOptions options;
/**
* @param upperSizeLimit A upper size to transcode to. The actual output size can be up to 10% smaller.
*/
public StreamingTranscoder(@NonNull MediaDataSource dataSource,
@Nullable TranscoderOptions options,
long upperSizeLimit)
throws IOException, VideoSourceException
{
this.dataSource = dataSource;
this.options = options;
final MediaMetadataRetriever mediaMetadataRetriever = new MediaMetadataRetriever();
try {
mediaMetadataRetriever.setDataSource(dataSource);
} catch (RuntimeException e) {
Log.w(TAG, "Unable to read datasource", e);
throw new VideoSourceException("Unable to read datasource", e);
}
this.inSize = dataSource.getSize();
this.duration = getDuration(mediaMetadataRetriever);
this.inputBitRate = VideoBitRateCalculator.bitRate(inSize, duration);
this.targetQuality = new VideoBitRateCalculator(upperSizeLimit).getTargetQuality(duration, inputBitRate);
this.upperSizeLimit = upperSizeLimit;
this.transcodeRequired = inputBitRate >= targetQuality.getTargetTotalBitRate() * 1.2 || inSize > upperSizeLimit || containsLocation(mediaMetadataRetriever) || options != null;
if (!transcodeRequired) {
Log.i(TAG, "Video is within 20% of target bitrate, below the size limit, contained no location metadata or custom options.");
}
this.fileSizeEstimate = targetQuality.getFileSizeEstimate();
}
public void transcode(@NonNull Progress progress,
@NonNull OutputStream stream,
@Nullable TranscoderCancelationSignal cancelationSignal)
throws IOException, EncodingException
{
float durationSec = duration / 1000f;
NumberFormat numberFormat = NumberFormat.getInstance(Locale.US);
Log.i(TAG, String.format(Locale.US,
"Transcoding:\n" +
"Target bitrate : %s + %s = %s\n" +
"Target format : %dp\n" +
"Video duration : %.1fs\n" +
"Size limit : %s kB\n" +
"Estimate : %s kB\n" +
"Input size : %s kB\n" +
"Input bitrate : %s bps",
numberFormat.format(targetQuality.getTargetVideoBitRate()),
numberFormat.format(targetQuality.getTargetAudioBitRate()),
numberFormat.format(targetQuality.getTargetTotalBitRate()),
targetQuality.getOutputResolution(),
durationSec,
numberFormat.format(upperSizeLimit / 1024),
numberFormat.format(fileSizeEstimate / 1024),
numberFormat.format(inSize / 1024),
numberFormat.format(inputBitRate)));
if (fileSizeEstimate > upperSizeLimit) {
throw new VideoSizeException("Size constraints could not be met!");
}
final long startTime = System.currentTimeMillis();
final MediaConverter converter = new MediaConverter();
final LimitedSizeOutputStream limitedSizeOutputStream = new LimitedSizeOutputStream(stream, upperSizeLimit);
converter.setInput(new MediaInput.MediaDataSourceMediaInput(dataSource));
converter.setOutput(limitedSizeOutputStream);
converter.setVideoResolution(targetQuality.getOutputResolution());
converter.setVideoBitrate(targetQuality.getTargetVideoBitRate());
converter.setAudioBitrate(targetQuality.getTargetAudioBitRate());
if (options != null) {
if (options.endTimeUs > 0) {
long timeFrom = options.startTimeUs / 1000;
long timeTo = options.endTimeUs / 1000;
converter.setTimeRange(timeFrom, timeTo);
Log.i(TAG, String.format(Locale.US, "Trimming:\nTotal duration: %d\nKeeping: %d..%d\nFinal duration:(%d)", duration, timeFrom, timeTo, timeTo - timeFrom));
}
}
converter.setListener(percent -> {
progress.onProgress(percent);
return cancelationSignal != null && cancelationSignal.isCanceled();
});
converter.convert();
long outSize = limitedSizeOutputStream.written;
float encodeDurationSec = (System.currentTimeMillis() - startTime) / 1000f;
Log.i(TAG, String.format(Locale.US,
"Transcoding complete:\n" +
"Transcode time : %.1fs (%.1fx)\n" +
"Output size : %s kB\n" +
" of Original : %.1f%%\n" +
" of Estimate : %.1f%%\n" +
"Output bitrate : %s bps",
encodeDurationSec,
durationSec / encodeDurationSec,
numberFormat.format(outSize / 1024),
(outSize * 100d) / inSize,
(outSize * 100d) / fileSizeEstimate,
numberFormat.format(VideoBitRateCalculator.bitRate(outSize, duration))));
if (outSize > upperSizeLimit) {
throw new VideoSizeException("Size constraints could not be met!");
}
stream.flush();
}
public boolean isTranscodeRequired() {
return transcodeRequired;
}
private static long getDuration(MediaMetadataRetriever mediaMetadataRetriever) throws VideoSourceException {
String durationString = mediaMetadataRetriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_DURATION);
if (durationString == null) {
throw new VideoSourceException("Cannot determine duration of video, null meta data");
}
try {
long duration = Long.parseLong(durationString);
if (duration <= 0) {
throw new VideoSourceException("Cannot determine duration of video, meta data: " + durationString);
}
return duration;
} catch (NumberFormatException e) {
throw new VideoSourceException("Cannot determine duration of video, meta data: " + durationString, e);
}
}
private static boolean containsLocation(MediaMetadataRetriever mediaMetadataRetriever) {
String locationString = mediaMetadataRetriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_LOCATION);
return locationString != null;
}
public interface Progress {
void onProgress(int percent);
}
private static class LimitedSizeOutputStream extends FilterOutputStream {
private final long sizeLimit;
private long written;
LimitedSizeOutputStream(@NonNull OutputStream inner, long sizeLimit) {
super(inner);
this.sizeLimit = sizeLimit;
}
@Override public void write(int b) throws IOException {
incWritten(1);
out.write(b);
}
@Override public void write(byte[] b, int off, int len) throws IOException {
incWritten(len);
out.write(b, off, len);
}
private void incWritten(int len) throws IOException {
long newWritten = written + len;
if (newWritten > sizeLimit) {
Log.w(TAG, String.format(Locale.US, "File size limit hit. Wrote %d, tried to write %d more. Limit is %d", written, len, sizeLimit));
throw new VideoSizeException("File size limit hit");
}
written = newWritten;
}
}
}

5
app/src/main/java/org/thoughtcrime/securesms/video/TranscoderCancelationSignal.java
View File

@@ -1,5 +0,0 @@
package org.thoughtcrime.securesms.video;
public interface TranscoderCancelationSignal {
boolean isCanceled();
}

11
app/src/main/java/org/thoughtcrime/securesms/video/TranscoderOptions.java
View File

@@ -1,11 +0,0 @@
package org.thoughtcrime.securesms.video;
public final class TranscoderOptions {
final long startTimeUs;
final long endTimeUs;
public TranscoderOptions(long startTimeUs, long endTimeUs) {
this.startTimeUs = startTimeUs;
this.endTimeUs = endTimeUs;
}
}

108
app/src/main/java/org/thoughtcrime/securesms/video/VideoBitRateCalculator.java
View File

@@ -1,108 +0,0 @@
package org.thoughtcrime.securesms.video;
/**
* Calculates a target quality output for a video to fit within a specified size.
*/
public final class VideoBitRateCalculator {
private static final int MAXIMUM_TARGET_VIDEO_BITRATE = VideoUtil.VIDEO_BIT_RATE;
private static final int LOW_RES_TARGET_VIDEO_BITRATE = 1_750_000;
private static final int MINIMUM_TARGET_VIDEO_BITRATE = 500_000;
private static final int AUDIO_BITRATE = VideoUtil.AUDIO_BIT_RATE;
private static final int OUTPUT_FORMAT = VideoUtil.VIDEO_SHORT_WIDTH;
private static final int LOW_RES_OUTPUT_FORMAT = 480;
private final long upperFileSizeLimitWithMargin;
public VideoBitRateCalculator(long upperFileSizeLimit) {
upperFileSizeLimitWithMargin = (long) (upperFileSizeLimit / 1.1);
}
/**
* Gets the output quality of a video of the given {@param duration}.
*/
public Quality getTargetQuality(long duration, int inputTotalBitRate) {
int maxVideoBitRate = Math.min(MAXIMUM_TARGET_VIDEO_BITRATE, inputTotalBitRate - AUDIO_BITRATE);
int minVideoBitRate = Math.min(MINIMUM_TARGET_VIDEO_BITRATE, maxVideoBitRate);
int targetVideoBitRate = Math.max(minVideoBitRate, Math.min(getTargetVideoBitRate(upperFileSizeLimitWithMargin, duration), maxVideoBitRate));
int bitRateRange = maxVideoBitRate - minVideoBitRate;
double quality = bitRateRange == 0 ? 1 : (targetVideoBitRate - minVideoBitRate) / (double) bitRateRange;
return new Quality(targetVideoBitRate, AUDIO_BITRATE, quality, duration);
}
private int getTargetVideoBitRate(long sizeGuideBytes, long duration) {
double durationSeconds = duration / 1000d;
sizeGuideBytes -= durationSeconds * AUDIO_BITRATE / 8;
double targetAttachmentSizeBits = sizeGuideBytes * 8L;
return (int) (targetAttachmentSizeBits / durationSeconds);
}
public static int bitRate(long bytes, long durationMs) {
return (int) (bytes * 8 / (durationMs / 1000f));
}
public static class Quality {
private final int targetVideoBitRate;
private final int targetAudioBitRate;
private final double quality;
private final long duration;
private Quality(int targetVideoBitRate, int targetAudioBitRate, double quality, long duration) {
this.targetVideoBitRate = targetVideoBitRate;
this.targetAudioBitRate = targetAudioBitRate;
this.quality = Math.max(0, Math.min(quality, 1));
this.duration = duration;
}
/**
* [0..1]
* <p>
* 0 = {@link #MINIMUM_TARGET_VIDEO_BITRATE}
* 1 = {@link #MAXIMUM_TARGET_VIDEO_BITRATE}
*/
public double getQuality() {
return quality;
}
public int getTargetVideoBitRate() {
return targetVideoBitRate;
}
public int getTargetAudioBitRate() {
return targetAudioBitRate;
}
public int getTargetTotalBitRate() {
return targetVideoBitRate + targetAudioBitRate;
}
public boolean useLowRes() {
return targetVideoBitRate < LOW_RES_TARGET_VIDEO_BITRATE;
}
public int getOutputResolution() {
return useLowRes() ? LOW_RES_OUTPUT_FORMAT
: OUTPUT_FORMAT;
}
public long getFileSizeEstimate() {
return getTargetTotalBitRate() * duration / 8000;
}
@Override
public String toString() {
return "Quality{" +
"targetVideoBitRate=" + targetVideoBitRate +
", targetAudioBitRate=" + targetAudioBitRate +
", quality=" + quality +
", duration=" + duration +
", filesize=" + getFileSizeEstimate() +
'}';
}
}
}

10
app/src/main/java/org/thoughtcrime/securesms/video/VideoSizeException.java
View File

@@ -1,10 +0,0 @@
package org.thoughtcrime.securesms.video;
import java.io.IOException;
public final class VideoSizeException extends IOException {
VideoSizeException(String message) {
super(message);
}
}

12
app/src/main/java/org/thoughtcrime/securesms/video/VideoSourceException.java
View File

@@ -1,12 +0,0 @@
package org.thoughtcrime.securesms.video;
public final class VideoSourceException extends Exception {
VideoSourceException(String message) {
super(message);
}
VideoSourceException(String message, Exception inner) {
super(message, inner);
}
}

28
app/src/main/java/org/thoughtcrime/securesms/video/VideoUtil.java
View File

@@ -10,45 +10,29 @@ import androidx.annotation.NonNull;
import org.thoughtcrime.securesms.mms.MediaConstraints;
import org.thoughtcrime.securesms.util.MediaUtil;
import org.thoughtcrime.securesms.video.videoconverter.VideoConstants;
import java.util.concurrent.TimeUnit;
public final class VideoUtil {
public static final int AUDIO_BIT_RATE = 192_000;
public static final int VIDEO_FRAME_RATE = 30;
public static final int VIDEO_BIT_RATE = 2_000_000;
static final int VIDEO_SHORT_WIDTH = 720;
private static final int VIDEO_LONG_WIDTH = 1280;
private static final int VIDEO_MAX_RECORD_LENGTH_S = 60;
private static final int VIDEO_MAX_UPLOAD_LENGTH_S = (int) TimeUnit.MINUTES.toSeconds(10);
private static final int TOTAL_BYTES_PER_SECOND = (VIDEO_BIT_RATE / 8) + (AUDIO_BIT_RATE / 8);
public static final String VIDEO_MIME_TYPE = MediaFormat.MIMETYPE_VIDEO_AVC;
public static final String AUDIO_MIME_TYPE = "audio/mp4a-latm";
public static final String RECORDED_VIDEO_CONTENT_TYPE = MediaUtil.VIDEO_MP4;
private VideoUtil() { }
public static Size getVideoRecordingSize() {
return isPortrait(screenSize())
? new Size(VIDEO_SHORT_WIDTH, VIDEO_LONG_WIDTH)
: new Size(VIDEO_LONG_WIDTH, VIDEO_SHORT_WIDTH);
? new Size(VideoConstants.VIDEO_SHORT_EDGE, VideoConstants.VIDEO_LONG_EDGE)
: new Size(VideoConstants.VIDEO_LONG_EDGE, VideoConstants.VIDEO_SHORT_EDGE);
}
public static int getMaxVideoRecordDurationInSeconds(@NonNull Context context, @NonNull MediaConstraints mediaConstraints) {
long allowedSize = mediaConstraints.getCompressedVideoMaxSize(context);
int duration = (int) Math.floor((float) allowedSize / TOTAL_BYTES_PER_SECOND);
int duration = (int) Math.floor((float) allowedSize / VideoConstants.TOTAL_BYTES_PER_SECOND);
return Math.min(duration, VIDEO_MAX_RECORD_LENGTH_S);
return Math.min(duration, VideoConstants.VIDEO_MAX_RECORD_LENGTH_S);
}
public static int getMaxVideoUploadDurationInSeconds() {
return VIDEO_MAX_UPLOAD_LENGTH_S;
return Math.toIntExact(TimeUnit.MINUTES.toSeconds(10));
}
private static Size screenSize() {

421
app/src/main/java/org/thoughtcrime/securesms/video/videoconverter/AudioTrackConverter.java
View File

@@ -1,421 +0,0 @@
package org.thoughtcrime.securesms.video.videoconverter;
import android.media.MediaCodec;
import android.media.MediaCodecInfo;
import android.media.MediaExtractor;
import android.media.MediaFormat;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import org.signal.core.util.logging.Log;
import org.thoughtcrime.securesms.media.MediaInput;
import org.thoughtcrime.securesms.video.VideoUtil;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Locale;
final class AudioTrackConverter {
private static final String TAG = "media-converter";
private static final boolean VERBOSE = false; // lots of logging
private static final String OUTPUT_AUDIO_MIME_TYPE = VideoUtil.AUDIO_MIME_TYPE; // Advanced Audio Coding
private static final int OUTPUT_AUDIO_AAC_PROFILE = MediaCodecInfo.CodecProfileLevel.AACObjectLC; //MediaCodecInfo.CodecProfileLevel.AACObjectHE;
private static final int TIMEOUT_USEC = 10000;
private final long mTimeFrom;
private final long mTimeTo;
private final int mAudioBitrate;
final long mInputDuration;
private final MediaExtractor mAudioExtractor;
private final MediaCodec mAudioDecoder;
private final MediaCodec mAudioEncoder;
private final ByteBuffer[] mAudioDecoderInputBuffers;
private ByteBuffer[] mAudioDecoderOutputBuffers;
private final ByteBuffer[] mAudioEncoderInputBuffers;
private ByteBuffer[] mAudioEncoderOutputBuffers;
private final MediaCodec.BufferInfo mAudioDecoderOutputBufferInfo;
private final MediaCodec.BufferInfo mAudioEncoderOutputBufferInfo;
MediaFormat mEncoderOutputAudioFormat;
boolean mAudioExtractorDone;
private boolean mAudioDecoderDone;
boolean mAudioEncoderDone;
private int mOutputAudioTrack = -1;
private int mPendingAudioDecoderOutputBufferIndex = -1;
long mMuxingAudioPresentationTime;
private int mAudioExtractedFrameCount;
private int mAudioDecodedFrameCount;
private int mAudioEncodedFrameCount;
private Muxer mMuxer;
static @Nullable
AudioTrackConverter create(
final @NonNull MediaInput input,
final long timeFrom,
final long timeTo,
final int audioBitrate) throws IOException {
final MediaExtractor audioExtractor = input.createExtractor();
final int audioInputTrack = getAndSelectAudioTrackIndex(audioExtractor);
if (audioInputTrack == -1) {
audioExtractor.release();
return null;
}
return new AudioTrackConverter(audioExtractor, audioInputTrack, timeFrom, timeTo, audioBitrate);
}
private AudioTrackConverter(
final @NonNull MediaExtractor audioExtractor,
final int audioInputTrack,
long timeFrom,
long timeTo,
int audioBitrate) throws IOException {
mTimeFrom = timeFrom;
mTimeTo = timeTo;
mAudioExtractor = audioExtractor;
mAudioBitrate = audioBitrate;
final MediaCodecInfo audioCodecInfo = MediaConverter.selectCodec(OUTPUT_AUDIO_MIME_TYPE);
if (audioCodecInfo == null) {
// Don't fail CTS if they don't have an AAC codec (not here, anyway).
Log.e(TAG, "Unable to find an appropriate codec for " + OUTPUT_AUDIO_MIME_TYPE);
throw new FileNotFoundException();
}
if (VERBOSE) Log.d(TAG, "audio found codec: " + audioCodecInfo.getName());
final MediaFormat inputAudioFormat = mAudioExtractor.getTrackFormat(audioInputTrack);
mInputDuration = inputAudioFormat.containsKey(MediaFormat.KEY_DURATION) ? inputAudioFormat.getLong(MediaFormat.KEY_DURATION) : 0;
final MediaFormat outputAudioFormat =
MediaFormat.createAudioFormat(
OUTPUT_AUDIO_MIME_TYPE,
inputAudioFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE),
inputAudioFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT));
outputAudioFormat.setInteger(MediaFormat.KEY_BIT_RATE, audioBitrate);
outputAudioFormat.setInteger(MediaFormat.KEY_AAC_PROFILE, OUTPUT_AUDIO_AAC_PROFILE);
outputAudioFormat.setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, 16 * 1024);
// Create a MediaCodec for the desired codec, then configure it as an encoder with
// our desired properties. Request a Surface to use for input.
mAudioEncoder = createAudioEncoder(audioCodecInfo, outputAudioFormat);
// Create a MediaCodec for the decoder, based on the extractor's format.
mAudioDecoder = createAudioDecoder(inputAudioFormat);
mAudioDecoderInputBuffers = mAudioDecoder.getInputBuffers();
mAudioDecoderOutputBuffers = mAudioDecoder.getOutputBuffers();
mAudioEncoderInputBuffers = mAudioEncoder.getInputBuffers();
mAudioEncoderOutputBuffers = mAudioEncoder.getOutputBuffers();
mAudioDecoderOutputBufferInfo = new MediaCodec.BufferInfo();
mAudioEncoderOutputBufferInfo = new MediaCodec.BufferInfo();
if (mTimeFrom > 0) {
mAudioExtractor.seekTo(mTimeFrom * 1000, MediaExtractor.SEEK_TO_PREVIOUS_SYNC);
Log.i(TAG, "Seek audio:" + mTimeFrom + " " + mAudioExtractor.getSampleTime());
}
}
void setMuxer(final @NonNull Muxer muxer) throws IOException {
mMuxer = muxer;
if (mEncoderOutputAudioFormat != null) {
Log.d(TAG, "muxer: adding audio track.");
if (!mEncoderOutputAudioFormat.containsKey(MediaFormat.KEY_BIT_RATE)) {
mEncoderOutputAudioFormat.setInteger(MediaFormat.KEY_BIT_RATE, mAudioBitrate);
}
if (!mEncoderOutputAudioFormat.containsKey(MediaFormat.KEY_AAC_PROFILE)) {
mEncoderOutputAudioFormat.setInteger(MediaFormat.KEY_AAC_PROFILE, OUTPUT_AUDIO_AAC_PROFILE);
}
mOutputAudioTrack = muxer.addTrack(mEncoderOutputAudioFormat);
}
}
void step() throws IOException {
// Extract audio from file and feed to decoder.
// Do not extract audio if we have determined the output format but we are not yet
// ready to mux the frames.
while (!mAudioExtractorDone && (mEncoderOutputAudioFormat == null || mMuxer != null)) {
int decoderInputBufferIndex = mAudioDecoder.dequeueInputBuffer(TIMEOUT_USEC);
if (decoderInputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) {
if (VERBOSE) Log.d(TAG, "no audio decoder input buffer");
break;
}
if (VERBOSE) {
Log.d(TAG, "audio decoder: returned input buffer: " + decoderInputBufferIndex);
}
final ByteBuffer decoderInputBuffer = mAudioDecoderInputBuffers[decoderInputBufferIndex];
final int size = mAudioExtractor.readSampleData(decoderInputBuffer, 0);
final long presentationTime = mAudioExtractor.getSampleTime();
if (VERBOSE) {
Log.d(TAG, "audio extractor: returned buffer of size " + size);
Log.d(TAG, "audio extractor: returned buffer for time " + presentationTime);
}
mAudioExtractorDone = size < 0 || (mTimeTo > 0 && presentationTime > mTimeTo * 1000);
if (mAudioExtractorDone) {
if (VERBOSE) Log.d(TAG, "audio extractor: EOS");
mAudioDecoder.queueInputBuffer(
decoderInputBufferIndex,
0,
0,
0,
MediaCodec.BUFFER_FLAG_END_OF_STREAM);
} else {
mAudioDecoder.queueInputBuffer(
decoderInputBufferIndex,
0,
size,
presentationTime,
mAudioExtractor.getSampleFlags());
}
mAudioExtractor.advance();
mAudioExtractedFrameCount++;
// We extracted a frame, let's try something else next.
break;
}
// Poll output frames from the audio decoder.
// Do not poll if we already have a pending buffer to feed to the encoder.
while (!mAudioDecoderDone && mPendingAudioDecoderOutputBufferIndex == -1
&& (mEncoderOutputAudioFormat == null || mMuxer != null)) {
final int decoderOutputBufferIndex =
mAudioDecoder.dequeueOutputBuffer(
mAudioDecoderOutputBufferInfo, TIMEOUT_USEC);
if (decoderOutputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) {
if (VERBOSE) Log.d(TAG, "no audio decoder output buffer");
break;
}
if (decoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
if (VERBOSE) Log.d(TAG, "audio decoder: output buffers changed");
mAudioDecoderOutputBuffers = mAudioDecoder.getOutputBuffers();
break;
}
if (decoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
if (VERBOSE) {
MediaFormat decoderOutputAudioFormat = mAudioDecoder.getOutputFormat();
Log.d(TAG, "audio decoder: output format changed: " + decoderOutputAudioFormat);
}
break;
}
if (VERBOSE) {
Log.d(TAG, "audio decoder: returned output buffer: " + decoderOutputBufferIndex);
Log.d(TAG, "audio decoder: returned buffer of size " + mAudioDecoderOutputBufferInfo.size);
}
if ((mAudioDecoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) {
if (VERBOSE) Log.d(TAG, "audio decoder: codec config buffer");
mAudioDecoder.releaseOutputBuffer(decoderOutputBufferIndex, false);
break;
}
if (mAudioDecoderOutputBufferInfo.presentationTimeUs < mTimeFrom * 1000 &&
(mAudioDecoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) == 0) {
if (VERBOSE)
Log.d(TAG, "audio decoder: frame prior to " + mAudioDecoderOutputBufferInfo.presentationTimeUs);
mAudioDecoder.releaseOutputBuffer(decoderOutputBufferIndex, false);
break;
}
if (VERBOSE) {
Log.d(TAG, "audio decoder: returned buffer for time " + mAudioDecoderOutputBufferInfo.presentationTimeUs);
Log.d(TAG, "audio decoder: output buffer is now pending: " + mPendingAudioDecoderOutputBufferIndex);
}
mPendingAudioDecoderOutputBufferIndex = decoderOutputBufferIndex;
mAudioDecodedFrameCount++;
// We extracted a pending frame, let's try something else next.
break;
}
// Feed the pending decoded audio buffer to the audio encoder.
while (mPendingAudioDecoderOutputBufferIndex != -1) {
if (VERBOSE) {
Log.d(TAG, "audio decoder: attempting to process pending buffer: " + mPendingAudioDecoderOutputBufferIndex);
}
final int encoderInputBufferIndex = mAudioEncoder.dequeueInputBuffer(TIMEOUT_USEC);
if (encoderInputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) {
if (VERBOSE) Log.d(TAG, "no audio encoder input buffer");
break;
}
if (VERBOSE) {
Log.d(TAG, "audio encoder: returned input buffer: " + encoderInputBufferIndex);
}
final ByteBuffer encoderInputBuffer = mAudioEncoderInputBuffers[encoderInputBufferIndex];
final int size = mAudioDecoderOutputBufferInfo.size;
final long presentationTime = mAudioDecoderOutputBufferInfo.presentationTimeUs;
if (VERBOSE) {
Log.d(TAG, "audio decoder: processing pending buffer: " + mPendingAudioDecoderOutputBufferIndex);
}
if (VERBOSE) {
Log.d(TAG, "audio decoder: pending buffer of size " + size);
Log.d(TAG, "audio decoder: pending buffer for time " + presentationTime);
}
if (size >= 0) {
final ByteBuffer decoderOutputBuffer = mAudioDecoderOutputBuffers[mPendingAudioDecoderOutputBufferIndex].duplicate();
decoderOutputBuffer.position(mAudioDecoderOutputBufferInfo.offset);
decoderOutputBuffer.limit(mAudioDecoderOutputBufferInfo.offset + size);
encoderInputBuffer.position(0);
encoderInputBuffer.put(decoderOutputBuffer);
mAudioEncoder.queueInputBuffer(
encoderInputBufferIndex,
0,
size,
presentationTime,
mAudioDecoderOutputBufferInfo.flags);
}
mAudioDecoder.releaseOutputBuffer(mPendingAudioDecoderOutputBufferIndex, false);
mPendingAudioDecoderOutputBufferIndex = -1;
if ((mAudioDecoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
if (VERBOSE) Log.d(TAG, "audio decoder: EOS");
mAudioDecoderDone = true;
}
// We enqueued a pending frame, let's try something else next.
break;
}
// Poll frames from the audio encoder and send them to the muxer.
while (!mAudioEncoderDone && (mEncoderOutputAudioFormat == null || mMuxer != null)) {
final int encoderOutputBufferIndex = mAudioEncoder.dequeueOutputBuffer(mAudioEncoderOutputBufferInfo, TIMEOUT_USEC);
if (encoderOutputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) {
if (VERBOSE) Log.d(TAG, "no audio encoder output buffer");
break;
}
if (encoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
if (VERBOSE) Log.d(TAG, "audio encoder: output buffers changed");
mAudioEncoderOutputBuffers = mAudioEncoder.getOutputBuffers();
break;
}
if (encoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
if (VERBOSE) Log.d(TAG, "audio encoder: output format changed");
Preconditions.checkState("audio encoder changed its output format again?", mOutputAudioTrack < 0);
mEncoderOutputAudioFormat = mAudioEncoder.getOutputFormat();
break;
}
Preconditions.checkState("should have added track before processing output", mMuxer != null);
if (VERBOSE) {
Log.d(TAG, "audio encoder: returned output buffer: " + encoderOutputBufferIndex);
Log.d(TAG, "audio encoder: returned buffer of size " + mAudioEncoderOutputBufferInfo.size);
}
final ByteBuffer encoderOutputBuffer = mAudioEncoderOutputBuffers[encoderOutputBufferIndex];
if ((mAudioEncoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) {
if (VERBOSE) Log.d(TAG, "audio encoder: codec config buffer");
// Simply ignore codec config buffers.
mAudioEncoder.releaseOutputBuffer(encoderOutputBufferIndex, false);
break;
}
if (VERBOSE) {
Log.d(TAG, "audio encoder: returned buffer for time " + mAudioEncoderOutputBufferInfo.presentationTimeUs);
}
if (mAudioEncoderOutputBufferInfo.size != 0) {
mMuxer.writeSampleData(mOutputAudioTrack, encoderOutputBuffer, mAudioEncoderOutputBufferInfo);
mMuxingAudioPresentationTime = Math.max(mMuxingAudioPresentationTime, mAudioEncoderOutputBufferInfo.presentationTimeUs);
}
if ((mAudioEncoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
if (VERBOSE) Log.d(TAG, "audio encoder: EOS");
mAudioEncoderDone = true;
}
mAudioEncoder.releaseOutputBuffer(encoderOutputBufferIndex, false);
mAudioEncodedFrameCount++;
// We enqueued an encoded frame, let's try something else next.
break;
}
}
void release() throws Exception {
Exception exception = null;
try {
if (mAudioExtractor != null) {
mAudioExtractor.release();
}
} catch (Exception e) {
Log.e(TAG, "error while releasing mAudioExtractor", e);
exception = e;
}
try {
if (mAudioDecoder != null) {
mAudioDecoder.stop();
mAudioDecoder.release();
}
} catch (Exception e) {
Log.e(TAG, "error while releasing mAudioDecoder", e);
if (exception == null) {
exception = e;
}
}
try {
if (mAudioEncoder != null) {
mAudioEncoder.stop();
mAudioEncoder.release();
}
} catch (Exception e) {
Log.e(TAG, "error while releasing mAudioEncoder", e);
if (exception == null) {
exception = e;
}
}
if (exception != null) {
throw exception;
}
}
String dumpState() {
return String.format(Locale.US,
"A{"
+ "extracted:%d(done:%b) "
+ "decoded:%d(done:%b) "
+ "encoded:%d(done:%b) "
+ "pending:%d "
+ "muxing:%b(track:%d} )",
mAudioExtractedFrameCount, mAudioExtractorDone,
mAudioDecodedFrameCount, mAudioDecoderDone,
mAudioEncodedFrameCount, mAudioEncoderDone,
mPendingAudioDecoderOutputBufferIndex,
mMuxer != null, mOutputAudioTrack);
}
void verifyEndState() {
Preconditions.checkState("no frame should be pending", -1 == mPendingAudioDecoderOutputBufferIndex);
}
private static @NonNull
MediaCodec createAudioDecoder(final @NonNull MediaFormat inputFormat) throws IOException {
final MediaCodec decoder = MediaCodec.createDecoderByType(MediaConverter.getMimeTypeFor(inputFormat));
decoder.configure(inputFormat, null, null, 0);
decoder.start();
return decoder;
}
private static @NonNull
MediaCodec createAudioEncoder(final @NonNull MediaCodecInfo codecInfo, final @NonNull MediaFormat format) throws IOException {
final MediaCodec encoder = MediaCodec.createByCodecName(codecInfo.getName());
encoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
encoder.start();
return encoder;
}
private static int getAndSelectAudioTrackIndex(MediaExtractor extractor) {
for (int index = 0; index < extractor.getTrackCount(); ++index) {
if (VERBOSE) {
Log.d(TAG, "format for track " + index + " is " + MediaConverter.getMimeTypeFor(extractor.getTrackFormat(index)));
}
if (isAudioFormat(extractor.getTrackFormat(index))) {
extractor.selectTrack(index);
return index;
}
}
return -1;
}
private static boolean isAudioFormat(final @NonNull MediaFormat format) {
return MediaConverter.getMimeTypeFor(format).startsWith("audio/");
}
}

355
app/src/main/java/org/thoughtcrime/securesms/video/videoconverter/MediaConverter.java
View File

@@ -1,355 +0,0 @@
/*
* Copyright (C) 2013 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* This file has been modified by Signal.
*/
package org.thoughtcrime.securesms.video.videoconverter;
import android.media.MediaCodecInfo;
import android.media.MediaCodecList;
import android.media.MediaFormat;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import androidx.annotation.RequiresApi;
import androidx.annotation.StringDef;
import androidx.annotation.WorkerThread;
import org.signal.core.util.logging.Log;
import org.thoughtcrime.securesms.media.MediaInput;
import org.thoughtcrime.securesms.video.videoconverter.muxer.StreamingMuxer;
import java.io.File;
import java.io.FileDescriptor;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.OutputStream;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
@SuppressWarnings("WeakerAccess")
public final class MediaConverter {
private static final String TAG = "media-converter";
private static final boolean VERBOSE = false; // lots of logging
// Describes when the annotation will be discarded
@Retention(RetentionPolicy.SOURCE)
@StringDef({VIDEO_CODEC_H264, VIDEO_CODEC_H265})
public @interface VideoCodec {}
public static final String VIDEO_CODEC_H264 = "video/avc";
public static final String VIDEO_CODEC_H265 = "video/hevc";
private MediaInput mInput;
private Output mOutput;
private long mTimeFrom;
private long mTimeTo;
private int mVideoResolution;
private int mVideoBitrate = 2000000; // 2Mbps
private @VideoCodec String mVideoCodec = VIDEO_CODEC_H264;
private int mAudioBitrate = 128000; // 128Kbps
private Listener mListener;
private boolean mCancelled;
public interface Listener {
boolean onProgress(int percent);
}
public MediaConverter() {
}
public void setInput(final @NonNull MediaInput videoInput) {
mInput = videoInput;
}
@SuppressWarnings("unused")
public void setOutput(final @NonNull File file) {
mOutput = new FileOutput(file);
}
@SuppressWarnings("unused")
@RequiresApi(26)
public void setOutput(final @NonNull FileDescriptor fileDescriptor) {
mOutput = new FileDescriptorOutput(fileDescriptor);
}
public void setOutput(final @NonNull OutputStream stream) {
mOutput = new StreamOutput(stream);
}
@SuppressWarnings("unused")
public void setTimeRange(long timeFrom, long timeTo) {
mTimeFrom = timeFrom;
mTimeTo = timeTo;
if (timeTo > 0 && timeFrom >= timeTo) {
throw new IllegalArgumentException("timeFrom:" + timeFrom + " timeTo:" + timeTo);
}
}
@SuppressWarnings("unused")
public void setVideoResolution(int videoResolution) {
mVideoResolution = videoResolution;
}
@SuppressWarnings("unused")
public void setVideoCodec(final @VideoCodec String videoCodec) throws FileNotFoundException {
if (selectCodec(videoCodec) == null) {
throw new FileNotFoundException();
}
mVideoCodec = videoCodec;
}
@SuppressWarnings("unused")
public void setVideoBitrate(final int videoBitrate) {
mVideoBitrate = videoBitrate;
}
@SuppressWarnings("unused")
public void setAudioBitrate(final int audioBitrate) {
mAudioBitrate = audioBitrate;
}
@SuppressWarnings("unused")
public void setListener(final Listener listener) {
mListener = listener;
}
@WorkerThread
@RequiresApi(23)
public void convert() throws EncodingException, IOException {
// Exception that may be thrown during release.
Exception exception = null;
Muxer muxer = null;
VideoTrackConverter videoTrackConverter = null;
AudioTrackConverter audioTrackConverter = null;
try {
videoTrackConverter = VideoTrackConverter.create(mInput, mTimeFrom, mTimeTo, mVideoResolution, mVideoBitrate, mVideoCodec);
audioTrackConverter = AudioTrackConverter.create(mInput, mTimeFrom, mTimeTo, mAudioBitrate);
if (videoTrackConverter == null && audioTrackConverter == null) {
throw new EncodingException("No video and audio tracks");
}
muxer = mOutput.createMuxer();
doExtractDecodeEditEncodeMux(
videoTrackConverter,
audioTrackConverter,
muxer);
} catch (EncodingException | IOException e) {
Log.e(TAG, "error converting", e);
exception = e;
throw e;
} catch (Exception e) {
Log.e(TAG, "error converting", e);
exception = e;
} finally {
if (VERBOSE) Log.d(TAG, "releasing extractor, decoder, encoder, and muxer");
// Try to release everything we acquired, even if one of the releases fails, in which
// case we save the first exception we got and re-throw at the end (unless something
// other exception has already been thrown). This guarantees the first exception thrown
// is reported as the cause of the error, everything is (attempted) to be released, and
// all other exceptions appear in the logs.
try {
if (videoTrackConverter != null) {
videoTrackConverter.release();
}
} catch (Exception e) {
if (exception == null) {
exception = e;
}
}
try {
if (audioTrackConverter != null) {
audioTrackConverter.release();
}
} catch (Exception e) {
if (exception == null) {
exception = e;
}
}
try {
if (muxer != null) {
muxer.stop();
muxer.release();
}
} catch (Exception e) {
Log.e(TAG, "error while releasing muxer", e);
if (exception == null) {
exception = e;
}
}
}
if (exception != null) {
throw new EncodingException("Transcode failed", exception);
}
}
/**
* Does the actual work for extracting, decoding, encoding and muxing.
*/
private void doExtractDecodeEditEncodeMux(
final @Nullable VideoTrackConverter videoTrackConverter,
final @Nullable AudioTrackConverter audioTrackConverter,
final @NonNull Muxer muxer) throws IOException, TranscodingException {
boolean muxing = false;
int percentProcessed = 0;
long inputDuration = Math.max(
videoTrackConverter == null ? 0 : videoTrackConverter.mInputDuration,
audioTrackConverter == null ? 0 : audioTrackConverter.mInputDuration);
while (!mCancelled &&
((videoTrackConverter != null && !videoTrackConverter.mVideoEncoderDone) ||
(audioTrackConverter != null &&!audioTrackConverter.mAudioEncoderDone))) {
if (VERBOSE) {
Log.d(TAG, "loop: " +
(videoTrackConverter == null ? "" : videoTrackConverter.dumpState()) +
(audioTrackConverter == null ? "" : audioTrackConverter.dumpState()) +
" muxing:" + muxing);
}
if (videoTrackConverter != null && (audioTrackConverter == null || audioTrackConverter.mAudioExtractorDone || videoTrackConverter.mMuxingVideoPresentationTime <= audioTrackConverter.mMuxingAudioPresentationTime)) {
videoTrackConverter.step();
}
if (audioTrackConverter != null && (videoTrackConverter == null || videoTrackConverter.mVideoExtractorDone || videoTrackConverter.mMuxingVideoPresentationTime >= audioTrackConverter.mMuxingAudioPresentationTime)) {
audioTrackConverter.step();
}
if (inputDuration != 0 && mListener != null) {
final long timeFromUs = mTimeFrom <= 0 ? 0 : mTimeFrom * 1000;
final long timeToUs = mTimeTo <= 0 ? inputDuration : mTimeTo * 1000;
final int curPercentProcessed = (int) (100 *
(Math.max(
videoTrackConverter == null ? 0 : videoTrackConverter.mMuxingVideoPresentationTime,
audioTrackConverter == null ? 0 : audioTrackConverter.mMuxingAudioPresentationTime)
- timeFromUs) / (timeToUs - timeFromUs));
if (curPercentProcessed != percentProcessed) {
percentProcessed = curPercentProcessed;
mCancelled = mCancelled || mListener.onProgress(percentProcessed);
}
}
if (!muxing
&& (videoTrackConverter == null || videoTrackConverter.mEncoderOutputVideoFormat != null)
&& (audioTrackConverter == null || audioTrackConverter.mEncoderOutputAudioFormat != null)) {
if (videoTrackConverter != null) {
videoTrackConverter.setMuxer(muxer);
}
if (audioTrackConverter != null) {
audioTrackConverter.setMuxer(muxer);
}
Log.d(TAG, "muxer: starting");
muxer.start();
muxing = true;
}
}
// Basic sanity checks.
if (videoTrackConverter != null) {
videoTrackConverter.verifyEndState();
}
if (audioTrackConverter != null) {
audioTrackConverter.verifyEndState();
}
// TODO: Check the generated output file.
}
static String getMimeTypeFor(MediaFormat format) {
return format.getString(MediaFormat.KEY_MIME);
}
/**
* Returns the first codec capable of encoding the specified MIME type, or null if no match was
* found.
*/
static MediaCodecInfo selectCodec(final String mimeType) {
final int numCodecs = MediaCodecList.getCodecCount();
for (int i = 0; i < numCodecs; i++) {
final MediaCodecInfo codecInfo = MediaCodecList.getCodecInfoAt(i);
if (!codecInfo.isEncoder()) {
continue;
}
final String[] types = codecInfo.getSupportedTypes();
for (String type : types) {
if (type.equalsIgnoreCase(mimeType)) {
return codecInfo;
}
}
}
return null;
}
interface Output {
@NonNull
Muxer createMuxer() throws IOException;
}
private static class FileOutput implements Output {
final File file;
FileOutput(final @NonNull File file) {
this.file = file;
}
@Override
public @NonNull
Muxer createMuxer() throws IOException {
return new AndroidMuxer(file);
}
}
@RequiresApi(26)
private static class FileDescriptorOutput implements Output {
final FileDescriptor fileDescriptor;
FileDescriptorOutput(final @NonNull FileDescriptor fileDescriptor) {
this.fileDescriptor = fileDescriptor;
}
@Override
public @NonNull
Muxer createMuxer() throws IOException {
return new AndroidMuxer(fileDescriptor);
}
}
private static class StreamOutput implements Output {
final OutputStream outputStream;
StreamOutput(final @NonNull OutputStream outputStream) {
this.outputStream = outputStream;
}
@Override
public @NonNull Muxer createMuxer() {
return new StreamingMuxer(outputStream);
}
}
}

202
app/src/main/java/org/thoughtcrime/securesms/video/videoconverter/VideoThumbnailsExtractor.java
View File

@@ -1,202 +0,0 @@
package org.thoughtcrime.securesms.video.videoconverter;
import android.graphics.Bitmap;
import android.media.MediaCodec;
import android.media.MediaExtractor;
import android.media.MediaFormat;
import android.opengl.GLES20;
import androidx.annotation.NonNull;
import androidx.annotation.RequiresApi;
import org.signal.core.util.logging.Log;
import org.thoughtcrime.securesms.media.MediaInput;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
@RequiresApi(api = 23)
final class VideoThumbnailsExtractor {
private static final String TAG = Log.tag(VideoThumbnailsExtractor.class);
interface Callback {
void durationKnown(long duration);
boolean publishProgress(int index, Bitmap thumbnail);
void failed();
}
static void extractThumbnails(final @NonNull MediaInput input,
final int thumbnailCount,
final int thumbnailResolution,
final @NonNull Callback callback)
{
MediaExtractor extractor = null;
MediaCodec decoder = null;
OutputSurface outputSurface = null;
try {
extractor = input.createExtractor();
MediaFormat mediaFormat = null;
for (int index = 0; index < extractor.getTrackCount(); ++index) {
if (extractor.getTrackFormat(index).getString(MediaFormat.KEY_MIME).startsWith("video/")) {
extractor.selectTrack(index);
mediaFormat = extractor.getTrackFormat(index);
break;
}
}
if (mediaFormat != null) {
final String mime = mediaFormat.getString(MediaFormat.KEY_MIME);
final int rotation = mediaFormat.containsKey(MediaFormat.KEY_ROTATION) ? mediaFormat.getInteger(MediaFormat.KEY_ROTATION) : 0;
final int width = mediaFormat.getInteger(MediaFormat.KEY_WIDTH);
final int height = mediaFormat.getInteger(MediaFormat.KEY_HEIGHT);
final int outputWidth;
final int outputHeight;
if (width < height) {
outputWidth = thumbnailResolution;
outputHeight = height * outputWidth / width;
} else {
outputHeight = thumbnailResolution;
outputWidth = width * outputHeight / height;
}
final int outputWidthRotated;
final int outputHeightRotated;
if ((rotation % 180 == 90)) {
//noinspection SuspiciousNameCombination
outputWidthRotated = outputHeight;
//noinspection SuspiciousNameCombination
outputHeightRotated = outputWidth;
} else {
outputWidthRotated = outputWidth;
outputHeightRotated = outputHeight;
}
Log.i(TAG, "video :" + width + "x" + height + " " + rotation);
Log.i(TAG, "output: " + outputWidthRotated + "x" + outputHeightRotated);
outputSurface = new OutputSurface(outputWidthRotated, outputHeightRotated, true);
decoder = MediaCodec.createDecoderByType(mime);
decoder.configure(mediaFormat, outputSurface.getSurface(), null, 0);
decoder.start();
long duration = 0;
if (mediaFormat.containsKey(MediaFormat.KEY_DURATION)) {
duration = mediaFormat.getLong(MediaFormat.KEY_DURATION);
} else {
Log.w(TAG, "Video is missing duration!");
}
callback.durationKnown(duration);
doExtract(extractor, decoder, outputSurface, outputWidthRotated, outputHeightRotated, duration, thumbnailCount, callback);
}
} catch (Throwable t) {
Log.w(TAG, t);
callback.failed();
} finally {
if (outputSurface != null) {
outputSurface.release();
}
if (decoder != null) {
try {
decoder.stop();
} catch (MediaCodec.CodecException codecException) {
Log.w(TAG, "Decoder stop failed: " + codecException.getDiagnosticInfo(), codecException);
} catch (IllegalStateException ise) {
Log.w(TAG, "Decoder stop failed", ise);
}
decoder.release();
}
if (extractor != null) {
extractor.release();
}
}
}
private static void doExtract(final @NonNull MediaExtractor extractor,
final @NonNull MediaCodec decoder,
final @NonNull OutputSurface outputSurface,
final int outputWidth, int outputHeight, long duration, int thumbnailCount,
final @NonNull Callback callback)
throws TranscodingException
{
final int TIMEOUT_USEC = 10000;
final ByteBuffer[] decoderInputBuffers = decoder.getInputBuffers();
final MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
int samplesExtracted = 0;
int thumbnailsCreated = 0;
Log.i(TAG, "doExtract started");
final ByteBuffer pixelBuf = ByteBuffer.allocateDirect(outputWidth * outputHeight * 4);
pixelBuf.order(ByteOrder.LITTLE_ENDIAN);
boolean outputDone = false;
boolean inputDone = false;
while (!outputDone) {
if (!inputDone) {
int inputBufIndex = decoder.dequeueInputBuffer(TIMEOUT_USEC);
if (inputBufIndex >= 0) {
final ByteBuffer inputBuf = decoderInputBuffers[inputBufIndex];
final int sampleSize = extractor.readSampleData(inputBuf, 0);
if (sampleSize < 0 || samplesExtracted >= thumbnailCount) {
decoder.queueInputBuffer(inputBufIndex, 0, 0, 0L, MediaCodec.BUFFER_FLAG_END_OF_STREAM);
inputDone = true;
Log.i(TAG, "input done");
} else {
final long presentationTimeUs = extractor.getSampleTime();
decoder.queueInputBuffer(inputBufIndex, 0, sampleSize, presentationTimeUs, 0 /*flags*/);
samplesExtracted++;
extractor.seekTo(duration * samplesExtracted / thumbnailCount, MediaExtractor.SEEK_TO_CLOSEST_SYNC);
Log.i(TAG, "seek to " + duration * samplesExtracted / thumbnailCount + ", actual " + extractor.getSampleTime());
}
}
}
final int outputBufIndex;
try {
outputBufIndex = decoder.dequeueOutputBuffer(info, TIMEOUT_USEC);
} catch (IllegalStateException e) {
Log.w(TAG, "Decoder not in the Executing state, or codec is configured in asynchronous mode.", e);
throw new TranscodingException("Decoder not in the Executing state, or codec is configured in asynchronous mode.", e);
}
if (outputBufIndex >= 0) {
if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
outputDone = true;
}
final boolean shouldRender = (info.size != 0) /*&& (info.presentationTimeUs >= duration * decodeCount / thumbnailCount)*/;
decoder.releaseOutputBuffer(outputBufIndex, shouldRender);
if (shouldRender) {
outputSurface.awaitNewImage();
outputSurface.drawImage();
if (thumbnailsCreated < thumbnailCount) {
pixelBuf.rewind();
GLES20.glReadPixels(0, 0, outputWidth, outputHeight, GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, pixelBuf);
final Bitmap bitmap = Bitmap.createBitmap(outputWidth, outputHeight, Bitmap.Config.ARGB_8888);
pixelBuf.rewind();
bitmap.copyPixelsFromBuffer(pixelBuf);
if (!callback.publishProgress(thumbnailsCreated, bitmap)) {
break;
}
Log.i(TAG, "publishProgress for frame " + thumbnailsCreated + " at " + info.presentationTimeUs + " (target " + duration * thumbnailsCreated / thumbnailCount + ")");
}
thumbnailsCreated++;
}
}
}
Log.i(TAG, "doExtract finished");
}
}

1
app/src/main/java/org/thoughtcrime/securesms/video/videoconverter/VideoThumbnailsView.java
View File

@@ -15,7 +15,6 @@ import androidx.annotation.Nullable;
import androidx.annotation.RequiresApi;
import org.signal.core.util.logging.Log;
import org.thoughtcrime.securesms.media.MediaInput;
import java.io.IOException;
import java.lang.ref.WeakReference;

512
app/src/main/java/org/thoughtcrime/securesms/video/videoconverter/VideoTrackConverter.java
View File

@@ -1,512 +0,0 @@
package org.thoughtcrime.securesms.video.videoconverter;
import android.media.MediaCodec;
import android.media.MediaCodecInfo;
import android.media.MediaExtractor;
import android.media.MediaFormat;
import android.view.Surface;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import androidx.annotation.RequiresApi;
import org.signal.core.util.logging.Log;
import org.thoughtcrime.securesms.media.MediaInput;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Locale;
import java.util.concurrent.atomic.AtomicReference;
final class VideoTrackConverter {
private static final String TAG = "media-converter";
private static final boolean VERBOSE = false; // lots of logging
private static final int OUTPUT_VIDEO_IFRAME_INTERVAL = 1; // 1 second between I-frames
private static final int OUTPUT_VIDEO_FRAME_RATE = 30; // needed only for MediaFormat.KEY_I_FRAME_INTERVAL to work; the actual frame rate matches the source
private static final int TIMEOUT_USEC = 10000;
private static final String MEDIA_FORMAT_KEY_DISPLAY_WIDTH = "display-width";
private static final String MEDIA_FORMAT_KEY_DISPLAY_HEIGHT = "display-height";
private final long mTimeFrom;
private final long mTimeTo;
final long mInputDuration;
private final MediaExtractor mVideoExtractor;
private final MediaCodec mVideoDecoder;
private final MediaCodec mVideoEncoder;
private final InputSurface mInputSurface;
private final OutputSurface mOutputSurface;
private final ByteBuffer[] mVideoDecoderInputBuffers;
private ByteBuffer[] mVideoEncoderOutputBuffers;
private final MediaCodec.BufferInfo mVideoDecoderOutputBufferInfo;
private final MediaCodec.BufferInfo mVideoEncoderOutputBufferInfo;
MediaFormat mEncoderOutputVideoFormat;
boolean mVideoExtractorDone;
private boolean mVideoDecoderDone;
boolean mVideoEncoderDone;
private int mOutputVideoTrack = -1;
long mMuxingVideoPresentationTime;
private int mVideoExtractedFrameCount;
private int mVideoDecodedFrameCount;
private int mVideoEncodedFrameCount;
private Muxer mMuxer;
@RequiresApi(23)
static @Nullable VideoTrackConverter create(
final @NonNull MediaInput input,
final long timeFrom,
final long timeTo,
final int videoResolution,
final int videoBitrate,
final @NonNull String videoCodec) throws IOException, TranscodingException {
final MediaExtractor videoExtractor = input.createExtractor();
final int videoInputTrack = getAndSelectVideoTrackIndex(videoExtractor);
if (videoInputTrack == -1) {
videoExtractor.release();
return null;
}
return new VideoTrackConverter(videoExtractor, videoInputTrack, timeFrom, timeTo, videoResolution, videoBitrate, videoCodec);
}
@RequiresApi(23)
private VideoTrackConverter(
final @NonNull MediaExtractor videoExtractor,
final int videoInputTrack,
final long timeFrom,
final long timeTo,
final int videoResolution,
final int videoBitrate,
final @NonNull String videoCodec) throws IOException, TranscodingException {
mTimeFrom = timeFrom;
mTimeTo = timeTo;
mVideoExtractor = videoExtractor;
final MediaCodecInfo videoCodecInfo = MediaConverter.selectCodec(videoCodec);
if (videoCodecInfo == null) {
// Don't fail CTS if they don't have an AVC codec (not here, anyway).
Log.e(TAG, "Unable to find an appropriate codec for " + videoCodec);
throw new FileNotFoundException();
}
if (VERBOSE) Log.d(TAG, "video found codec: " + videoCodecInfo.getName());
final MediaFormat inputVideoFormat = mVideoExtractor.getTrackFormat(videoInputTrack);
mInputDuration = inputVideoFormat.containsKey(MediaFormat.KEY_DURATION) ? inputVideoFormat.getLong(MediaFormat.KEY_DURATION) : 0;
final int rotation = inputVideoFormat.containsKey(MediaFormat.KEY_ROTATION) ? inputVideoFormat.getInteger(MediaFormat.KEY_ROTATION) : 0;
final int width = inputVideoFormat.containsKey(MEDIA_FORMAT_KEY_DISPLAY_WIDTH)
? inputVideoFormat.getInteger(MEDIA_FORMAT_KEY_DISPLAY_WIDTH)
: inputVideoFormat.getInteger(MediaFormat.KEY_WIDTH);
final int height = inputVideoFormat.containsKey(MEDIA_FORMAT_KEY_DISPLAY_HEIGHT)
? inputVideoFormat.getInteger(MEDIA_FORMAT_KEY_DISPLAY_HEIGHT)
: inputVideoFormat.getInteger(MediaFormat.KEY_HEIGHT);
int outputWidth = width;
int outputHeight = height;
if (outputWidth < outputHeight) {
outputWidth = videoResolution;
outputHeight = height * outputWidth / width;
} else {
outputHeight = videoResolution;
outputWidth = width * outputHeight / height;
}
// many encoders do not work when height and width are not multiple of 16 (also, some iPhones do not play some heights)
outputHeight = (outputHeight + 7) & ~0xF;
outputWidth = (outputWidth + 7) & ~0xF;
final int outputWidthRotated;
final int outputHeightRotated;
if ((rotation % 180 == 90)) {
//noinspection SuspiciousNameCombination
outputWidthRotated = outputHeight;
//noinspection SuspiciousNameCombination
outputHeightRotated = outputWidth;
} else {
outputWidthRotated = outputWidth;
outputHeightRotated = outputHeight;
}
final MediaFormat outputVideoFormat = MediaFormat.createVideoFormat(videoCodec, outputWidthRotated, outputHeightRotated);
// Set some properties. Failing to specify some of these can cause the MediaCodec
// configure() call to throw an unhelpful exception.
outputVideoFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface);
outputVideoFormat.setInteger(MediaFormat.KEY_BIT_RATE, videoBitrate);
outputVideoFormat.setInteger(MediaFormat.KEY_FRAME_RATE, OUTPUT_VIDEO_FRAME_RATE);
outputVideoFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, OUTPUT_VIDEO_IFRAME_INTERVAL);
if (VERBOSE) Log.d(TAG, "video format: " + outputVideoFormat);
// Create a MediaCodec for the desired codec, then configure it as an encoder with
// our desired properties. Request a Surface to use for input.
final AtomicReference<Surface> inputSurfaceReference = new AtomicReference<>();
mVideoEncoder = createVideoEncoder(videoCodecInfo, outputVideoFormat, inputSurfaceReference);
mInputSurface = new InputSurface(inputSurfaceReference.get());
mInputSurface.makeCurrent();
// Create a MediaCodec for the decoder, based on the extractor's format.
mOutputSurface = new OutputSurface();
mOutputSurface.changeFragmentShader(createFragmentShader(
inputVideoFormat.getInteger(MediaFormat.KEY_WIDTH), inputVideoFormat.getInteger(MediaFormat.KEY_HEIGHT),
outputWidth, outputHeight));
mVideoDecoder = createVideoDecoder(inputVideoFormat, mOutputSurface.getSurface());
mVideoDecoderInputBuffers = mVideoDecoder.getInputBuffers();
mVideoEncoderOutputBuffers = mVideoEncoder.getOutputBuffers();
mVideoDecoderOutputBufferInfo = new MediaCodec.BufferInfo();
mVideoEncoderOutputBufferInfo = new MediaCodec.BufferInfo();
if (mTimeFrom > 0) {
mVideoExtractor.seekTo(mTimeFrom * 1000, MediaExtractor.SEEK_TO_PREVIOUS_SYNC);
Log.i(TAG, "Seek video:" + mTimeFrom + " " + mVideoExtractor.getSampleTime());
}
}
void setMuxer(final @NonNull Muxer muxer) throws IOException {
mMuxer = muxer;
if (mEncoderOutputVideoFormat != null) {
Log.d(TAG, "muxer: adding video track.");
mOutputVideoTrack = muxer.addTrack(mEncoderOutputVideoFormat);
}
}
void step() throws IOException, TranscodingException {
// Extract video from file and feed to decoder.
// Do not extract video if we have determined the output format but we are not yet
// ready to mux the frames.
while (!mVideoExtractorDone
&& (mEncoderOutputVideoFormat == null || mMuxer != null)) {
int decoderInputBufferIndex = mVideoDecoder.dequeueInputBuffer(TIMEOUT_USEC);
if (decoderInputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) {
if (VERBOSE) Log.d(TAG, "no video decoder input buffer");
break;
}
if (VERBOSE) {
Log.d(TAG, "video decoder: returned input buffer: " + decoderInputBufferIndex);
}
final ByteBuffer decoderInputBuffer = mVideoDecoderInputBuffers[decoderInputBufferIndex];
final int size = mVideoExtractor.readSampleData(decoderInputBuffer, 0);
final long presentationTime = mVideoExtractor.getSampleTime();
if (VERBOSE) {
Log.d(TAG, "video extractor: returned buffer of size " + size);
Log.d(TAG, "video extractor: returned buffer for time " + presentationTime);
}
mVideoExtractorDone = size < 0 || (mTimeTo > 0 && presentationTime > mTimeTo * 1000);
if (mVideoExtractorDone) {
if (VERBOSE) Log.d(TAG, "video extractor: EOS");
mVideoDecoder.queueInputBuffer(
decoderInputBufferIndex,
0,
0,
0,
MediaCodec.BUFFER_FLAG_END_OF_STREAM);
} else {
mVideoDecoder.queueInputBuffer(
decoderInputBufferIndex,
0,
size,
presentationTime,
mVideoExtractor.getSampleFlags());
}
mVideoExtractor.advance();
mVideoExtractedFrameCount++;
// We extracted a frame, let's try something else next.
break;
}
// Poll output frames from the video decoder and feed the encoder.
while (!mVideoDecoderDone && (mEncoderOutputVideoFormat == null || mMuxer != null)) {
final int decoderOutputBufferIndex =
mVideoDecoder.dequeueOutputBuffer(
mVideoDecoderOutputBufferInfo, TIMEOUT_USEC);
if (decoderOutputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) {
if (VERBOSE) Log.d(TAG, "no video decoder output buffer");
break;
}
if (decoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
if (VERBOSE) Log.d(TAG, "video decoder: output buffers changed");
break;
}
if (decoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
if (VERBOSE) {
Log.d(TAG, "video decoder: output format changed: " + mVideoDecoder.getOutputFormat());
}
break;
}
if (VERBOSE) {
Log.d(TAG, "video decoder: returned output buffer: "
+ decoderOutputBufferIndex);
Log.d(TAG, "video decoder: returned buffer of size "
+ mVideoDecoderOutputBufferInfo.size);
}
if ((mVideoDecoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) {
if (VERBOSE) Log.d(TAG, "video decoder: codec config buffer");
mVideoDecoder.releaseOutputBuffer(decoderOutputBufferIndex, false);
break;
}
if (mVideoDecoderOutputBufferInfo.presentationTimeUs < mTimeFrom * 1000 &&
(mVideoDecoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) == 0) {
if (VERBOSE) Log.d(TAG, "video decoder: frame prior to " + mVideoDecoderOutputBufferInfo.presentationTimeUs);
mVideoDecoder.releaseOutputBuffer(decoderOutputBufferIndex, false);
break;
}
if (VERBOSE) {
Log.d(TAG, "video decoder: returned buffer for time " + mVideoDecoderOutputBufferInfo.presentationTimeUs);
}
boolean render = mVideoDecoderOutputBufferInfo.size != 0;
mVideoDecoder.releaseOutputBuffer(decoderOutputBufferIndex, render);
if (render) {
if (VERBOSE) Log.d(TAG, "output surface: await new image");
mOutputSurface.awaitNewImage();
// Edit the frame and send it to the encoder.
if (VERBOSE) Log.d(TAG, "output surface: draw image");
mOutputSurface.drawImage();
mInputSurface.setPresentationTime(mVideoDecoderOutputBufferInfo.presentationTimeUs * 1000);
if (VERBOSE) Log.d(TAG, "input surface: swap buffers");
mInputSurface.swapBuffers();
if (VERBOSE) Log.d(TAG, "video encoder: notified of new frame");
}
if ((mVideoDecoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
if (VERBOSE) Log.d(TAG, "video decoder: EOS");
mVideoDecoderDone = true;
mVideoEncoder.signalEndOfInputStream();
}
mVideoDecodedFrameCount++;
// We extracted a pending frame, let's try something else next.
break;
}
// Poll frames from the video encoder and send them to the muxer.
while (!mVideoEncoderDone && (mEncoderOutputVideoFormat == null || mMuxer != null)) {
final int encoderOutputBufferIndex = mVideoEncoder.dequeueOutputBuffer(mVideoEncoderOutputBufferInfo, TIMEOUT_USEC);
if (encoderOutputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) {
if (VERBOSE) Log.d(TAG, "no video encoder output buffer");
if (mVideoDecoderDone) {
// on some devices and encoder stops after signalEndOfInputStream
Log.w(TAG, "mVideoDecoderDone, but didn't get BUFFER_FLAG_END_OF_STREAM");
mVideoEncodedFrameCount = mVideoDecodedFrameCount;
mVideoEncoderDone = true;
}
break;
}
if (encoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
if (VERBOSE) Log.d(TAG, "video encoder: output buffers changed");
mVideoEncoderOutputBuffers = mVideoEncoder.getOutputBuffers();
break;
}
if (encoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
if (VERBOSE) Log.d(TAG, "video encoder: output format changed");
Preconditions.checkState("video encoder changed its output format again?", mOutputVideoTrack < 0);
mEncoderOutputVideoFormat = mVideoEncoder.getOutputFormat();
break;
}
Preconditions.checkState("should have added track before processing output", mMuxer != null);
if (VERBOSE) {
Log.d(TAG, "video encoder: returned output buffer: " + encoderOutputBufferIndex);
Log.d(TAG, "video encoder: returned buffer of size " + mVideoEncoderOutputBufferInfo.size);
}
final ByteBuffer encoderOutputBuffer = mVideoEncoderOutputBuffers[encoderOutputBufferIndex];
if ((mVideoEncoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) {
if (VERBOSE) Log.d(TAG, "video encoder: codec config buffer");
// Simply ignore codec config buffers.
mVideoEncoder.releaseOutputBuffer(encoderOutputBufferIndex, false);
break;
}
if (VERBOSE) {
Log.d(TAG, "video encoder: returned buffer for time " + mVideoEncoderOutputBufferInfo.presentationTimeUs);
}
if (mVideoEncoderOutputBufferInfo.size != 0) {
mMuxer.writeSampleData(mOutputVideoTrack, encoderOutputBuffer, mVideoEncoderOutputBufferInfo);
mMuxingVideoPresentationTime = Math.max(mMuxingVideoPresentationTime, mVideoEncoderOutputBufferInfo.presentationTimeUs);
}
if ((mVideoEncoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
if (VERBOSE) Log.d(TAG, "video encoder: EOS");
mVideoEncoderDone = true;
}
mVideoEncoder.releaseOutputBuffer(encoderOutputBufferIndex, false);
mVideoEncodedFrameCount++;
// We enqueued an encoded frame, let's try something else next.
break;
}
}
void release() throws Exception {
Exception exception = null;
try {
if (mVideoExtractor != null) {
mVideoExtractor.release();
}
} catch (Exception e) {
Log.e(TAG, "error while releasing mVideoExtractor", e);
exception = e;
}
try {
if (mVideoDecoder != null) {
mVideoDecoder.stop();
mVideoDecoder.release();
}
} catch (Exception e) {
Log.e(TAG, "error while releasing mVideoDecoder", e);
if (exception == null) {
exception = e;
}
}
try {
if (mOutputSurface != null) {
mOutputSurface.release();
}
} catch (Exception e) {
Log.e(TAG, "error while releasing mOutputSurface", e);
if (exception == null) {
exception = e;
}
}
try {
if (mInputSurface != null) {
mInputSurface.release();
}
} catch (Exception e) {
Log.e(TAG, "error while releasing mInputSurface", e);
if (exception == null) {
exception = e;
}
}
try {
if (mVideoEncoder != null) {
mVideoEncoder.stop();
mVideoEncoder.release();
}
} catch (Exception e) {
Log.e(TAG, "error while releasing mVideoEncoder", e);
if (exception == null) {
exception = e;
}
}
if (exception != null) {
throw exception;
}
}
String dumpState() {
return String.format(Locale.US,
"V{"
+ "extracted:%d(done:%b) "
+ "decoded:%d(done:%b) "
+ "encoded:%d(done:%b) "
+ "muxing:%b(track:%d)} ",
mVideoExtractedFrameCount, mVideoExtractorDone,
mVideoDecodedFrameCount, mVideoDecoderDone,
mVideoEncodedFrameCount, mVideoEncoderDone,
mMuxer != null, mOutputVideoTrack);
}
void verifyEndState() {
Preconditions.checkState("encoded (" + mVideoEncodedFrameCount + ") and decoded (" + mVideoDecodedFrameCount + ") video frame counts should match", mVideoDecodedFrameCount == mVideoEncodedFrameCount);
Preconditions.checkState("decoded frame count should be less than extracted frame count", mVideoDecodedFrameCount <= mVideoExtractedFrameCount);
}
private static String createFragmentShader(
final int srcWidth,
final int srcHeight,
final int dstWidth,
final int dstHeight) {
final float kernelSizeX = (float) srcWidth / (float) dstWidth;
final float kernelSizeY = (float) srcHeight / (float) dstHeight;
Log.i(TAG, "kernel " + kernelSizeX + "x" + kernelSizeY);
final String shader;
if (kernelSizeX <= 2 && kernelSizeY <= 2) {
shader =
"#extension GL_OES_EGL_image_external : require\n" +
"precision mediump float;\n" + // highp here doesn't seem to matter
"varying vec2 vTextureCoord;\n" +
"uniform samplerExternalOES sTexture;\n" +
"void main() {\n" +
" gl_FragColor = texture2D(sTexture, vTextureCoord);\n" +
"}\n";
} else {
final int kernelRadiusX = (int) Math.ceil(kernelSizeX - .1f) / 2;
final int kernelRadiusY = (int) Math.ceil(kernelSizeY - .1f) / 2;
final float stepX = kernelSizeX / (1 + 2 * kernelRadiusX) * (1f / srcWidth);
final float stepY = kernelSizeY / (1 + 2 * kernelRadiusY) * (1f / srcHeight);
final float sum = (1 + 2 * kernelRadiusX) * (1 + 2 * kernelRadiusY);
final StringBuilder colorLoop = new StringBuilder();
for (int i = -kernelRadiusX; i <=kernelRadiusX; i++) {
for (int j = -kernelRadiusY; j <=kernelRadiusY; j++) {
if (i != 0 || j != 0) {
colorLoop.append(" + texture2D(sTexture, vTextureCoord.xy + vec2(")
.append(i * stepX).append(", ").append(j * stepY).append("))\n");
}
}
}
shader =
"#extension GL_OES_EGL_image_external : require\n" +
"precision mediump float;\n" + // highp here doesn't seem to matter
"varying vec2 vTextureCoord;\n" +
"uniform samplerExternalOES sTexture;\n" +
"void main() {\n" +
" gl_FragColor = (texture2D(sTexture, vTextureCoord)\n" +
colorLoop.toString() +
" ) / " + sum + ";\n" +
"}\n";
}
Log.i(TAG, shader);
return shader;
}
private @NonNull
MediaCodec createVideoDecoder(
final @NonNull MediaFormat inputFormat,
final @NonNull Surface surface) throws IOException {
final MediaCodec decoder = MediaCodec.createDecoderByType(MediaConverter.getMimeTypeFor(inputFormat));
decoder.configure(inputFormat, surface, null, 0);
decoder.start();
return decoder;
}
private @NonNull
MediaCodec createVideoEncoder(
final @NonNull MediaCodecInfo codecInfo,
final @NonNull MediaFormat format,
final @NonNull AtomicReference<Surface> surfaceReference) throws IOException {
final MediaCodec encoder = MediaCodec.createByCodecName(codecInfo.getName());
encoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
// Must be called before start()
surfaceReference.set(encoder.createInputSurface());
encoder.start();
return encoder;
}
private static int getAndSelectVideoTrackIndex(@NonNull MediaExtractor extractor) {
for (int index = 0; index < extractor.getTrackCount(); ++index) {
if (VERBOSE) {
Log.d(TAG, "format for track " + index + " is " + MediaConverter.getMimeTypeFor(extractor.getTrackFormat(index)));
}
if (isVideoFormat(extractor.getTrackFormat(index))) {
extractor.selectTrack(index);
return index;
}
}
return -1;
}
private static boolean isVideoFormat(final @NonNull MediaFormat format) {
return MediaConverter.getMimeTypeFor(format).startsWith("video/");
}
}