Signal/Android
1
0
Fork 0
mirror of https://github.com/signalapp/Signal-Android.git synced 2026-04-21 17:29:32 +01:00
Code Issues Packages Projects Releases Wiki Activity

Video trimming behind feature flag.

Browse Source
This commit is contained in:
Alan Evans
2020-02-13 14:22:21 -04:00
committed by Greyson Parrelli
parent 7f867a6185
commit 40fd7ca332
41 changed files with 1966 additions and 268 deletions
Download Patch File Download Diff File Expand all files Collapse all files

54
app/src/main/java/org/thoughtcrime/securesms/video/DecryptableUriVideoInput.java Normal file
View File

@@ -0,0 +1,54 @@
package org.thoughtcrime.securesms.video;
import android.content.Context;
import android.media.MediaDataSource;
import android.net.Uri;
import androidx.annotation.NonNull;
import androidx.annotation.RequiresApi;
import org.thoughtcrime.securesms.attachments.AttachmentId;
import org.thoughtcrime.securesms.database.DatabaseFactory;
import org.thoughtcrime.securesms.mms.PartAuthority;
import org.thoughtcrime.securesms.mms.PartUriParser;
import org.thoughtcrime.securesms.providers.BlobProvider;
import org.thoughtcrime.securesms.video.videoconverter.VideoInput;
import java.io.IOException;
@RequiresApi(api = 23)
public final class DecryptableUriVideoInput {
private DecryptableUriVideoInput() {
}
public static VideoInput createForUri(@NonNull Context context, @NonNull Uri uri) throws IOException {
if (BlobProvider.isAuthority(uri)) {
return new VideoInput.MediaDataSourceVideoInput(BlobProvider.getInstance().getMediaDataSource(context, uri));
}
if (PartAuthority.isLocalUri(uri)) {
return createForAttachmentUri(context, uri);
}
return new VideoInput.UriVideoInput(context, uri);
}
private static VideoInput createForAttachmentUri(@NonNull Context context, @NonNull Uri uri) {
AttachmentId partId = new PartUriParser(uri).getPartId();
if (!partId.isValid()) {
throw new AssertionError();
}
MediaDataSource mediaDataSource = DatabaseFactory.getAttachmentDatabase(context)
.mediaDataSourceFor(partId);
if (mediaDataSource == null) {
throw new AssertionError();
}
return new VideoInput.MediaDataSourceVideoInput(mediaDataSource);
}
}

33
app/src/main/java/org/thoughtcrime/securesms/video/InMemoryTranscoder.java
View File

@@ -15,6 +15,7 @@ import org.thoughtcrime.securesms.mms.MediaStream;
import org.thoughtcrime.securesms.util.MemoryFileDescriptor;
import org.thoughtcrime.securesms.video.videoconverter.EncodingException;
import org.thoughtcrime.securesms.video.videoconverter.MediaConverter;
import org.thoughtcrime.securesms.video.videoconverter.VideoInput;
import java.io.Closeable;
import java.io.FileDescriptor;
@@ -46,15 +47,17 @@ public final class InMemoryTranscoder implements Closeable {
private final boolean transcodeRequired;
private final long fileSizeEstimate;
private final int outputFormat;
private final @Nullable Options options;
private @Nullable MemoryFileDescriptor memoryFile;
/**
* @param upperSizeLimit A upper size to transcode to. The actual output size can be up to 10% smaller.
*/
public InMemoryTranscoder(@NonNull Context context, @NonNull MediaDataSource dataSource, long upperSizeLimit) throws IOException, VideoSourceException {
public InMemoryTranscoder(@NonNull Context context, @NonNull MediaDataSource dataSource, @Nullable Options options, long upperSizeLimit) throws IOException, VideoSourceException {
this.context = context;
this.dataSource = dataSource;
this.options = options;
final MediaMetadataRetriever mediaMetadataRetriever = new MediaMetadataRetriever();
try {
@@ -72,9 +75,9 @@ public final class InMemoryTranscoder implements Closeable {
this.targetVideoBitRate = getTargetVideoBitRate(upperSizeLimitWithMargin, duration);
this.upperSizeLimit = upperSizeLimit;
this.transcodeRequired = inputBitRate >= targetVideoBitRate * 1.2 || inSize > upperSizeLimit || containsLocation(mediaMetadataRetriever);
this.transcodeRequired = inputBitRate >= targetVideoBitRate * 1.2 || inSize > upperSizeLimit || containsLocation(mediaMetadataRetriever) || options != null;
if (!transcodeRequired) {
Log.i(TAG, "Video is within 20% of target bitrate, below the size limit and contained no location metadata.");
Log.i(TAG, "Video is within 20% of target bitrate, below the size limit, contained no location metadata or custom options.");
}
this.fileSizeEstimate = (targetVideoBitRate + AUDIO_BITRATE) * duration / 8000;
@@ -84,7 +87,8 @@ public final class InMemoryTranscoder implements Closeable {
: OUTPUT_FORMAT;
}
public @NonNull MediaStream transcode(@NonNull Progress progress, @Nullable CancelationSignal cancelationSignal)
public @NonNull MediaStream transcode(@NonNull Progress progress,
@Nullable CancelationSignal cancelationSignal)
throws IOException, EncodingException, VideoSizeException
{
if (memoryFile != null) throw new AssertionError("Not expecting to reuse transcoder");
@@ -125,12 +129,21 @@ public final class InMemoryTranscoder implements Closeable {
final MediaConverter converter = new MediaConverter();
converter.setInput(dataSource);
converter.setInput(new VideoInput.MediaDataSourceVideoInput(dataSource));
converter.setOutput(memoryFileFileDescriptor);
converter.setVideoResolution(outputFormat);
converter.setVideoBitrate(targetVideoBitRate);
converter.setAudioBitrate(AUDIO_BITRATE);
if (options != null) {
if (options.endTimeUs > 0) {
long timeFrom = options.startTimeUs / 1000;
long timeTo = options.endTimeUs / 1000;
converter.setTimeRange(timeFrom, timeTo);
Log.i(TAG, String.format(Locale.US, "Trimming:\nTotal duration: %d\nKeeping: %d..%d\nFinal duration:(%d)", duration, timeFrom, timeTo, timeTo - timeFrom));
}
}
converter.setListener(percent -> {
progress.onProgress(percent);
return cancelationSignal != null && cancelationSignal.isCanceled();
@@ -219,4 +232,14 @@ public final class InMemoryTranscoder implements Closeable {
public interface CancelationSignal {
boolean isCanceled();
}
public final static class Options {
final long startTimeUs;
final long endTimeUs;
public Options(long startTimeUs, long endTimeUs) {
this.startTimeUs = startTimeUs;
this.endTimeUs = endTimeUs;
}
}
}

130
app/src/main/java/org/thoughtcrime/securesms/video/VideoPlayer.java
View File

@@ -17,21 +17,24 @@
package org.thoughtcrime.securesms.video;
import android.content.Context;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import android.util.AttributeSet;
import android.view.View;
import android.view.Window;
import android.view.WindowManager;
import android.widget.FrameLayout;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import com.google.android.exoplayer2.DefaultLoadControl;
import com.google.android.exoplayer2.DefaultRenderersFactory;
import com.google.android.exoplayer2.ExoPlayerFactory;
import com.google.android.exoplayer2.LoadControl;
import com.google.android.exoplayer2.Player;
import com.google.android.exoplayer2.SimpleExoPlayer;
import com.google.android.exoplayer2.extractor.DefaultExtractorsFactory;
import com.google.android.exoplayer2.extractor.ExtractorsFactory;
import com.google.android.exoplayer2.source.ClippingMediaSource;
import com.google.android.exoplayer2.source.ExtractorMediaSource;
import com.google.android.exoplayer2.source.MediaSource;
import com.google.android.exoplayer2.trackselection.AdaptiveTrackSelection;
@@ -40,25 +43,30 @@ import com.google.android.exoplayer2.trackselection.TrackSelection;
import com.google.android.exoplayer2.trackselection.TrackSelector;
import com.google.android.exoplayer2.ui.PlayerControlView;
import com.google.android.exoplayer2.ui.PlayerView;
import com.google.android.exoplayer2.upstream.BandwidthMeter;
import com.google.android.exoplayer2.upstream.DefaultBandwidthMeter;
import com.google.android.exoplayer2.upstream.DefaultDataSourceFactory;
import org.thoughtcrime.securesms.R;
import org.thoughtcrime.securesms.logging.Log;
import org.thoughtcrime.securesms.mms.VideoSlide;
import org.thoughtcrime.securesms.util.ViewUtil;
import org.thoughtcrime.securesms.video.exo.AttachmentDataSourceFactory;
import java.util.concurrent.TimeUnit;
public class VideoPlayer extends FrameLayout {
private static final String TAG = VideoPlayer.class.getSimpleName();
@SuppressWarnings("unused")
private static final String TAG = Log.tag(VideoPlayer.class);
private final PlayerView exoView;
private final PlayerView exoView;
private final PlayerControlView exoControls;
private SimpleExoPlayer exoPlayer;
private PlayerControlView exoControls;
private Window window;
private PlayerStateCallback playerStateCallback;
private SimpleExoPlayer exoPlayer;
private Window window;
private PlayerStateCallback playerStateCallback;
private PlayerCallback playerCallback;
private boolean clipped;
private long clippedStartUs;
public VideoPlayer(Context context) {
this(context, null);
@@ -73,29 +81,49 @@ public class VideoPlayer extends FrameLayout {
inflate(context, R.layout.video_player, this);
this.exoView = ViewUtil.findById(this, R.id.video_view);
this.exoView = ViewUtil.findById(this, R.id.video_view);
this.exoControls = new PlayerControlView(getContext());
this.exoControls.setShowTimeoutMs(-1);
}
public void setVideoSource(@NonNull VideoSlide videoSource, boolean autoplay) {
BandwidthMeter bandwidthMeter = new DefaultBandwidthMeter();
TrackSelection.Factory videoTrackSelectionFactory = new AdaptiveTrackSelection.Factory(bandwidthMeter);
TrackSelector trackSelector = new DefaultTrackSelector(videoTrackSelectionFactory);
LoadControl loadControl = new DefaultLoadControl();
private CreateMediaSource createMediaSource;
exoPlayer = ExoPlayerFactory.newSimpleInstance(getContext(), trackSelector, loadControl);
public void setVideoSource(@NonNull VideoSlide videoSource, boolean autoplay) {
Context context = getContext();
DefaultRenderersFactory renderersFactory = new DefaultRenderersFactory(context);
TrackSelection.Factory videoTrackSelectionFactory = new AdaptiveTrackSelection.Factory();
TrackSelector trackSelector = new DefaultTrackSelector(videoTrackSelectionFactory);
LoadControl loadControl = new DefaultLoadControl();
exoPlayer = ExoPlayerFactory.newSimpleInstance(context, renderersFactory, trackSelector, loadControl);
exoPlayer.addListener(new ExoPlayerListener(window, playerStateCallback));
exoPlayer.addListener(new Player.DefaultEventListener() {
@Override
public void onPlayerStateChanged(boolean playWhenReady, int playbackState) {
if (playerCallback != null) {
switch (playbackState) {
case Player.STATE_READY:
if (playWhenReady) playerCallback.onPlaying();
break;
case Player.STATE_ENDED:
playerCallback.onStopped();
break;
}
}
}
});
exoView.setPlayer(exoPlayer);
exoControls.setPlayer(exoPlayer);
DefaultDataSourceFactory defaultDataSourceFactory = new DefaultDataSourceFactory(getContext(), "GenericUserAgent", null);
AttachmentDataSourceFactory attachmentDataSourceFactory = new AttachmentDataSourceFactory(getContext(), defaultDataSourceFactory, null);
DefaultDataSourceFactory defaultDataSourceFactory = new DefaultDataSourceFactory(context, "GenericUserAgent", null);
AttachmentDataSourceFactory attachmentDataSourceFactory = new AttachmentDataSourceFactory(context, defaultDataSourceFactory, null);
ExtractorsFactory extractorsFactory = new DefaultExtractorsFactory();
MediaSource mediaSource = new ExtractorMediaSource(videoSource.getUri(), attachmentDataSourceFactory, extractorsFactory, null, null);
createMediaSource = () -> new ExtractorMediaSource.Factory(attachmentDataSourceFactory)
.setExtractorsFactory(extractorsFactory)
.createMediaSource(videoSource.getUri());
exoPlayer.prepare(mediaSource);
exoPlayer.prepare(createMediaSource.create());
exoPlayer.setPlayWhenReady(autoplay);
}
@@ -142,6 +170,40 @@ public class VideoPlayer extends FrameLayout {
return 0L;
}
public long getPlaybackPositionUs() {
if (this.exoPlayer != null) {
return TimeUnit.MILLISECONDS.toMicros(this.exoPlayer.getCurrentPosition()) + clippedStartUs;
}
return 0L;
}
public void setPlaybackPosition(long positionMs) {
if (this.exoPlayer != null) {
this.exoPlayer.seekTo(positionMs);
}
}
public void clip(long fromUs, long toUs, boolean playWhenReady) {
if (this.exoPlayer != null && createMediaSource != null) {
MediaSource clippedMediaSource = new ClippingMediaSource(createMediaSource.create(), fromUs, toUs);
exoPlayer.prepare(clippedMediaSource);
exoPlayer.setPlayWhenReady(playWhenReady);
clipped = true;
clippedStartUs = fromUs;
}
}
public void removeClip(boolean playWhenReady) {
if (exoPlayer != null && createMediaSource != null) {
if (clipped) {
exoPlayer.prepare(createMediaSource.create());
clipped = false;
clippedStartUs = 0;
}
exoPlayer.setPlayWhenReady(playWhenReady);
}
}
public void setWindow(@Nullable Window window) {
this.window = window;
}
@@ -150,12 +212,23 @@ public class VideoPlayer extends FrameLayout {
this.playerStateCallback = playerStateCallback;
}
public void setPlayerCallback(PlayerCallback playerCallback) {
this.playerCallback = playerCallback;
}
public void playFromStart() {
if (exoPlayer != null) {
exoPlayer.setPlayWhenReady(true);
exoPlayer.seekTo(0);
}
}
private static class ExoPlayerListener extends Player.DefaultEventListener {
private final Window window;
private final Window window;
private final PlayerStateCallback playerStateCallback;
ExoPlayerListener(Window window, PlayerStateCallback playerStateCallback) {
this.window = window;
this.window = window;
this.playerStateCallback = playerStateCallback;
}
@@ -188,4 +261,15 @@ public class VideoPlayer extends FrameLayout {
public interface PlayerStateCallback {
void onPlayerReady();
}
public interface PlayerCallback {
void onPlaying();
void onStopped();
}
private interface CreateMediaSource {
MediaSource create();
}
}

3
app/src/main/java/org/thoughtcrime/securesms/video/videoconverter/AudioTrackConverter.java
View File

@@ -62,7 +62,7 @@ final class AudioTrackConverter {
static @Nullable
AudioTrackConverter create(
final @NonNull MediaConverter.Input input,
final @NonNull VideoInput input,
final long timeFrom,
final long timeTo,
final int audioBitrate) throws IOException {
@@ -106,6 +106,7 @@ final class AudioTrackConverter {
inputAudioFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT));
outputAudioFormat.setInteger(MediaFormat.KEY_BIT_RATE, audioBitrate);
outputAudioFormat.setInteger(MediaFormat.KEY_AAC_PROFILE, OUTPUT_AUDIO_AAC_PROFILE);
outputAudioFormat.setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, 16 * 1024);
// Create a MediaCodec for the desired codec, then configure it as an encoder with
// our desired properties. Request a Surface to use for input.

81
app/src/main/java/org/thoughtcrime/securesms/video/videoconverter/MediaConverter.java
View File

@@ -18,13 +18,9 @@
package org.thoughtcrime.securesms.video.videoconverter;
import android.content.Context;
import android.media.MediaCodecInfo;
import android.media.MediaCodecList;
import android.media.MediaDataSource;
import android.media.MediaExtractor;
import android.media.MediaFormat;
import android.net.Uri;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
@@ -53,7 +49,7 @@ public final class MediaConverter {
public static final String VIDEO_CODEC_H264 = "video/avc";
public static final String VIDEO_CODEC_H265 = "video/hevc";
private Input mInput;
private VideoInput mInput;
private Output mOutput;
private long mTimeFrom;
@@ -73,20 +69,8 @@ public final class MediaConverter {
public MediaConverter() {
}
@SuppressWarnings("unused")
public void setInput(final @NonNull File file) {
mInput = new FileInput(file);
}
@SuppressWarnings("unused")
public void setInput(final @NonNull Context context, final @NonNull Uri uri) {
mInput = new UriInput(context, uri);
}
@RequiresApi(23)
@SuppressWarnings("unused")
public void setInput(final @NonNull MediaDataSource mediaDataSource) {
mInput = new MediaDataSourceInput(mediaDataSource);
public void setInput(final @NonNull VideoInput videoInput) {
mInput = videoInput;
}
@SuppressWarnings("unused")
@@ -312,65 +296,6 @@ public final class MediaConverter {
return null;
}
interface Input {
@NonNull
MediaExtractor createExtractor() throws IOException;
}
private static class FileInput implements Input {
final File file;
FileInput(final @NonNull File file) {
this.file = file;
}
@Override
public @NonNull
MediaExtractor createExtractor() throws IOException {
final MediaExtractor extractor = new MediaExtractor();
extractor.setDataSource(file.getAbsolutePath());
return extractor;
}
}
private static class UriInput implements Input {
final Uri uri;
final Context context;
UriInput(final @NonNull Context context, final @NonNull Uri uri) {
this.uri = uri;
this.context = context;
}
@Override
public @NonNull
MediaExtractor createExtractor() throws IOException {
final MediaExtractor extractor = new MediaExtractor();
extractor.setDataSource(context, uri, null);
return extractor;
}
}
@RequiresApi(23)
private static class MediaDataSourceInput implements Input {
private final MediaDataSource mediaDataSource;
MediaDataSourceInput(final @NonNull MediaDataSource mediaDataSource) {
this.mediaDataSource = mediaDataSource;
}
@Override
public @NonNull
MediaExtractor createExtractor() throws IOException {
final MediaExtractor extractor = new MediaExtractor();
extractor.setDataSource(mediaDataSource);
return extractor;
}
}
interface Output {
@NonNull
Muxer createMuxer() throws IOException;

10
app/src/main/java/org/thoughtcrime/securesms/video/videoconverter/OutputSurface.java
View File

@@ -69,7 +69,7 @@ final class OutputSurface implements SurfaceTexture.OnFrameAvailableListener {
* EGL context and surface will be made current. Creates a Surface that can be passed
* to MediaCodec.configure().
*/
OutputSurface(int width, int height) throws TranscodingException {
OutputSurface(int width, int height, boolean flipX) throws TranscodingException {
if (width <= 0 || height <= 0) {
throw new IllegalArgumentException();
}
@@ -77,7 +77,7 @@ final class OutputSurface implements SurfaceTexture.OnFrameAvailableListener {
eglSetup(width, height);
makeCurrent();
setup();
setup(flipX);
}
/**
@@ -85,15 +85,15 @@ final class OutputSurface implements SurfaceTexture.OnFrameAvailableListener {
* passed to MediaCodec.configure().
*/
OutputSurface() throws TranscodingException {
setup();
setup(false);
}
/**
* Creates instances of TextureRender and SurfaceTexture, and a Surface associated
* with the SurfaceTexture.
*/
private void setup() throws TranscodingException {
mTextureRender = new TextureRender();
private void setup(boolean flipX) throws TranscodingException {
mTextureRender = new TextureRender(flipX);
mTextureRender.surfaceCreated();
// Even if we don't access the SurfaceTexture after the constructor returns, we

15
app/src/main/java/org/thoughtcrime/securesms/video/videoconverter/TextureRender.java
View File

@@ -47,6 +47,14 @@ final class TextureRender {
1.0f, 1.0f, 0, 1.f, 1.f,
};
private final float[] mTriangleVerticesDataFlippedX = {
// X, Y, Z, U, V
-1.0f, -1.0f, 0, 1.f, 0.f,
1.0f, -1.0f, 0, 0.f, 0.f,
-1.0f, 1.0f, 0, 1.f, 1.f,
1.0f, 1.0f, 0, 0.f, 1.f,
};
private final FloatBuffer mTriangleVertices;
private static final String VERTEX_SHADER =
@@ -79,11 +87,12 @@ final class TextureRender {
private int maPositionHandle;
private int maTextureHandle;
TextureRender() {
TextureRender(boolean flipX) {
float[] verticesData = flipX ? mTriangleVerticesDataFlippedX : mTriangleVerticesData;
mTriangleVertices = ByteBuffer.allocateDirect(
mTriangleVerticesData.length * FLOAT_SIZE_BYTES)
verticesData.length * FLOAT_SIZE_BYTES)
.order(ByteOrder.nativeOrder()).asFloatBuffer();
mTriangleVertices.put(mTriangleVerticesData).position(0);
mTriangleVertices.put(verticesData).position(0);
Matrix.setIdentityM(mSTMatrix, 0);
}

83
app/src/main/java/org/thoughtcrime/securesms/video/videoconverter/VideoInput.java Normal file
View File

@@ -0,0 +1,83 @@
package org.thoughtcrime.securesms.video.videoconverter;
import android.content.Context;
import android.media.MediaDataSource;
import android.media.MediaExtractor;
import android.net.Uri;
import androidx.annotation.NonNull;
import androidx.annotation.RequiresApi;
import java.io.Closeable;
import java.io.File;
import java.io.IOException;
public abstract class VideoInput implements Closeable {
@NonNull
abstract MediaExtractor createExtractor() throws IOException;
public static class FileVideoInput extends VideoInput {
final File file;
public FileVideoInput(final @NonNull File file) {
this.file = file;
}
@Override
public @NonNull MediaExtractor createExtractor() throws IOException {
final MediaExtractor extractor = new MediaExtractor();
extractor.setDataSource(file.getAbsolutePath());
return extractor;
}
@Override
public void close() {
}
}
public static class UriVideoInput extends VideoInput {
final Uri uri;
final Context context;
public UriVideoInput(final @NonNull Context context, final @NonNull Uri uri) {
this.uri = uri;
this.context = context;
}
@Override
public @NonNull MediaExtractor createExtractor() throws IOException {
final MediaExtractor extractor = new MediaExtractor();
extractor.setDataSource(context, uri, null);
return extractor;
}
@Override
public void close() {
}
}
@RequiresApi(23)
public static class MediaDataSourceVideoInput extends VideoInput {
private final MediaDataSource mediaDataSource;
public MediaDataSourceVideoInput(final @NonNull MediaDataSource mediaDataSource) {
this.mediaDataSource = mediaDataSource;
}
@Override
public @NonNull MediaExtractor createExtractor() throws IOException {
final MediaExtractor extractor = new MediaExtractor();
extractor.setDataSource(mediaDataSource);
return extractor;
}
@Override
public void close() throws IOException {
mediaDataSource.close();
}
}
}

182
app/src/main/java/org/thoughtcrime/securesms/video/videoconverter/VideoThumbnailsExtractor.java Normal file
View File

@@ -0,0 +1,182 @@
package org.thoughtcrime.securesms.video.videoconverter;
import android.graphics.Bitmap;
import android.media.MediaCodec;
import android.media.MediaExtractor;
import android.media.MediaFormat;
import android.opengl.GLES20;
import androidx.annotation.NonNull;
import androidx.annotation.RequiresApi;
import org.thoughtcrime.securesms.logging.Log;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
@RequiresApi(api = 23)
final class VideoThumbnailsExtractor {
private static final String TAG = Log.tag(VideoThumbnailsExtractor.class);
interface Callback {
void durationKnown(long duration);
boolean publishProgress(int index, Bitmap thumbnail);
void failed();
}
static void extractThumbnails(final @NonNull VideoInput input,
final int thumbnailCount,
final int thumbnailResolution,
final @NonNull Callback callback)
{
MediaExtractor extractor = null;
MediaCodec decoder = null;
OutputSurface outputSurface = null;
try {
extractor = input.createExtractor();
MediaFormat mediaFormat = null;
for (int index = 0; index < extractor.getTrackCount(); ++index) {
if (extractor.getTrackFormat(index).getString(MediaFormat.KEY_MIME).startsWith("video/")) {
extractor.selectTrack(index);
mediaFormat = extractor.getTrackFormat(index);
break;
}
}
if (mediaFormat != null) {
final String mime = mediaFormat.getString(MediaFormat.KEY_MIME);
final int rotation = mediaFormat.containsKey(MediaFormat.KEY_ROTATION) ? mediaFormat.getInteger(MediaFormat.KEY_ROTATION) : 0;
final int width = mediaFormat.getInteger(MediaFormat.KEY_WIDTH);
final int height = mediaFormat.getInteger(MediaFormat.KEY_HEIGHT);
final int outputWidth;
final int outputHeight;
if (width < height) {
outputWidth = thumbnailResolution;
outputHeight = height * outputWidth / width;
} else {
outputHeight = thumbnailResolution;
outputWidth = width * outputHeight / height;
}
final int outputWidthRotated;
final int outputHeightRotated;
if ((rotation % 180 == 90)) {
//noinspection SuspiciousNameCombination
outputWidthRotated = outputHeight;
//noinspection SuspiciousNameCombination
outputHeightRotated = outputWidth;
} else {
outputWidthRotated = outputWidth;
outputHeightRotated = outputHeight;
}
Log.i(TAG, "video: " + width + "x" + height + " " + rotation);
Log.i(TAG, "output: " + outputWidthRotated + "x" + outputHeightRotated);
outputSurface = new OutputSurface(outputWidthRotated, outputHeightRotated, true);
decoder = MediaCodec.createDecoderByType(mime);
decoder.configure(mediaFormat, outputSurface.getSurface(), null, 0);
decoder.start();
long duration = mediaFormat.getLong(MediaFormat.KEY_DURATION);
callback.durationKnown(duration);
doExtract(extractor, decoder, outputSurface, outputWidthRotated, outputHeightRotated, duration, thumbnailCount, callback);
}
} catch (IOException | TranscodingException e) {
Log.w(TAG, e);
callback.failed();
} finally {
if (outputSurface != null) {
outputSurface.release();
}
if (decoder != null) {
decoder.stop();
decoder.release();
}
if (extractor != null) {
extractor.release();
}
}
}
private static void doExtract(final @NonNull MediaExtractor extractor,
final @NonNull MediaCodec decoder,
final @NonNull OutputSurface outputSurface,
final int outputWidth, int outputHeight, long duration, int thumbnailCount,
final @NonNull Callback callback)
throws TranscodingException
{
final int TIMEOUT_USEC = 10000;
final ByteBuffer[] decoderInputBuffers = decoder.getInputBuffers();
final MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
int samplesExtracted = 0;
int thumbnailsCreated = 0;
Log.i(TAG, "doExtract started");
final ByteBuffer pixelBuf = ByteBuffer.allocateDirect(outputWidth * outputHeight * 4);
pixelBuf.order(ByteOrder.LITTLE_ENDIAN);
boolean outputDone = false;
boolean inputDone = false;
while (!outputDone) {
if (!inputDone) {
int inputBufIndex = decoder.dequeueInputBuffer(TIMEOUT_USEC);
if (inputBufIndex >= 0) {
final ByteBuffer inputBuf = decoderInputBuffers[inputBufIndex];
final int sampleSize = extractor.readSampleData(inputBuf, 0);
if (sampleSize < 0 || samplesExtracted >= thumbnailCount) {
decoder.queueInputBuffer(inputBufIndex, 0, 0, 0L, MediaCodec.BUFFER_FLAG_END_OF_STREAM);
inputDone = true;
Log.i(TAG, "input done");
} else {
final long presentationTimeUs = extractor.getSampleTime();
decoder.queueInputBuffer(inputBufIndex, 0, sampleSize, presentationTimeUs, 0 /*flags*/);
samplesExtracted++;
extractor.seekTo(duration * samplesExtracted / thumbnailCount, MediaExtractor.SEEK_TO_CLOSEST_SYNC);
Log.i(TAG, "seek to " + duration * samplesExtracted / thumbnailCount + ", actual " + extractor.getSampleTime());
}
}
}
int outputBufIndex = decoder.dequeueOutputBuffer(info, TIMEOUT_USEC);
if (outputBufIndex >= 0) {
if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
outputDone = true;
}
final boolean shouldRender = (info.size != 0) /*&& (info.presentationTimeUs >= duration * decodeCount / thumbnailCount)*/;
decoder.releaseOutputBuffer(outputBufIndex, shouldRender);
if (shouldRender) {
outputSurface.awaitNewImage();
outputSurface.drawImage();
if (thumbnailsCreated < thumbnailCount) {
pixelBuf.rewind();
GLES20.glReadPixels(0, 0, outputWidth, outputHeight, GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, pixelBuf);
final Bitmap bitmap = Bitmap.createBitmap(outputWidth, outputHeight, Bitmap.Config.ARGB_8888);
pixelBuf.rewind();
bitmap.copyPixelsFromBuffer(pixelBuf);
if (!callback.publishProgress(thumbnailsCreated, bitmap)) {
break;
}
Log.i(TAG, "publishProgress for frame " + thumbnailsCreated + " at " + info.presentationTimeUs + " (target " + duration * thumbnailsCreated / thumbnailCount + ")");
}
thumbnailsCreated++;
}
}
}
Log.i(TAG, "doExtract finished");
}
}

363
app/src/main/java/org/thoughtcrime/securesms/video/videoconverter/VideoThumbnailsRangeSelectorView.java Normal file
View File

@@ -0,0 +1,363 @@
package org.thoughtcrime.securesms.video.videoconverter;
import android.content.Context;
import android.content.res.TypedArray;
import android.graphics.Canvas;
import android.graphics.Paint;
import android.graphics.Rect;
import android.graphics.drawable.Drawable;
import android.util.AttributeSet;
import android.view.MotionEvent;
import androidx.annotation.ColorInt;
import androidx.annotation.Nullable;
import androidx.annotation.Px;
import androidx.annotation.RequiresApi;
import androidx.vectordrawable.graphics.drawable.VectorDrawableCompat;
import org.thoughtcrime.securesms.R;
import java.util.concurrent.TimeUnit;
@RequiresApi(api = 23)
public final class VideoThumbnailsRangeSelectorView extends VideoThumbnailsView {
private static final long MINIMUM_SELECTABLE_RANGE = TimeUnit.MILLISECONDS.toMicros(500);
private final Paint paint = new Paint(Paint.ANTI_ALIAS_FLAG);
private final Paint paintGrey = new Paint(Paint.ANTI_ALIAS_FLAG);
private final Rect tempDrawRect = new Rect();
private Drawable chevronLeft;
private Drawable chevronRight;
@Px private int left;
@Px private int right;
@Px private int cursor;
private Long minValue;
private Long maxValue;
private Long externalMinValue;
private Long externalMaxValue;
private float xDown;
private long downCursor;
private long downMin;
private long downMax;
private Thumb dragThumb;
private OnRangeChangeListener onRangeChangeListener;
@Px private int thumbSizePixels;
@Px private int thumbTouchRadius;
@Px private int cursorPixels;
@ColorInt private int cursorColor;
@ColorInt private int thumbColor;
@ColorInt private int thumbColorEdited;
private long actualPosition;
private long dragPosition;
public VideoThumbnailsRangeSelectorView(final Context context) {
super(context);
init(null);
}
public VideoThumbnailsRangeSelectorView(final Context context, final @Nullable AttributeSet attrs) {
super(context, attrs);
init(attrs);
}
public VideoThumbnailsRangeSelectorView(final Context context, final @Nullable AttributeSet attrs, final int defStyleAttr) {
super(context, attrs, defStyleAttr);
init(attrs);
}
private void init(final @Nullable AttributeSet attrs) {
if (attrs != null) {
TypedArray typedArray = getContext().getTheme().obtainStyledAttributes(attrs, R.styleable.VideoThumbnailsRangeSelectorView, 0, 0);
thumbSizePixels = typedArray.getDimensionPixelSize(R.styleable.VideoThumbnailsRangeSelectorView_thumbWidth, 1);
cursorPixels = typedArray.getDimensionPixelSize(R.styleable.VideoThumbnailsRangeSelectorView_cursorWidth, 1);
thumbColor = typedArray.getColor(R.styleable.VideoThumbnailsRangeSelectorView_thumbColor, 0xffff0000);
thumbColorEdited = typedArray.getColor(R.styleable.VideoThumbnailsRangeSelectorView_thumbColorEdited, thumbColor);
cursorColor = typedArray.getColor(R.styleable.VideoThumbnailsRangeSelectorView_cursorColor, thumbColor);
thumbTouchRadius = typedArray.getDimensionPixelSize(R.styleable.VideoThumbnailsRangeSelectorView_thumbTouchRadius, 50);
}
chevronLeft = VectorDrawableCompat.create(getResources(), R.drawable.ic_chevron_left_black_8dp, null);
chevronRight = VectorDrawableCompat.create(getResources(), R.drawable.ic_chevron_right_black_8dp, null);
paintGrey.setColor(0x7f000000);
paintGrey.setStyle(Paint.Style.FILL_AND_STROKE);
paintGrey.setStrokeWidth(1);
paint.setStrokeWidth(2);
}
@Override
protected void afterDurationChange(long duration) {
super.afterDurationChange(duration);
if (maxValue != null && duration < maxValue) {
maxValue = duration;
}
if (minValue != null && duration < minValue) {
minValue = duration;
}
if (duration > 0) {
if (externalMinValue != null) {
setMinMax(externalMinValue, getMaxValue(), Thumb.MIN);
externalMinValue = null;
}
if (externalMaxValue != null) {
setMinMax(getMinValue(), externalMaxValue, Thumb.MAX);
externalMaxValue = null;
}
}
invalidate();
}
public void setOnRangeChangeListener(OnRangeChangeListener onRangeChangeListener) {
this.onRangeChangeListener = onRangeChangeListener;
}
public void setActualPosition(long position) {
if (this.actualPosition != position) {
this.actualPosition = position;
invalidate();
}
}
private void setDragPosition(long position) {
if (this.dragPosition != position) {
this.dragPosition = Math.max(getMinValue(), Math.min(getMaxValue(), position));
invalidate();
}
}
@Override
protected void onDraw(final Canvas canvas) {
super.onDraw(canvas);
canvas.translate(getPaddingLeft(), getPaddingTop());
int drawableWidth = getDrawableWidth();
int drawableHeight = getDrawableHeight();
long duration = getDuration();
long min = getMinValue();
long max = getMaxValue();
boolean edited = min != 0 || max != duration;
long drawPosAt = dragThumb == Thumb.POSITION ? dragPosition : actualPosition;
left = duration != 0 ? (int) ((min * drawableWidth) / duration) : 0;
right = duration != 0 ? (int) ((max * drawableWidth) / duration) : drawableWidth;
cursor = duration != 0 ? (int) ((drawPosAt * drawableWidth) / duration) : drawableWidth;
// draw greyed out areas
tempDrawRect.set(0, 0, left - 1, drawableHeight);
canvas.drawRect(tempDrawRect, paintGrey);
tempDrawRect.set(right + 1, 0, drawableWidth, drawableHeight);
canvas.drawRect(tempDrawRect, paintGrey);
// draw area rectangle
paint.setStyle(Paint.Style.STROKE);
tempDrawRect.set(left, 0, right, drawableHeight);
paint.setColor(edited ? thumbColorEdited : thumbColor);
canvas.drawRect(tempDrawRect, paint);
// draw thumb rectangles
paint.setStyle(Paint.Style.FILL_AND_STROKE);
tempDrawRect.set(left, 0, left + thumbSizePixels, drawableHeight);
canvas.drawRect(tempDrawRect, paint);
tempDrawRect.set(right - thumbSizePixels, 0, right, drawableHeight);
canvas.drawRect(tempDrawRect, paint);
int arrowSize = Math.min(drawableHeight, thumbSizePixels * 2);
chevronLeft .setBounds(0, 0, arrowSize, arrowSize);
chevronRight.setBounds(0, 0, arrowSize, arrowSize);
float dy = (drawableHeight - arrowSize) / 2f;
float arrowPaddingX = (thumbSizePixels - arrowSize) / 2f;
// draw left thumb chevron
canvas.save();
canvas.translate(left + arrowPaddingX, dy);
chevronLeft.draw(canvas);
canvas.restore();
// draw right thumb chevron
canvas.save();
canvas.translate(right - thumbSizePixels + arrowPaddingX, dy);
chevronRight.draw(canvas);
canvas.restore();
// draw current position marker
if (left <= cursor && cursor <= right && dragThumb != Thumb.MIN && dragThumb != Thumb.MAX) {
canvas.translate(cursorPixels / 2, 0);
tempDrawRect.set(cursor, 0, cursor + cursorPixels, drawableHeight);
paint.setColor(cursorColor);
canvas.drawRect(tempDrawRect, paint);
}
}
public long getMinValue() {
return minValue == null ? 0 : minValue;
}
public long getMaxValue() {
return maxValue == null ? getDuration() : maxValue;
}
private boolean setMinValue(long minValue) {
if (this.minValue == null || this.minValue != minValue) {
return setMinMax(minValue, getMaxValue(), Thumb.MIN);
} else{
return false;
}
}
public boolean setMaxValue(long maxValue) {
if (this.maxValue == null || this.maxValue != maxValue) {
return setMinMax(getMinValue(), maxValue, Thumb.MAX);
} else{
return false;
}
}
private boolean setMinMax(long newMin, long newMax, Thumb thumb) {
final long currentMin = getMinValue();
final long currentMax = getMaxValue();
final long duration = getDuration();
final long minDiff = Math.max(MINIMUM_SELECTABLE_RANGE, pixelToDuration(thumbSizePixels * 2.5f));
if (thumb == Thumb.MIN) {
newMin = clamp(newMin, 0, currentMax - minDiff);
} else {
newMax = clamp(newMax, currentMin + minDiff, duration);
}
if (newMin != currentMin || newMax != currentMax) {
this.minValue = newMin;
this.maxValue = newMax;
invalidate();
return true;
}
return false;
}
private static long clamp(long value, long min, long max) {
return Math.min(Math.max(min, value), max);
}
@Override
public boolean onTouchEvent(MotionEvent event) {
int actionMasked = event.getActionMasked();
if (actionMasked == MotionEvent.ACTION_DOWN) {
xDown = event.getX();
downCursor = actualPosition;
downMin = getMinValue();
downMax = getMaxValue();
dragThumb = closestThumb(event.getX());
return dragThumb != null;
}
if (actionMasked == MotionEvent.ACTION_MOVE) {
boolean changed = false;
long delta = pixelToDuration(event.getX() - xDown);
switch (dragThumb) {
case POSITION:
setDragPosition(downCursor + delta);
changed = true;
break;
case MIN:
changed = setMinValue(downMin + delta);
break;
case MAX:
changed = setMaxValue(downMax + delta);
break;
}
if (changed && onRangeChangeListener != null) {
if (dragThumb == Thumb.POSITION) {
onRangeChangeListener.onPositionDrag(dragPosition);
} else {
onRangeChangeListener.onRangeDrag(getMinValue(), getMaxValue(), getDuration(), dragThumb);
}
}
return true;
}
if (actionMasked == MotionEvent.ACTION_UP) {
if (onRangeChangeListener != null) {
if (dragThumb == Thumb.POSITION) {
onRangeChangeListener.onEndPositionDrag(dragPosition);
} else {
onRangeChangeListener.onRangeDragEnd(getMinValue(), getMaxValue(), getDuration(), dragThumb);
}
dragThumb = null;
invalidate();
}
return true;
}
if (actionMasked == MotionEvent.ACTION_CANCEL) {
dragThumb = null;
}
return true;
}
private @Nullable Thumb closestThumb(@Px float x) {
float midPoint = (right + left) / 2f;
Thumb possibleThumb = x < midPoint ? Thumb.MIN : Thumb.MAX;
int possibleThumbX = x < midPoint ? left : right;
if (Math.abs(x - possibleThumbX) < thumbTouchRadius) {
return possibleThumb;
}
return null;
}
private long pixelToDuration(float pixel) {
return (long) (pixel / getDrawableWidth() * getDuration());
}
private int getDrawableWidth() {
return getWidth() - getPaddingLeft() - getPaddingRight();
}
private int getDrawableHeight() {
return getHeight() - getPaddingBottom() - getPaddingTop();
}
public void setRange(long minValue, long maxValue) {
if (getDuration() > 0) {
setMinMax(minValue, maxValue, Thumb.MIN);
setMinMax(minValue, maxValue, Thumb.MAX);
} else {
externalMinValue = minValue;
externalMaxValue = maxValue;
}
}
public enum Thumb {
MIN,
MAX,
POSITION
}
public interface OnRangeChangeListener {
void onPositionDrag(long position);
void onEndPositionDrag(long position);
void onRangeDrag(long minValue, long maxValue, long duration, Thumb thumb);
void onRangeDragEnd(long minValue, long maxValue, long duration, Thumb thumb);
}
}

228
app/src/main/java/org/thoughtcrime/securesms/video/videoconverter/VideoThumbnailsView.java Normal file
View File

@@ -0,0 +1,228 @@
package org.thoughtcrime.securesms.video.videoconverter;
import android.content.Context;
import android.graphics.Bitmap;
import android.graphics.Canvas;
import android.graphics.Paint;
import android.graphics.Rect;
import android.graphics.RectF;
import android.os.AsyncTask;
import android.util.AttributeSet;
import android.view.View;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import androidx.annotation.RequiresApi;
import org.thoughtcrime.securesms.logging.Log;
import java.io.IOException;
import java.lang.ref.WeakReference;
import java.util.ArrayList;
import java.util.Arrays;
@RequiresApi(api = 23)
public class VideoThumbnailsView extends View {
private static final String TAG = Log.tag(VideoThumbnailsView.class);
private VideoInput input;
private ArrayList<Bitmap> thumbnails;
private AsyncTask<Void, Bitmap, Void> thumbnailsTask;
private OnDurationListener durationListener;
private final Paint paint = new Paint(Paint.ANTI_ALIAS_FLAG);
private final RectF tempRect = new RectF();
private final Rect drawRect = new Rect();
private final Rect tempDrawRect = new Rect();
private long duration = 0;
public VideoThumbnailsView(final Context context) {
super(context);
}
public VideoThumbnailsView(final Context context, final @Nullable AttributeSet attrs) {
super(context, attrs);
}
public VideoThumbnailsView(final Context context, final @Nullable AttributeSet attrs, final int defStyleAttr) {
super(context, attrs, defStyleAttr);
}
public void setInput(VideoInput input) {
this.input = input;
thumbnails = null;
if (thumbnailsTask != null) {
thumbnailsTask.cancel(true);
thumbnailsTask = null;
}
invalidate();
}
@Override
protected void onDetachedFromWindow() {
super.onDetachedFromWindow();
thumbnails = null;
if (thumbnailsTask != null) {
thumbnailsTask.cancel(true);
thumbnailsTask = null;
}
if (input != null) {
try {
input.close();
} catch (IOException e) {
Log.w(TAG, e);
}
}
}
@Override
protected void onDraw(final Canvas canvas) {
super.onDraw(canvas);
if (input == null) {
return;
}
tempDrawRect.set(getPaddingLeft(), getPaddingTop(), getWidth() - getPaddingRight(), getHeight() - getPaddingBottom());
if (!drawRect.equals(tempDrawRect)) {
drawRect.set(tempDrawRect);
thumbnails = null;
if (thumbnailsTask != null) {
thumbnailsTask.cancel(true);
thumbnailsTask = null;
}
}
if (thumbnails == null) {
if (thumbnailsTask == null) {
final int thumbnailCount = drawRect.width() / drawRect.height();
final float thumbnailWidth = (float) drawRect.width() / thumbnailCount;
final float thumbnailHeight = drawRect.height();
thumbnails = new ArrayList<>(thumbnailCount);
thumbnailsTask = new ThumbnailsTask(this, input, thumbnailWidth, thumbnailHeight, thumbnailCount);
thumbnailsTask.execute();
}
} else {
final int thumbnailCount = drawRect.width() / drawRect.height();
final float thumbnailWidth = (float) drawRect.width() / thumbnailCount;
final float thumbnailHeight = drawRect.height();
tempRect.top = drawRect.top;
tempRect.bottom = drawRect.bottom;
for (int i = 0; i < thumbnails.size(); i++) {
tempRect.left = drawRect.left + i * thumbnailWidth;
tempRect.right = tempRect.left + thumbnailWidth;
final Bitmap thumbnailBitmap = thumbnails.get(i);
if (thumbnailBitmap != null) {
canvas.save();
canvas.rotate(180, tempRect.centerX(), tempRect.centerY());
tempDrawRect.set(0, 0, thumbnailBitmap.getWidth(), thumbnailBitmap.getHeight());
if (tempDrawRect.width() * thumbnailHeight > tempDrawRect.height() * thumbnailWidth) {
float w = tempDrawRect.height() * thumbnailWidth / thumbnailHeight;
tempDrawRect.left = tempDrawRect.centerX() - (int) (w / 2);
tempDrawRect.right = tempDrawRect.left + (int) w;
} else {
float h = tempDrawRect.width() * thumbnailHeight / thumbnailWidth;
tempDrawRect.top = tempDrawRect.centerY() - (int) (h / 2);
tempDrawRect.bottom = tempDrawRect.top + (int) h;
}
canvas.drawBitmap(thumbnailBitmap, tempDrawRect, tempRect, paint);
canvas.restore();
}
}
}
}
public void setDurationListener(OnDurationListener durationListener) {
this.durationListener = durationListener;
}
private void setDuration(long duration) {
if (durationListener != null) {
durationListener.onDurationKnown(duration);
}
if (this.duration != duration) {
this.duration = duration;
afterDurationChange(duration);
}
}
protected void afterDurationChange(long duration) {
}
protected long getDuration() {
return duration;
}
private static class ThumbnailsTask extends AsyncTask<Void, Bitmap, Void> {
final WeakReference<VideoThumbnailsView> viewReference;
final VideoInput input;
final float thumbnailWidth;
final float thumbnailHeight;
final int thumbnailCount;
long duration;
ThumbnailsTask(final @NonNull VideoThumbnailsView view, final @NonNull VideoInput input, final float thumbnailWidth, final float thumbnailHeight, final int thumbnailCount) {
viewReference = new WeakReference<>(view);
this.input = input;
this.thumbnailWidth = thumbnailWidth;
this.thumbnailHeight = thumbnailHeight;
this.thumbnailCount = thumbnailCount;
}
@Override
protected Void doInBackground(Void... params) {
Log.i(TAG, "generate " + thumbnailCount + " thumbnails " + thumbnailWidth + "x" + thumbnailHeight);
VideoThumbnailsExtractor.extractThumbnails(input, thumbnailCount, (int) thumbnailHeight, new VideoThumbnailsExtractor.Callback() {
@Override
public void durationKnown(long duration) {
ThumbnailsTask.this.duration = duration;
}
@Override
public boolean publishProgress(int index, Bitmap thumbnail) {
ThumbnailsTask.this.publishProgress(thumbnail);
return !isCancelled();
}
@Override
public void failed() {
Log.w(TAG, "Thumbnail extraction failed");
}
});
return null;
}
@Override
protected void onProgressUpdate(Bitmap... values) {
final VideoThumbnailsView view = viewReference.get();
if (view != null) {
view.thumbnails.addAll(Arrays.asList(values));
view.invalidate();
}
}
@Override
protected void onPostExecute(Void result) {
final VideoThumbnailsView view = viewReference.get();
if (view != null) {
view.setDuration(duration);
view.invalidate();
Log.i(TAG, "onPostExecute, we have " + view.thumbnails.size() + " thumbs");
}
}
}
public interface OnDurationListener {
void onDurationKnown(long duration);
}
}

2
app/src/main/java/org/thoughtcrime/securesms/video/videoconverter/VideoTrackConverter.java
View File

@@ -66,7 +66,7 @@ final class VideoTrackConverter {
@RequiresApi(23)
static @Nullable VideoTrackConverter create(
final @NonNull MediaConverter.Input input,
final @NonNull VideoInput input,
final long timeFrom,
final long timeTo,
final int videoResolution,