diff --git a/app/src/main/java/org/thoughtcrime/securesms/video/videoconverter/VideoThumbnailsView.java b/app/src/main/java/org/thoughtcrime/securesms/video/videoconverter/VideoThumbnailsView.java index 63e3cffd2e..a4ba756174 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/video/videoconverter/VideoThumbnailsView.java +++ b/app/src/main/java/org/thoughtcrime/securesms/video/videoconverter/VideoThumbnailsView.java @@ -1,258 +1,258 @@ -package org.thoughtcrime.securesms.video.videoconverter; - -import android.content.Context; -import android.graphics.Bitmap; -import android.graphics.Canvas; -import android.graphics.Paint; -import android.graphics.Path; -import android.graphics.Rect; -import android.graphics.RectF; -import android.net.Uri; -import android.os.AsyncTask; -import android.util.AttributeSet; -import android.view.View; - -import androidx.annotation.NonNull; -import androidx.annotation.Nullable; -import androidx.annotation.RequiresApi; - -import org.signal.core.util.logging.Log; -import org.thoughtcrime.securesms.media.DecryptableUriMediaInput; -import org.thoughtcrime.securesms.util.ViewUtil; -import org.thoughtcrime.securesms.video.interfaces.MediaInput; - -import java.io.IOException; -import java.lang.ref.WeakReference; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -@RequiresApi(api = 23) -abstract public class VideoThumbnailsView extends View { - - private static final String TAG = Log.tag(VideoThumbnailsView.class); - private static final int CORNER_RADIUS = ViewUtil.dpToPx(8); - - protected Uri currentUri; - - private MediaInput input; - private volatile ArrayList thumbnails; - private AsyncTask thumbnailsTask; - - private final Paint paint = new Paint(Paint.ANTI_ALIAS_FLAG); - private final RectF tempRect = new RectF(); - private final Rect drawRect = new Rect(); - private final Rect tempDrawRect = new Rect(); - private long duration = 0; - - protected final Path clippingPath = new Path(); - - public VideoThumbnailsView(final Context context) { - super(context); - } - - public VideoThumbnailsView(final Context context, final @Nullable AttributeSet attrs) { - super(context, attrs); - } - - public VideoThumbnailsView(final Context context, final @Nullable AttributeSet attrs, final int defStyleAttr) { - super(context, attrs, defStyleAttr); - } - - /** - * @return Whether or not the current URI was changed. - */ - public boolean setInput(@NonNull Uri uri) throws IOException { - if (uri.equals(this.currentUri)) { - return false; - } - - this.currentUri = uri; - this.input = DecryptableUriMediaInput.createForUri(getContext(), uri); - this.thumbnails = null; - if (thumbnailsTask != null) { - thumbnailsTask.cancel(true); - thumbnailsTask = null; - } - invalidate(); - return true; - } - - @Override - protected void onDetachedFromWindow() { - super.onDetachedFromWindow(); - - thumbnails = null; - if (thumbnailsTask != null) { - thumbnailsTask.cancel(true); - thumbnailsTask = null; - } - - if (input != null) { - try { - input.close(); - } catch (IOException e) { - Log.w(TAG, e); - } - } - } - - @Override - protected void onDraw(final Canvas canvas) { - super.onDraw(canvas); - - if (input == null) { - return; - } - - final int left = getPaddingLeft(); - final int top = getPaddingTop(); - final int right = getWidth() - getPaddingRight(); - final int bottom = getHeight() - getPaddingBottom(); - - clippingPath.reset(); - clippingPath.addRoundRect(left, top, right, bottom, CORNER_RADIUS, CORNER_RADIUS, Path.Direction.CW); - - tempDrawRect.set(left, top, right, bottom); - - if (!drawRect.equals(tempDrawRect)) { - drawRect.set(tempDrawRect); - thumbnails = null; - if (thumbnailsTask != null) { - thumbnailsTask.cancel(true); - thumbnailsTask = null; - } - } - - if (thumbnails == null) { - if (thumbnailsTask == null) { - final int thumbnailCount = drawRect.width() / drawRect.height(); - final float thumbnailWidth = (float) drawRect.width() / thumbnailCount; - final float thumbnailHeight = drawRect.height(); - - thumbnails = new ArrayList<>(thumbnailCount); - thumbnailsTask = new ThumbnailsTask(this, input, thumbnailWidth, thumbnailHeight, thumbnailCount); - thumbnailsTask.execute(); - } - } else { - final int thumbnailCount = drawRect.width() / drawRect.height(); - final float thumbnailWidth = (float) drawRect.width() / thumbnailCount; - final float thumbnailHeight = drawRect.height(); - - tempRect.top = drawRect.top; - tempRect.bottom = drawRect.bottom; - canvas.save(); - - canvas.clipPath(clippingPath); - - for (int i = 0; i < thumbnails.size(); i++) { - tempRect.left = drawRect.left + i * thumbnailWidth; - tempRect.right = tempRect.left + thumbnailWidth; - - final Bitmap thumbnailBitmap = thumbnails.get(i); - if (thumbnailBitmap != null) { - canvas.save(); - canvas.rotate(180, tempRect.centerX(), tempRect.centerY()); - tempDrawRect.set(0, 0, thumbnailBitmap.getWidth(), thumbnailBitmap.getHeight()); - if (tempDrawRect.width() * thumbnailHeight > tempDrawRect.height() * thumbnailWidth) { - float w = tempDrawRect.height() * thumbnailWidth / thumbnailHeight; - tempDrawRect.left = tempDrawRect.centerX() - (int) (w / 2); - tempDrawRect.right = tempDrawRect.left + (int) w; - } else { - float h = tempDrawRect.width() * thumbnailHeight / thumbnailWidth; - tempDrawRect.top = tempDrawRect.centerY() - (int) (h / 2); - tempDrawRect.bottom = tempDrawRect.top + (int) h; - } - canvas.drawBitmap(thumbnailBitmap, tempDrawRect, tempRect, paint); - canvas.restore(); - } - } - - canvas.restore(); - } - } - - private void setDuration(long duration) { - if (this.duration != duration) { - this.duration = duration; - afterDurationChange(duration); - } - } - - abstract void afterDurationChange(long duration); - - public long getDuration() { - return duration; - } - - private static class ThumbnailsTask extends AsyncTask { - - final WeakReference viewReference; - final MediaInput input; - final float thumbnailWidth; - final float thumbnailHeight; - final int thumbnailCount; - - long duration; - - ThumbnailsTask(final @NonNull VideoThumbnailsView view, final @NonNull MediaInput input, final float thumbnailWidth, final float thumbnailHeight, final int thumbnailCount) { - this.viewReference = new WeakReference<>(view); - this.input = input; - this.thumbnailWidth = thumbnailWidth; - this.thumbnailHeight = thumbnailHeight; - this.thumbnailCount = thumbnailCount; - } - - @Override - protected Void doInBackground(Void... params) { - Log.i(TAG, "generate " + thumbnailCount + " thumbnails " + thumbnailWidth + "x" + thumbnailHeight); - VideoThumbnailsExtractor.extractThumbnails(input, thumbnailCount, (int) thumbnailHeight, new VideoThumbnailsExtractor.Callback() { - - @Override - public void durationKnown(long duration) { - ThumbnailsTask.this.duration = duration; - } - - @Override - public boolean publishProgress(int index, Bitmap thumbnail) { - boolean notCanceled = !isCancelled(); - if (notCanceled) { - ThumbnailsTask.this.publishProgress(thumbnail); - } - return notCanceled; - } - - @Override - public void failed() { - Log.w(TAG, "Thumbnail extraction failed"); - } - }); - return null; - } - - @Override - protected void onProgressUpdate(Bitmap... values) { - if (isCancelled()) { - return; - } - - VideoThumbnailsView view = viewReference.get(); - List thumbnails = view != null ? view.thumbnails : null; - if (thumbnails != null) { - thumbnails.addAll(Arrays.asList(values)); - view.invalidate(); - } - } - - @Override - protected void onPostExecute(Void result) { - VideoThumbnailsView view = viewReference.get(); - List thumbnails = view != null ? view.thumbnails : null; - if (view != null) { - view.setDuration(ThumbnailsTask.this.duration); - view.invalidate(); - Log.i(TAG, "onPostExecute, we have " + (thumbnails != null ? thumbnails.size() : "null") + " thumbs"); - } - } - } -} +package org.thoughtcrime.securesms.video.videoconverter; + +import android.content.Context; +import android.graphics.Bitmap; +import android.graphics.Canvas; +import android.graphics.Paint; +import android.graphics.Path; +import android.graphics.Rect; +import android.graphics.RectF; +import android.net.Uri; +import android.os.AsyncTask; +import android.util.AttributeSet; +import android.view.View; + +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; +import androidx.annotation.RequiresApi; + +import org.signal.core.util.logging.Log; +import org.thoughtcrime.securesms.media.DecryptableUriMediaInput; +import org.thoughtcrime.securesms.util.ViewUtil; +import org.thoughtcrime.securesms.video.interfaces.MediaInput; + +import java.io.IOException; +import java.lang.ref.WeakReference; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +@RequiresApi(api = 23) +abstract public class VideoThumbnailsView extends View { + + private static final String TAG = Log.tag(VideoThumbnailsView.class); + private static final int CORNER_RADIUS = ViewUtil.dpToPx(8); + + protected Uri currentUri; + + private MediaInput input; + private volatile ArrayList thumbnails; + private AsyncTask thumbnailsTask; + + private final Paint paint = new Paint(Paint.ANTI_ALIAS_FLAG); + private final RectF tempRect = new RectF(); + private final Rect drawRect = new Rect(); + private final Rect tempDrawRect = new Rect(); + private long duration = 0; + + protected final Path clippingPath = new Path(); + + public VideoThumbnailsView(final Context context) { + super(context); + } + + public VideoThumbnailsView(final Context context, final @Nullable AttributeSet attrs) { + super(context, attrs); + } + + public VideoThumbnailsView(final Context context, final @Nullable AttributeSet attrs, final int defStyleAttr) { + super(context, attrs, defStyleAttr); + } + + /** + * @return Whether or not the current URI was changed. + */ + public boolean setInput(@NonNull Uri uri) throws IOException { + if (uri.equals(this.currentUri)) { + return false; + } + + this.currentUri = uri; + this.input = DecryptableUriMediaInput.createForUri(getContext(), uri); + this.thumbnails = null; + if (thumbnailsTask != null) { + thumbnailsTask.cancel(true); + thumbnailsTask = null; + } + invalidate(); + return true; + } + + @Override + protected void onDetachedFromWindow() { + super.onDetachedFromWindow(); + + thumbnails = null; + if (thumbnailsTask != null) { + thumbnailsTask.cancel(true); + thumbnailsTask = null; + } + + if (input != null) { + try { + input.close(); + } catch (IOException e) { + Log.w(TAG, e); + } + } + } + + @Override + protected void onDraw(final Canvas canvas) { + super.onDraw(canvas); + + if (input == null) { + return; + } + + final int left = getPaddingLeft(); + final int top = getPaddingTop(); + final int right = getWidth() - getPaddingRight(); + final int bottom = getHeight() - getPaddingBottom(); + + clippingPath.reset(); + clippingPath.addRoundRect(left, top, right, bottom, CORNER_RADIUS, CORNER_RADIUS, Path.Direction.CW); + + tempDrawRect.set(left, top, right, bottom); + + if (!drawRect.equals(tempDrawRect)) { + drawRect.set(tempDrawRect); + thumbnails = null; + if (thumbnailsTask != null) { + thumbnailsTask.cancel(true); + thumbnailsTask = null; + } + } + + if (thumbnails == null) { + if (thumbnailsTask == null) { + final int thumbnailCount = drawRect.width() / drawRect.height(); + final float thumbnailWidth = (float) drawRect.width() / thumbnailCount; + final float thumbnailHeight = drawRect.height(); + + thumbnails = new ArrayList<>(thumbnailCount); + thumbnailsTask = new ThumbnailsTask(this, input, thumbnailWidth, thumbnailHeight, thumbnailCount); + thumbnailsTask.execute(); + } + } else { + final int thumbnailCount = drawRect.width() / drawRect.height(); + final float thumbnailWidth = (float) drawRect.width() / thumbnailCount; + final float thumbnailHeight = drawRect.height(); + + tempRect.top = drawRect.top; + tempRect.bottom = drawRect.bottom; + canvas.save(); + + canvas.clipPath(clippingPath); + + for (int i = 0; i < thumbnails.size(); i++) { + tempRect.left = drawRect.left + i * thumbnailWidth; + tempRect.right = tempRect.left + thumbnailWidth; + + final Bitmap thumbnailBitmap = thumbnails.get(i); + if (thumbnailBitmap != null) { + canvas.save(); + canvas.rotate(180, tempRect.centerX(), tempRect.centerY()); + tempDrawRect.set(0, 0, thumbnailBitmap.getWidth(), thumbnailBitmap.getHeight()); + if (tempDrawRect.width() * thumbnailHeight > tempDrawRect.height() * thumbnailWidth) { + float w = tempDrawRect.height() * thumbnailWidth / thumbnailHeight; + tempDrawRect.left = tempDrawRect.centerX() - (int) (w / 2); + tempDrawRect.right = tempDrawRect.left + (int) w; + } else { + float h = tempDrawRect.width() * thumbnailHeight / thumbnailWidth; + tempDrawRect.top = tempDrawRect.centerY() - (int) (h / 2); + tempDrawRect.bottom = tempDrawRect.top + (int) h; + } + canvas.drawBitmap(thumbnailBitmap, tempDrawRect, tempRect, paint); + canvas.restore(); + } + } + + canvas.restore(); + } + } + + private void setDuration(long duration) { + if (this.duration != duration) { + this.duration = duration; + afterDurationChange(duration); + } + } + + abstract void afterDurationChange(long duration); + + public long getDuration() { + return duration; + } + + private static class ThumbnailsTask extends AsyncTask { + + final WeakReference viewReference; + final MediaInput input; + final float thumbnailWidth; + final float thumbnailHeight; + final int thumbnailCount; + + long duration; + + ThumbnailsTask(final @NonNull VideoThumbnailsView view, final @NonNull MediaInput input, final float thumbnailWidth, final float thumbnailHeight, final int thumbnailCount) { + this.viewReference = new WeakReference<>(view); + this.input = input; + this.thumbnailWidth = thumbnailWidth; + this.thumbnailHeight = thumbnailHeight; + this.thumbnailCount = thumbnailCount; + } + + @Override + protected Void doInBackground(Void... params) { + Log.i(TAG, "generate " + thumbnailCount + " thumbnails " + thumbnailWidth + "x" + thumbnailHeight); + VideoThumbnailsExtractor.extractThumbnails(input, thumbnailCount, (int) thumbnailHeight, new VideoThumbnailsExtractor.Callback() { + + @Override + public void durationKnown(long duration) { + ThumbnailsTask.this.duration = duration; + } + + @Override + public boolean publishProgress(int index, Bitmap thumbnail) { + boolean notCanceled = !isCancelled(); + if (notCanceled) { + ThumbnailsTask.this.publishProgress(thumbnail); + } + return notCanceled; + } + + @Override + public void failed() { + Log.w(TAG, "Thumbnail extraction failed"); + } + }); + return null; + } + + @Override + protected void onProgressUpdate(Bitmap... values) { + if (isCancelled()) { + return; + } + + VideoThumbnailsView view = viewReference.get(); + List thumbnails = view != null ? view.thumbnails : null; + if (thumbnails != null) { + thumbnails.addAll(Arrays.asList(values)); + view.invalidate(); + } + } + + @Override + protected void onPostExecute(Void result) { + VideoThumbnailsView view = viewReference.get(); + List thumbnails = view != null ? view.thumbnails : null; + if (view != null) { + view.setDuration(ThumbnailsTask.this.duration); + view.invalidate(); + Log.i(TAG, "onPostExecute, we have " + (thumbnails != null ? thumbnails.size() : "null") + " thumbs"); + } + } + } +} diff --git a/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/AndroidMuxer.java b/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/AndroidMuxer.java index 9ed0536d43..615bc94715 100644 --- a/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/AndroidMuxer.java +++ b/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/AndroidMuxer.java @@ -1,60 +1,60 @@ -package org.thoughtcrime.securesms.video.videoconverter; - -import android.media.MediaCodec; -import android.media.MediaFormat; -import android.media.MediaMuxer; - -import androidx.annotation.NonNull; -import androidx.annotation.RequiresApi; - -import org.thoughtcrime.securesms.video.interfaces.Muxer; - -import java.io.File; -import java.io.FileDescriptor; -import java.io.IOException; -import java.nio.ByteBuffer; - -final class AndroidMuxer implements Muxer { - - private final MediaMuxer muxer; - - AndroidMuxer(final @NonNull File file) throws IOException { - muxer = new MediaMuxer(file.getAbsolutePath(), MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4); - } - - @RequiresApi(26) - AndroidMuxer(final @NonNull FileDescriptor fileDescriptor) throws IOException { - muxer = new MediaMuxer(fileDescriptor, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4); - } - - @Override - public void start() { - muxer.start(); - } - - @Override - public long stop() { - muxer.stop(); - return 0; - } - - @Override - public int addTrack(final @NonNull MediaFormat format) { - return muxer.addTrack(format); - } - - @Override - public void writeSampleData(final int trackIndex, final @NonNull ByteBuffer byteBuf, final @NonNull MediaCodec.BufferInfo bufferInfo) { - muxer.writeSampleData(trackIndex, byteBuf, bufferInfo); - } - - @Override - public void release() { - muxer.release(); - } - - @Override - public boolean supportsAudioRemux() { - return false; - } -} +package org.thoughtcrime.securesms.video.videoconverter; + +import android.media.MediaCodec; +import android.media.MediaFormat; +import android.media.MediaMuxer; + +import androidx.annotation.NonNull; +import androidx.annotation.RequiresApi; + +import org.thoughtcrime.securesms.video.interfaces.Muxer; + +import java.io.File; +import java.io.FileDescriptor; +import java.io.IOException; +import java.nio.ByteBuffer; + +final class AndroidMuxer implements Muxer { + + private final MediaMuxer muxer; + + AndroidMuxer(final @NonNull File file) throws IOException { + muxer = new MediaMuxer(file.getAbsolutePath(), MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4); + } + + @RequiresApi(26) + AndroidMuxer(final @NonNull FileDescriptor fileDescriptor) throws IOException { + muxer = new MediaMuxer(fileDescriptor, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4); + } + + @Override + public void start() { + muxer.start(); + } + + @Override + public long stop() { + muxer.stop(); + return 0; + } + + @Override + public int addTrack(final @NonNull MediaFormat format) { + return muxer.addTrack(format); + } + + @Override + public void writeSampleData(final int trackIndex, final @NonNull ByteBuffer byteBuf, final @NonNull MediaCodec.BufferInfo bufferInfo) { + muxer.writeSampleData(trackIndex, byteBuf, bufferInfo); + } + + @Override + public void release() { + muxer.release(); + } + + @Override + public boolean supportsAudioRemux() { + return false; + } +} diff --git a/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/AudioTrackConverter.java b/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/AudioTrackConverter.java index d5d2380d32..baf54a06b4 100644 --- a/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/AudioTrackConverter.java +++ b/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/AudioTrackConverter.java @@ -1,506 +1,506 @@ -package org.thoughtcrime.securesms.video.videoconverter; - -import android.annotation.SuppressLint; -import android.media.MediaCodec; -import android.media.MediaCodecInfo; -import android.media.MediaExtractor; -import android.media.MediaFormat; - -import androidx.annotation.NonNull; -import androidx.annotation.Nullable; - -import org.signal.core.util.logging.Log; -import org.thoughtcrime.securesms.video.interfaces.MediaInput; -import org.thoughtcrime.securesms.video.interfaces.Muxer; -import org.thoughtcrime.securesms.video.videoconverter.utils.Preconditions; -import org.thoughtcrime.securesms.video.videoconverter.utils.VideoConstants; - -import java.io.FileNotFoundException; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.Locale; - -final class AudioTrackConverter { - - private static final String TAG = "media-converter"; - private static final boolean VERBOSE = false; // lots of logging - - private static final String OUTPUT_AUDIO_MIME_TYPE = VideoConstants.AUDIO_MIME_TYPE; // Advanced Audio Coding - private static final int OUTPUT_AUDIO_AAC_PROFILE = MediaCodecInfo.CodecProfileLevel.AACObjectLC; //MediaCodecInfo.CodecProfileLevel.AACObjectHE; - - private static final int SAMPLE_BUFFER_SIZE = 16 * 1024; - private static final int TIMEOUT_USEC = 10000; - - private final long mTimeFrom; - private final long mTimeTo; - private final int mAudioBitrate; - - final long mInputDuration; - - private final MediaExtractor mAudioExtractor; - private final MediaCodec mAudioDecoder; - private final MediaCodec mAudioEncoder; - - private final ByteBuffer instanceSampleBuffer = ByteBuffer.allocateDirect(SAMPLE_BUFFER_SIZE); - private final MediaCodec.BufferInfo instanceBufferInfo = new MediaCodec.BufferInfo(); - - private final ByteBuffer[] mAudioDecoderInputBuffers; - private ByteBuffer[] mAudioDecoderOutputBuffers; - private final ByteBuffer[] mAudioEncoderInputBuffers; - private ByteBuffer[] mAudioEncoderOutputBuffers; - private final MediaCodec.BufferInfo mAudioDecoderOutputBufferInfo; - private final MediaCodec.BufferInfo mAudioEncoderOutputBufferInfo; - - MediaFormat mEncoderOutputAudioFormat; - - boolean mAudioExtractorDone; - private boolean mAudioDecoderDone; - boolean mAudioEncoderDone; - private boolean skipTrancode; - - private int mOutputAudioTrack = -1; - - private int mPendingAudioDecoderOutputBufferIndex = -1; - long mMuxingAudioPresentationTime; - - private int mAudioExtractedFrameCount; - private int mAudioDecodedFrameCount; - private int mAudioEncodedFrameCount; - - private Muxer mMuxer; - - static @Nullable - AudioTrackConverter create( - final @NonNull MediaInput input, - final long timeFrom, - final long timeTo, - final int audioBitrate, - final boolean allowSkipTranscode) throws IOException { - - final MediaExtractor audioExtractor = input.createExtractor(); - final int audioInputTrack = getAndSelectAudioTrackIndex(audioExtractor); - if (audioInputTrack == -1) { - audioExtractor.release(); - return null; - } - return new AudioTrackConverter(audioExtractor, audioInputTrack, timeFrom, timeTo, audioBitrate, allowSkipTranscode); - } - - private AudioTrackConverter( - final @NonNull MediaExtractor audioExtractor, - final int audioInputTrack, - long timeFrom, - long timeTo, - int audioBitrate, - final boolean allowSkipTranscode) throws IOException { - - mTimeFrom = timeFrom; - mTimeTo = timeTo; - mAudioExtractor = audioExtractor; - mAudioBitrate = audioBitrate; - - final MediaCodecInfo audioCodecInfo = MediaConverter.selectCodec(OUTPUT_AUDIO_MIME_TYPE); - if (audioCodecInfo == null) { - // Don't fail CTS if they don't have an AAC codec (not here, anyway). - Log.e(TAG, "Unable to find an appropriate codec for " + OUTPUT_AUDIO_MIME_TYPE); - throw new FileNotFoundException(); - } - if (VERBOSE) Log.d(TAG, "audio found codec: " + audioCodecInfo.getName()); - - final MediaFormat inputAudioFormat = mAudioExtractor.getTrackFormat(audioInputTrack); - mInputDuration = inputAudioFormat.containsKey(MediaFormat.KEY_DURATION) ? inputAudioFormat.getLong(MediaFormat.KEY_DURATION) : 0; - - skipTrancode = allowSkipTranscode && formatCanSkipTranscode(inputAudioFormat, audioBitrate); - if (skipTrancode) { - mEncoderOutputAudioFormat = inputAudioFormat; - } - - if (VERBOSE) Log.d(TAG, "audio skipping transcoding: " + skipTrancode); - - final MediaFormat outputAudioFormat = - MediaFormat.createAudioFormat( - OUTPUT_AUDIO_MIME_TYPE, - inputAudioFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE), - inputAudioFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT)); - outputAudioFormat.setInteger(MediaFormat.KEY_BIT_RATE, audioBitrate); - outputAudioFormat.setInteger(MediaFormat.KEY_AAC_PROFILE, OUTPUT_AUDIO_AAC_PROFILE); - outputAudioFormat.setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, SAMPLE_BUFFER_SIZE); - - // Create a MediaCodec for the desired codec, then configure it as an encoder with - // our desired properties. Request a Surface to use for input. - mAudioEncoder = createAudioEncoder(audioCodecInfo, outputAudioFormat); - // Create a MediaCodec for the decoder, based on the extractor's format. - mAudioDecoder = createAudioDecoder(inputAudioFormat); - - mAudioDecoderInputBuffers = mAudioDecoder.getInputBuffers(); - mAudioDecoderOutputBuffers = mAudioDecoder.getOutputBuffers(); - mAudioEncoderInputBuffers = mAudioEncoder.getInputBuffers(); - mAudioEncoderOutputBuffers = mAudioEncoder.getOutputBuffers(); - mAudioDecoderOutputBufferInfo = new MediaCodec.BufferInfo(); - mAudioEncoderOutputBufferInfo = new MediaCodec.BufferInfo(); - - if (mTimeFrom > 0) { - mAudioExtractor.seekTo(mTimeFrom * 1000, MediaExtractor.SEEK_TO_PREVIOUS_SYNC); - Log.i(TAG, "Seek audio:" + mTimeFrom + " " + mAudioExtractor.getSampleTime()); - } - } - - void setMuxer(final @NonNull Muxer muxer) throws IOException { - mMuxer = muxer; - if (mEncoderOutputAudioFormat != null) { - Log.d(TAG, "muxer: adding audio track."); - if (!mEncoderOutputAudioFormat.containsKey(MediaFormat.KEY_BIT_RATE)) { - Log.d(TAG, "muxer: fixed MediaFormat to add bitrate."); - mEncoderOutputAudioFormat.setInteger(MediaFormat.KEY_BIT_RATE, mAudioBitrate); - } - if (!mEncoderOutputAudioFormat.containsKey(MediaFormat.KEY_AAC_PROFILE)) { - Log.d(TAG, "muxer: fixed MediaFormat to add AAC profile."); - mEncoderOutputAudioFormat.setInteger(MediaFormat.KEY_AAC_PROFILE, OUTPUT_AUDIO_AAC_PROFILE); - } - mOutputAudioTrack = muxer.addTrack(mEncoderOutputAudioFormat); - } - } - - void step() throws IOException { - - if (skipTrancode && mEncoderOutputAudioFormat != null) { - try { - extractAndRemux(); - return; - } catch (IllegalArgumentException e) { - Log.w(TAG, "Remuxer threw an exception! Disabling remux.", e); - skipTrancode = false; - } - } - - // Extract audio from file and feed to decoder. - // Do not extract audio if we have determined the output format but we are not yet - // ready to mux the frames. - while (!mAudioExtractorDone && (mEncoderOutputAudioFormat == null || mMuxer != null)) { - int decoderInputBufferIndex = mAudioDecoder.dequeueInputBuffer(TIMEOUT_USEC); - if (decoderInputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) { - if (VERBOSE) Log.d(TAG, "no audio decoder input buffer"); - break; - } - if (VERBOSE) { - Log.d(TAG, "audio decoder: returned input buffer: " + decoderInputBufferIndex); - } - final ByteBuffer decoderInputBuffer = mAudioDecoderInputBuffers[decoderInputBufferIndex]; - final int size = mAudioExtractor.readSampleData(decoderInputBuffer, 0); - final long presentationTime = mAudioExtractor.getSampleTime(); - if (VERBOSE) { - Log.d(TAG, "audio extractor: returned buffer of size " + size); - Log.d(TAG, "audio extractor: returned buffer for time " + presentationTime); - } - mAudioExtractorDone = isAudioExtractorDone(size, presentationTime); - - if (mAudioExtractorDone) { - if (VERBOSE) Log.d(TAG, "audio extractor: EOS"); - mAudioDecoder.queueInputBuffer( - decoderInputBufferIndex, - 0, - 0, - 0, - MediaCodec.BUFFER_FLAG_END_OF_STREAM); - } else { - mAudioDecoder.queueInputBuffer( - decoderInputBufferIndex, - 0, - size, - presentationTime, - mAudioExtractor.getSampleFlags()); - } - mAudioExtractor.advance(); - mAudioExtractedFrameCount++; - // We extracted a frame, let's try something else next. - break; - } - - // Poll output frames from the audio decoder. - // Do not poll if we already have a pending buffer to feed to the encoder. - while (!mAudioDecoderDone && mPendingAudioDecoderOutputBufferIndex == -1 - && (mEncoderOutputAudioFormat == null || mMuxer != null)) { - final int decoderOutputBufferIndex = - mAudioDecoder.dequeueOutputBuffer( - mAudioDecoderOutputBufferInfo, TIMEOUT_USEC); - if (decoderOutputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) { - if (VERBOSE) Log.d(TAG, "no audio decoder output buffer"); - break; - } - if (decoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { - if (VERBOSE) Log.d(TAG, "audio decoder: output buffers changed"); - mAudioDecoderOutputBuffers = mAudioDecoder.getOutputBuffers(); - break; - } - if (decoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { - if (VERBOSE) { - MediaFormat decoderOutputAudioFormat = mAudioDecoder.getOutputFormat(); - Log.d(TAG, "audio decoder: output format changed: " + decoderOutputAudioFormat); - } - break; - } - if (VERBOSE) { - Log.d(TAG, "audio decoder: returned output buffer: " + decoderOutputBufferIndex); - Log.d(TAG, "audio decoder: returned buffer of size " + mAudioDecoderOutputBufferInfo.size); - } - if ((mAudioDecoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) { - if (VERBOSE) Log.d(TAG, "audio decoder: codec config buffer"); - mAudioDecoder.releaseOutputBuffer(decoderOutputBufferIndex, false); - break; - } - if (mAudioDecoderOutputBufferInfo.presentationTimeUs < mTimeFrom * 1000 && - (mAudioDecoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) == 0) { - if (VERBOSE) - Log.d(TAG, "audio decoder: frame prior to " + mAudioDecoderOutputBufferInfo.presentationTimeUs); - mAudioDecoder.releaseOutputBuffer(decoderOutputBufferIndex, false); - break; - } - if (VERBOSE) { - Log.d(TAG, "audio decoder: returned buffer for time " + mAudioDecoderOutputBufferInfo.presentationTimeUs); - Log.d(TAG, "audio decoder: output buffer is now pending: " + mPendingAudioDecoderOutputBufferIndex); - } - mPendingAudioDecoderOutputBufferIndex = decoderOutputBufferIndex; - mAudioDecodedFrameCount++; - // We extracted a pending frame, let's try something else next. - break; - } - - // Feed the pending decoded audio buffer to the audio encoder. - while (mPendingAudioDecoderOutputBufferIndex != -1) { - if (VERBOSE) { - Log.d(TAG, "audio decoder: attempting to process pending buffer: " + mPendingAudioDecoderOutputBufferIndex); - } - final int encoderInputBufferIndex = mAudioEncoder.dequeueInputBuffer(TIMEOUT_USEC); - if (encoderInputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) { - if (VERBOSE) Log.d(TAG, "no audio encoder input buffer"); - break; - } - if (VERBOSE) { - Log.d(TAG, "audio encoder: returned input buffer: " + encoderInputBufferIndex); - } - final ByteBuffer encoderInputBuffer = mAudioEncoderInputBuffers[encoderInputBufferIndex]; - final int size = mAudioDecoderOutputBufferInfo.size; - final long presentationTime = mAudioDecoderOutputBufferInfo.presentationTimeUs; - if (VERBOSE) { - Log.d(TAG, "audio decoder: processing pending buffer: " + mPendingAudioDecoderOutputBufferIndex); - } - if (VERBOSE) { - Log.d(TAG, "audio decoder: pending buffer of size " + size); - Log.d(TAG, "audio decoder: pending buffer for time " + presentationTime); - } - if (size >= 0) { - final ByteBuffer decoderOutputBuffer = mAudioDecoderOutputBuffers[mPendingAudioDecoderOutputBufferIndex].duplicate(); - decoderOutputBuffer.position(mAudioDecoderOutputBufferInfo.offset); - decoderOutputBuffer.limit(mAudioDecoderOutputBufferInfo.offset + size); - encoderInputBuffer.position(0); - encoderInputBuffer.put(decoderOutputBuffer); - - mAudioEncoder.queueInputBuffer( - encoderInputBufferIndex, - 0, - size, - presentationTime, - mAudioDecoderOutputBufferInfo.flags); - } - mAudioDecoder.releaseOutputBuffer(mPendingAudioDecoderOutputBufferIndex, false); - mPendingAudioDecoderOutputBufferIndex = -1; - if ((mAudioDecoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { - if (VERBOSE) Log.d(TAG, "audio decoder: EOS"); - mAudioDecoderDone = true; - } - // We enqueued a pending frame, let's try something else next. - break; - } - - // Poll frames from the audio encoder and send them to the muxer. - while (!mAudioEncoderDone && (mEncoderOutputAudioFormat == null || mMuxer != null)) { - final int encoderOutputBufferIndex = mAudioEncoder.dequeueOutputBuffer(mAudioEncoderOutputBufferInfo, TIMEOUT_USEC); - if (encoderOutputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) { - if (VERBOSE) Log.d(TAG, "no audio encoder output buffer"); - break; - } - if (encoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { - if (VERBOSE) Log.d(TAG, "audio encoder: output buffers changed"); - mAudioEncoderOutputBuffers = mAudioEncoder.getOutputBuffers(); - break; - } - if (encoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { - if (VERBOSE) Log.d(TAG, "audio encoder: output format changed"); - Preconditions.checkState("audio encoder changed its output format again?", mOutputAudioTrack < 0); - - mEncoderOutputAudioFormat = mAudioEncoder.getOutputFormat(); - break; - } - Preconditions.checkState("should have added track before processing output", mMuxer != null); - if (VERBOSE) { - Log.d(TAG, "audio encoder: returned output buffer: " + encoderOutputBufferIndex); - Log.d(TAG, "audio encoder: returned buffer of size " + mAudioEncoderOutputBufferInfo.size); - } - final ByteBuffer encoderOutputBuffer = mAudioEncoderOutputBuffers[encoderOutputBufferIndex]; - if ((mAudioEncoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) { - if (VERBOSE) Log.d(TAG, "audio encoder: codec config buffer"); - // Simply ignore codec config buffers. - mAudioEncoder.releaseOutputBuffer(encoderOutputBufferIndex, false); - break; - } - if (VERBOSE) { - Log.d(TAG, "audio encoder: returned buffer for time " + mAudioEncoderOutputBufferInfo.presentationTimeUs); - } - if (mAudioEncoderOutputBufferInfo.size != 0) { - mMuxer.writeSampleData(mOutputAudioTrack, encoderOutputBuffer, mAudioEncoderOutputBufferInfo); - mMuxingAudioPresentationTime = Math.max(mMuxingAudioPresentationTime, mAudioEncoderOutputBufferInfo.presentationTimeUs); - } - if ((mAudioEncoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { - if (VERBOSE) Log.d(TAG, "audio encoder: EOS"); - mAudioEncoderDone = true; - } - mAudioEncoder.releaseOutputBuffer(encoderOutputBufferIndex, false); - mAudioEncodedFrameCount++; - // We enqueued an encoded frame, let's try something else next. - break; - } - } - - void release() throws Exception { - Exception exception = null; - try { - if (mAudioExtractor != null) { - mAudioExtractor.release(); - } - } catch (Exception e) { - Log.e(TAG, "error while releasing mAudioExtractor", e); - exception = e; - } - try { - if (mAudioDecoder != null) { - mAudioDecoder.stop(); - mAudioDecoder.release(); - } - } catch (Exception e) { - Log.e(TAG, "error while releasing mAudioDecoder", e); - if (exception == null) { - exception = e; - } - } - try { - if (mAudioEncoder != null) { - mAudioEncoder.stop(); - mAudioEncoder.release(); - } - } catch (Exception e) { - Log.e(TAG, "error while releasing mAudioEncoder", e); - if (exception == null) { - exception = e; - } - } - if (exception != null) { - throw exception; - } - } - - AudioTrackConverterState dumpState() { - return new AudioTrackConverterState( - mAudioExtractedFrameCount, mAudioExtractorDone, - mAudioDecodedFrameCount, mAudioDecoderDone, - mAudioEncodedFrameCount, mAudioEncoderDone, - mPendingAudioDecoderOutputBufferIndex, - mMuxer != null, mOutputAudioTrack); - } - - void verifyEndState() { - Preconditions.checkState("no frame should be pending", -1 == mPendingAudioDecoderOutputBufferIndex); - } - - @SuppressLint("WrongConstant") // flags extracted from sample by MediaExtractor should be safe for MediaCodec.BufferInfo - private void extractAndRemux() throws IOException { - if (mMuxer == null) { - Log.d(TAG, "audio remuxer: tried to execute before muxer was ready"); - return; - } - int size = mAudioExtractor.readSampleData(instanceSampleBuffer, 0); - long presentationTime = mAudioExtractor.getSampleTime(); - int sampleFlags = mAudioExtractor.getSampleFlags(); - if (VERBOSE) { - Log.d(TAG, "audio extractor: returned buffer of size " + size); - Log.d(TAG, "audio extractor: returned buffer for time " + presentationTime); - Log.d(TAG, "audio extractor: returned buffer with flags " + Integer.toBinaryString(sampleFlags)); - } - mAudioExtractorDone = isAudioExtractorDone(size, presentationTime); - - if (mAudioExtractorDone) { - if (VERBOSE) Log.d(TAG, "audio encoder: EOS"); - instanceBufferInfo.set(0, 0, presentationTime, MediaCodec.BUFFER_FLAG_END_OF_STREAM); - mAudioEncoderDone = true; - } else { - instanceBufferInfo.set(0, size, presentationTime, sampleFlags); - } - - mMuxer.writeSampleData(mOutputAudioTrack, instanceSampleBuffer, instanceBufferInfo); - - if (VERBOSE) { - Log.d(TAG, "audio extractor: wrote sample at " + presentationTime); - } - - mAudioExtractor.advance(); - - mAudioExtractedFrameCount++; - mAudioEncodedFrameCount++; - mMuxingAudioPresentationTime = Math.max(mMuxingAudioPresentationTime, presentationTime); - } - - private boolean isAudioExtractorDone(int size, long presentationTime) { - return presentationTime == -1 || size < 0 || (mTimeTo > 0 && presentationTime > mTimeTo * 1000); - } - - private static @NonNull - MediaCodec createAudioDecoder(final @NonNull MediaFormat inputFormat) throws IOException { - final MediaCodec decoder = MediaCodec.createDecoderByType(MediaConverter.getMimeTypeFor(inputFormat)); - decoder.configure(inputFormat, null, null, 0); - decoder.start(); - return decoder; - } - - private static @NonNull - MediaCodec createAudioEncoder(final @NonNull MediaCodecInfo codecInfo, final @NonNull MediaFormat format) throws IOException { - final MediaCodec encoder = MediaCodec.createByCodecName(codecInfo.getName()); - encoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); - encoder.start(); - return encoder; - } - - private static int getAndSelectAudioTrackIndex(MediaExtractor extractor) { - for (int index = 0; index < extractor.getTrackCount(); ++index) { - if (VERBOSE) { - Log.d(TAG, "format for track " + index + " is " + MediaConverter.getMimeTypeFor(extractor.getTrackFormat(index))); - } - if (isAudioFormat(extractor.getTrackFormat(index))) { - extractor.selectTrack(index); - return index; - } - } - return -1; - } - - private static boolean isAudioFormat(final @NonNull MediaFormat format) { - return MediaConverter.getMimeTypeFor(format).startsWith("audio/"); - } - - /** - * HE-AAC input bitstreams exhibit bad decoder behavior: the decoder's output buffer's presentation timestamp is way larger than the input sample's. - * This mismatch propagates throughout the transcoding pipeline and results in slowed, distorted audio in the output file. - * To sidestep this: AAC and its variants are a supported output codec, and HE-AAC bitrates are almost always lower than our target bitrate, - * so we can pass through the input bitstream unaltered, relying on consumers of the output file to render HE-AAC correctly. - */ - private static boolean formatCanSkipTranscode(MediaFormat audioFormat, int desiredBitrate) { - try { - int inputBitrate = audioFormat.getInteger(MediaFormat.KEY_BIT_RATE); - String inputMimeType = audioFormat.getString(MediaFormat.KEY_MIME); - return OUTPUT_AUDIO_MIME_TYPE.equals(inputMimeType) && inputBitrate <= desiredBitrate; - } catch (NullPointerException exception) { - if (VERBOSE) { - Log.d(TAG, "could not find bitrate in mediaFormat, can't skip transcoding."); - } - return false; - } - } -} +package org.thoughtcrime.securesms.video.videoconverter; + +import android.annotation.SuppressLint; +import android.media.MediaCodec; +import android.media.MediaCodecInfo; +import android.media.MediaExtractor; +import android.media.MediaFormat; + +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; + +import org.signal.core.util.logging.Log; +import org.thoughtcrime.securesms.video.interfaces.MediaInput; +import org.thoughtcrime.securesms.video.interfaces.Muxer; +import org.thoughtcrime.securesms.video.videoconverter.utils.Preconditions; +import org.thoughtcrime.securesms.video.videoconverter.utils.VideoConstants; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Locale; + +final class AudioTrackConverter { + + private static final String TAG = "media-converter"; + private static final boolean VERBOSE = false; // lots of logging + + private static final String OUTPUT_AUDIO_MIME_TYPE = VideoConstants.AUDIO_MIME_TYPE; // Advanced Audio Coding + private static final int OUTPUT_AUDIO_AAC_PROFILE = MediaCodecInfo.CodecProfileLevel.AACObjectLC; //MediaCodecInfo.CodecProfileLevel.AACObjectHE; + + private static final int SAMPLE_BUFFER_SIZE = 16 * 1024; + private static final int TIMEOUT_USEC = 10000; + + private final long mTimeFrom; + private final long mTimeTo; + private final int mAudioBitrate; + + final long mInputDuration; + + private final MediaExtractor mAudioExtractor; + private final MediaCodec mAudioDecoder; + private final MediaCodec mAudioEncoder; + + private final ByteBuffer instanceSampleBuffer = ByteBuffer.allocateDirect(SAMPLE_BUFFER_SIZE); + private final MediaCodec.BufferInfo instanceBufferInfo = new MediaCodec.BufferInfo(); + + private final ByteBuffer[] mAudioDecoderInputBuffers; + private ByteBuffer[] mAudioDecoderOutputBuffers; + private final ByteBuffer[] mAudioEncoderInputBuffers; + private ByteBuffer[] mAudioEncoderOutputBuffers; + private final MediaCodec.BufferInfo mAudioDecoderOutputBufferInfo; + private final MediaCodec.BufferInfo mAudioEncoderOutputBufferInfo; + + MediaFormat mEncoderOutputAudioFormat; + + boolean mAudioExtractorDone; + private boolean mAudioDecoderDone; + boolean mAudioEncoderDone; + private boolean skipTrancode; + + private int mOutputAudioTrack = -1; + + private int mPendingAudioDecoderOutputBufferIndex = -1; + long mMuxingAudioPresentationTime; + + private int mAudioExtractedFrameCount; + private int mAudioDecodedFrameCount; + private int mAudioEncodedFrameCount; + + private Muxer mMuxer; + + static @Nullable + AudioTrackConverter create( + final @NonNull MediaInput input, + final long timeFrom, + final long timeTo, + final int audioBitrate, + final boolean allowSkipTranscode) throws IOException { + + final MediaExtractor audioExtractor = input.createExtractor(); + final int audioInputTrack = getAndSelectAudioTrackIndex(audioExtractor); + if (audioInputTrack == -1) { + audioExtractor.release(); + return null; + } + return new AudioTrackConverter(audioExtractor, audioInputTrack, timeFrom, timeTo, audioBitrate, allowSkipTranscode); + } + + private AudioTrackConverter( + final @NonNull MediaExtractor audioExtractor, + final int audioInputTrack, + long timeFrom, + long timeTo, + int audioBitrate, + final boolean allowSkipTranscode) throws IOException { + + mTimeFrom = timeFrom; + mTimeTo = timeTo; + mAudioExtractor = audioExtractor; + mAudioBitrate = audioBitrate; + + final MediaCodecInfo audioCodecInfo = MediaConverter.selectCodec(OUTPUT_AUDIO_MIME_TYPE); + if (audioCodecInfo == null) { + // Don't fail CTS if they don't have an AAC codec (not here, anyway). + Log.e(TAG, "Unable to find an appropriate codec for " + OUTPUT_AUDIO_MIME_TYPE); + throw new FileNotFoundException(); + } + if (VERBOSE) Log.d(TAG, "audio found codec: " + audioCodecInfo.getName()); + + final MediaFormat inputAudioFormat = mAudioExtractor.getTrackFormat(audioInputTrack); + mInputDuration = inputAudioFormat.containsKey(MediaFormat.KEY_DURATION) ? inputAudioFormat.getLong(MediaFormat.KEY_DURATION) : 0; + + skipTrancode = allowSkipTranscode && formatCanSkipTranscode(inputAudioFormat, audioBitrate); + if (skipTrancode) { + mEncoderOutputAudioFormat = inputAudioFormat; + } + + if (VERBOSE) Log.d(TAG, "audio skipping transcoding: " + skipTrancode); + + final MediaFormat outputAudioFormat = + MediaFormat.createAudioFormat( + OUTPUT_AUDIO_MIME_TYPE, + inputAudioFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE), + inputAudioFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT)); + outputAudioFormat.setInteger(MediaFormat.KEY_BIT_RATE, audioBitrate); + outputAudioFormat.setInteger(MediaFormat.KEY_AAC_PROFILE, OUTPUT_AUDIO_AAC_PROFILE); + outputAudioFormat.setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, SAMPLE_BUFFER_SIZE); + + // Create a MediaCodec for the desired codec, then configure it as an encoder with + // our desired properties. Request a Surface to use for input. + mAudioEncoder = createAudioEncoder(audioCodecInfo, outputAudioFormat); + // Create a MediaCodec for the decoder, based on the extractor's format. + mAudioDecoder = createAudioDecoder(inputAudioFormat); + + mAudioDecoderInputBuffers = mAudioDecoder.getInputBuffers(); + mAudioDecoderOutputBuffers = mAudioDecoder.getOutputBuffers(); + mAudioEncoderInputBuffers = mAudioEncoder.getInputBuffers(); + mAudioEncoderOutputBuffers = mAudioEncoder.getOutputBuffers(); + mAudioDecoderOutputBufferInfo = new MediaCodec.BufferInfo(); + mAudioEncoderOutputBufferInfo = new MediaCodec.BufferInfo(); + + if (mTimeFrom > 0) { + mAudioExtractor.seekTo(mTimeFrom * 1000, MediaExtractor.SEEK_TO_PREVIOUS_SYNC); + Log.i(TAG, "Seek audio:" + mTimeFrom + " " + mAudioExtractor.getSampleTime()); + } + } + + void setMuxer(final @NonNull Muxer muxer) throws IOException { + mMuxer = muxer; + if (mEncoderOutputAudioFormat != null) { + Log.d(TAG, "muxer: adding audio track."); + if (!mEncoderOutputAudioFormat.containsKey(MediaFormat.KEY_BIT_RATE)) { + Log.d(TAG, "muxer: fixed MediaFormat to add bitrate."); + mEncoderOutputAudioFormat.setInteger(MediaFormat.KEY_BIT_RATE, mAudioBitrate); + } + if (!mEncoderOutputAudioFormat.containsKey(MediaFormat.KEY_AAC_PROFILE)) { + Log.d(TAG, "muxer: fixed MediaFormat to add AAC profile."); + mEncoderOutputAudioFormat.setInteger(MediaFormat.KEY_AAC_PROFILE, OUTPUT_AUDIO_AAC_PROFILE); + } + mOutputAudioTrack = muxer.addTrack(mEncoderOutputAudioFormat); + } + } + + void step() throws IOException { + + if (skipTrancode && mEncoderOutputAudioFormat != null) { + try { + extractAndRemux(); + return; + } catch (IllegalArgumentException e) { + Log.w(TAG, "Remuxer threw an exception! Disabling remux.", e); + skipTrancode = false; + } + } + + // Extract audio from file and feed to decoder. + // Do not extract audio if we have determined the output format but we are not yet + // ready to mux the frames. + while (!mAudioExtractorDone && (mEncoderOutputAudioFormat == null || mMuxer != null)) { + int decoderInputBufferIndex = mAudioDecoder.dequeueInputBuffer(TIMEOUT_USEC); + if (decoderInputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) { + if (VERBOSE) Log.d(TAG, "no audio decoder input buffer"); + break; + } + if (VERBOSE) { + Log.d(TAG, "audio decoder: returned input buffer: " + decoderInputBufferIndex); + } + final ByteBuffer decoderInputBuffer = mAudioDecoderInputBuffers[decoderInputBufferIndex]; + final int size = mAudioExtractor.readSampleData(decoderInputBuffer, 0); + final long presentationTime = mAudioExtractor.getSampleTime(); + if (VERBOSE) { + Log.d(TAG, "audio extractor: returned buffer of size " + size); + Log.d(TAG, "audio extractor: returned buffer for time " + presentationTime); + } + mAudioExtractorDone = isAudioExtractorDone(size, presentationTime); + + if (mAudioExtractorDone) { + if (VERBOSE) Log.d(TAG, "audio extractor: EOS"); + mAudioDecoder.queueInputBuffer( + decoderInputBufferIndex, + 0, + 0, + 0, + MediaCodec.BUFFER_FLAG_END_OF_STREAM); + } else { + mAudioDecoder.queueInputBuffer( + decoderInputBufferIndex, + 0, + size, + presentationTime, + mAudioExtractor.getSampleFlags()); + } + mAudioExtractor.advance(); + mAudioExtractedFrameCount++; + // We extracted a frame, let's try something else next. + break; + } + + // Poll output frames from the audio decoder. + // Do not poll if we already have a pending buffer to feed to the encoder. + while (!mAudioDecoderDone && mPendingAudioDecoderOutputBufferIndex == -1 + && (mEncoderOutputAudioFormat == null || mMuxer != null)) { + final int decoderOutputBufferIndex = + mAudioDecoder.dequeueOutputBuffer( + mAudioDecoderOutputBufferInfo, TIMEOUT_USEC); + if (decoderOutputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) { + if (VERBOSE) Log.d(TAG, "no audio decoder output buffer"); + break; + } + if (decoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { + if (VERBOSE) Log.d(TAG, "audio decoder: output buffers changed"); + mAudioDecoderOutputBuffers = mAudioDecoder.getOutputBuffers(); + break; + } + if (decoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { + if (VERBOSE) { + MediaFormat decoderOutputAudioFormat = mAudioDecoder.getOutputFormat(); + Log.d(TAG, "audio decoder: output format changed: " + decoderOutputAudioFormat); + } + break; + } + if (VERBOSE) { + Log.d(TAG, "audio decoder: returned output buffer: " + decoderOutputBufferIndex); + Log.d(TAG, "audio decoder: returned buffer of size " + mAudioDecoderOutputBufferInfo.size); + } + if ((mAudioDecoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) { + if (VERBOSE) Log.d(TAG, "audio decoder: codec config buffer"); + mAudioDecoder.releaseOutputBuffer(decoderOutputBufferIndex, false); + break; + } + if (mAudioDecoderOutputBufferInfo.presentationTimeUs < mTimeFrom * 1000 && + (mAudioDecoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) == 0) { + if (VERBOSE) + Log.d(TAG, "audio decoder: frame prior to " + mAudioDecoderOutputBufferInfo.presentationTimeUs); + mAudioDecoder.releaseOutputBuffer(decoderOutputBufferIndex, false); + break; + } + if (VERBOSE) { + Log.d(TAG, "audio decoder: returned buffer for time " + mAudioDecoderOutputBufferInfo.presentationTimeUs); + Log.d(TAG, "audio decoder: output buffer is now pending: " + mPendingAudioDecoderOutputBufferIndex); + } + mPendingAudioDecoderOutputBufferIndex = decoderOutputBufferIndex; + mAudioDecodedFrameCount++; + // We extracted a pending frame, let's try something else next. + break; + } + + // Feed the pending decoded audio buffer to the audio encoder. + while (mPendingAudioDecoderOutputBufferIndex != -1) { + if (VERBOSE) { + Log.d(TAG, "audio decoder: attempting to process pending buffer: " + mPendingAudioDecoderOutputBufferIndex); + } + final int encoderInputBufferIndex = mAudioEncoder.dequeueInputBuffer(TIMEOUT_USEC); + if (encoderInputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) { + if (VERBOSE) Log.d(TAG, "no audio encoder input buffer"); + break; + } + if (VERBOSE) { + Log.d(TAG, "audio encoder: returned input buffer: " + encoderInputBufferIndex); + } + final ByteBuffer encoderInputBuffer = mAudioEncoderInputBuffers[encoderInputBufferIndex]; + final int size = mAudioDecoderOutputBufferInfo.size; + final long presentationTime = mAudioDecoderOutputBufferInfo.presentationTimeUs; + if (VERBOSE) { + Log.d(TAG, "audio decoder: processing pending buffer: " + mPendingAudioDecoderOutputBufferIndex); + } + if (VERBOSE) { + Log.d(TAG, "audio decoder: pending buffer of size " + size); + Log.d(TAG, "audio decoder: pending buffer for time " + presentationTime); + } + if (size >= 0) { + final ByteBuffer decoderOutputBuffer = mAudioDecoderOutputBuffers[mPendingAudioDecoderOutputBufferIndex].duplicate(); + decoderOutputBuffer.position(mAudioDecoderOutputBufferInfo.offset); + decoderOutputBuffer.limit(mAudioDecoderOutputBufferInfo.offset + size); + encoderInputBuffer.position(0); + encoderInputBuffer.put(decoderOutputBuffer); + + mAudioEncoder.queueInputBuffer( + encoderInputBufferIndex, + 0, + size, + presentationTime, + mAudioDecoderOutputBufferInfo.flags); + } + mAudioDecoder.releaseOutputBuffer(mPendingAudioDecoderOutputBufferIndex, false); + mPendingAudioDecoderOutputBufferIndex = -1; + if ((mAudioDecoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { + if (VERBOSE) Log.d(TAG, "audio decoder: EOS"); + mAudioDecoderDone = true; + } + // We enqueued a pending frame, let's try something else next. + break; + } + + // Poll frames from the audio encoder and send them to the muxer. + while (!mAudioEncoderDone && (mEncoderOutputAudioFormat == null || mMuxer != null)) { + final int encoderOutputBufferIndex = mAudioEncoder.dequeueOutputBuffer(mAudioEncoderOutputBufferInfo, TIMEOUT_USEC); + if (encoderOutputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) { + if (VERBOSE) Log.d(TAG, "no audio encoder output buffer"); + break; + } + if (encoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { + if (VERBOSE) Log.d(TAG, "audio encoder: output buffers changed"); + mAudioEncoderOutputBuffers = mAudioEncoder.getOutputBuffers(); + break; + } + if (encoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { + if (VERBOSE) Log.d(TAG, "audio encoder: output format changed"); + Preconditions.checkState("audio encoder changed its output format again?", mOutputAudioTrack < 0); + + mEncoderOutputAudioFormat = mAudioEncoder.getOutputFormat(); + break; + } + Preconditions.checkState("should have added track before processing output", mMuxer != null); + if (VERBOSE) { + Log.d(TAG, "audio encoder: returned output buffer: " + encoderOutputBufferIndex); + Log.d(TAG, "audio encoder: returned buffer of size " + mAudioEncoderOutputBufferInfo.size); + } + final ByteBuffer encoderOutputBuffer = mAudioEncoderOutputBuffers[encoderOutputBufferIndex]; + if ((mAudioEncoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) { + if (VERBOSE) Log.d(TAG, "audio encoder: codec config buffer"); + // Simply ignore codec config buffers. + mAudioEncoder.releaseOutputBuffer(encoderOutputBufferIndex, false); + break; + } + if (VERBOSE) { + Log.d(TAG, "audio encoder: returned buffer for time " + mAudioEncoderOutputBufferInfo.presentationTimeUs); + } + if (mAudioEncoderOutputBufferInfo.size != 0) { + mMuxer.writeSampleData(mOutputAudioTrack, encoderOutputBuffer, mAudioEncoderOutputBufferInfo); + mMuxingAudioPresentationTime = Math.max(mMuxingAudioPresentationTime, mAudioEncoderOutputBufferInfo.presentationTimeUs); + } + if ((mAudioEncoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { + if (VERBOSE) Log.d(TAG, "audio encoder: EOS"); + mAudioEncoderDone = true; + } + mAudioEncoder.releaseOutputBuffer(encoderOutputBufferIndex, false); + mAudioEncodedFrameCount++; + // We enqueued an encoded frame, let's try something else next. + break; + } + } + + void release() throws Exception { + Exception exception = null; + try { + if (mAudioExtractor != null) { + mAudioExtractor.release(); + } + } catch (Exception e) { + Log.e(TAG, "error while releasing mAudioExtractor", e); + exception = e; + } + try { + if (mAudioDecoder != null) { + mAudioDecoder.stop(); + mAudioDecoder.release(); + } + } catch (Exception e) { + Log.e(TAG, "error while releasing mAudioDecoder", e); + if (exception == null) { + exception = e; + } + } + try { + if (mAudioEncoder != null) { + mAudioEncoder.stop(); + mAudioEncoder.release(); + } + } catch (Exception e) { + Log.e(TAG, "error while releasing mAudioEncoder", e); + if (exception == null) { + exception = e; + } + } + if (exception != null) { + throw exception; + } + } + + AudioTrackConverterState dumpState() { + return new AudioTrackConverterState( + mAudioExtractedFrameCount, mAudioExtractorDone, + mAudioDecodedFrameCount, mAudioDecoderDone, + mAudioEncodedFrameCount, mAudioEncoderDone, + mPendingAudioDecoderOutputBufferIndex, + mMuxer != null, mOutputAudioTrack); + } + + void verifyEndState() { + Preconditions.checkState("no frame should be pending", -1 == mPendingAudioDecoderOutputBufferIndex); + } + + @SuppressLint("WrongConstant") // flags extracted from sample by MediaExtractor should be safe for MediaCodec.BufferInfo + private void extractAndRemux() throws IOException { + if (mMuxer == null) { + Log.d(TAG, "audio remuxer: tried to execute before muxer was ready"); + return; + } + int size = mAudioExtractor.readSampleData(instanceSampleBuffer, 0); + long presentationTime = mAudioExtractor.getSampleTime(); + int sampleFlags = mAudioExtractor.getSampleFlags(); + if (VERBOSE) { + Log.d(TAG, "audio extractor: returned buffer of size " + size); + Log.d(TAG, "audio extractor: returned buffer for time " + presentationTime); + Log.d(TAG, "audio extractor: returned buffer with flags " + Integer.toBinaryString(sampleFlags)); + } + mAudioExtractorDone = isAudioExtractorDone(size, presentationTime); + + if (mAudioExtractorDone) { + if (VERBOSE) Log.d(TAG, "audio encoder: EOS"); + instanceBufferInfo.set(0, 0, presentationTime, MediaCodec.BUFFER_FLAG_END_OF_STREAM); + mAudioEncoderDone = true; + } else { + instanceBufferInfo.set(0, size, presentationTime, sampleFlags); + } + + mMuxer.writeSampleData(mOutputAudioTrack, instanceSampleBuffer, instanceBufferInfo); + + if (VERBOSE) { + Log.d(TAG, "audio extractor: wrote sample at " + presentationTime); + } + + mAudioExtractor.advance(); + + mAudioExtractedFrameCount++; + mAudioEncodedFrameCount++; + mMuxingAudioPresentationTime = Math.max(mMuxingAudioPresentationTime, presentationTime); + } + + private boolean isAudioExtractorDone(int size, long presentationTime) { + return presentationTime == -1 || size < 0 || (mTimeTo > 0 && presentationTime > mTimeTo * 1000); + } + + private static @NonNull + MediaCodec createAudioDecoder(final @NonNull MediaFormat inputFormat) throws IOException { + final MediaCodec decoder = MediaCodec.createDecoderByType(MediaConverter.getMimeTypeFor(inputFormat)); + decoder.configure(inputFormat, null, null, 0); + decoder.start(); + return decoder; + } + + private static @NonNull + MediaCodec createAudioEncoder(final @NonNull MediaCodecInfo codecInfo, final @NonNull MediaFormat format) throws IOException { + final MediaCodec encoder = MediaCodec.createByCodecName(codecInfo.getName()); + encoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); + encoder.start(); + return encoder; + } + + private static int getAndSelectAudioTrackIndex(MediaExtractor extractor) { + for (int index = 0; index < extractor.getTrackCount(); ++index) { + if (VERBOSE) { + Log.d(TAG, "format for track " + index + " is " + MediaConverter.getMimeTypeFor(extractor.getTrackFormat(index))); + } + if (isAudioFormat(extractor.getTrackFormat(index))) { + extractor.selectTrack(index); + return index; + } + } + return -1; + } + + private static boolean isAudioFormat(final @NonNull MediaFormat format) { + return MediaConverter.getMimeTypeFor(format).startsWith("audio/"); + } + + /** + * HE-AAC input bitstreams exhibit bad decoder behavior: the decoder's output buffer's presentation timestamp is way larger than the input sample's. + * This mismatch propagates throughout the transcoding pipeline and results in slowed, distorted audio in the output file. + * To sidestep this: AAC and its variants are a supported output codec, and HE-AAC bitrates are almost always lower than our target bitrate, + * so we can pass through the input bitstream unaltered, relying on consumers of the output file to render HE-AAC correctly. + */ + private static boolean formatCanSkipTranscode(MediaFormat audioFormat, int desiredBitrate) { + try { + int inputBitrate = audioFormat.getInteger(MediaFormat.KEY_BIT_RATE); + String inputMimeType = audioFormat.getString(MediaFormat.KEY_MIME); + return OUTPUT_AUDIO_MIME_TYPE.equals(inputMimeType) && inputBitrate <= desiredBitrate; + } catch (NullPointerException exception) { + if (VERBOSE) { + Log.d(TAG, "could not find bitrate in mediaFormat, can't skip transcoding."); + } + return false; + } + } +} diff --git a/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/VideoThumbnailsExtractor.java b/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/VideoThumbnailsExtractor.java index f5f4477ed0..669f8a9d46 100644 --- a/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/VideoThumbnailsExtractor.java +++ b/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/VideoThumbnailsExtractor.java @@ -1,228 +1,228 @@ -package org.thoughtcrime.securesms.video.videoconverter; - -import android.graphics.Bitmap; -import android.media.MediaCodec; -import android.media.MediaExtractor; -import android.media.MediaFormat; -import android.opengl.GLES20; -import android.os.Build; -import android.os.Bundle; - -import androidx.annotation.NonNull; -import androidx.annotation.RequiresApi; - -import org.signal.core.util.logging.Log; -import org.thoughtcrime.securesms.video.interfaces.MediaInput; -import org.thoughtcrime.securesms.video.videoconverter.utils.MediaCodecCompat; - -import java.nio.ByteBuffer; -import java.nio.ByteOrder; - -@RequiresApi(api = 23) -final class VideoThumbnailsExtractor { - - private static final String TAG = Log.tag(VideoThumbnailsExtractor.class); - - interface Callback { - void durationKnown(long duration); - - boolean publishProgress(int index, Bitmap thumbnail); - - void failed(); - } - - static void extractThumbnails(final @NonNull MediaInput input, - final int thumbnailCount, - final int thumbnailResolution, - final @NonNull Callback callback) - { - MediaExtractor extractor = null; - MediaCodec decoder = null; - OutputSurface outputSurface = null; - try { - extractor = input.createExtractor(); - MediaFormat mediaFormat = null; - for (int index = 0; index < extractor.getTrackCount(); ++index) { - final String mimeType = extractor.getTrackFormat(index).getString(MediaFormat.KEY_MIME); - if (mimeType != null && mimeType.startsWith("video/")) { - extractor.selectTrack(index); - mediaFormat = extractor.getTrackFormat(index); - break; - } - } - if (mediaFormat != null) { - final String mime = mediaFormat.getString(MediaFormat.KEY_MIME); - if (mime == null) { - throw new IllegalArgumentException("Mime type for MediaFormat was null: \t" + mediaFormat); - } - - final int rotation = mediaFormat.containsKey(MediaFormat.KEY_ROTATION) ? mediaFormat.getInteger(MediaFormat.KEY_ROTATION) : 0; - final int width = mediaFormat.getInteger(MediaFormat.KEY_WIDTH); - final int height = mediaFormat.getInteger(MediaFormat.KEY_HEIGHT); - final int outputWidth; - final int outputHeight; - - if (width < height) { - outputWidth = thumbnailResolution; - outputHeight = height * outputWidth / width; - } else { - outputHeight = thumbnailResolution; - outputWidth = width * outputHeight / height; - } - - final int outputWidthRotated; - final int outputHeightRotated; - - if ((rotation % 180 == 90)) { - //noinspection SuspiciousNameCombination - outputWidthRotated = outputHeight; - //noinspection SuspiciousNameCombination - outputHeightRotated = outputWidth; - } else { - outputWidthRotated = outputWidth; - outputHeightRotated = outputHeight; - } - - Log.i(TAG, "video :" + width + "x" + height + " " + rotation); - Log.i(TAG, "output: " + outputWidthRotated + "x" + outputHeightRotated); - - outputSurface = new OutputSurface(outputWidthRotated, outputHeightRotated, true); - - decoder = MediaCodec.createDecoderByType(mime); - final boolean isHdr = MediaCodecCompat.isHdrVideo(mediaFormat); - if (Build.VERSION.SDK_INT >= 31 && isHdr) { - mediaFormat.setInteger(MediaFormat.KEY_COLOR_TRANSFER_REQUEST, MediaFormat.COLOR_TRANSFER_SDR_VIDEO); - } - decoder.configure(mediaFormat, outputSurface.getSurface(), null, 0); - decoder.start(); - if (Build.VERSION.SDK_INT >= 31 && isHdr) { - try { - final String VENDOR_DOLBY_CODEC_TRANSFER_PARAMKEY = "vendor.dolby.codec.transfer.value"; - MediaCodec.ParameterDescriptor descriptor = decoder.getParameterDescriptor(VENDOR_DOLBY_CODEC_TRANSFER_PARAMKEY); - if (descriptor != null) { - Bundle transferBundle = new Bundle(); - transferBundle.putString(VENDOR_DOLBY_CODEC_TRANSFER_PARAMKEY, "transfer.sdr.normal"); - decoder.setParameters(transferBundle); - } - } catch (IllegalStateException e) { - Log.w(TAG, "Failed to set Dolby Vision transfer parameter", e); - } - } - - long duration = 0; - - if (mediaFormat.containsKey(MediaFormat.KEY_DURATION)) { - duration = mediaFormat.getLong(MediaFormat.KEY_DURATION); - } else { - Log.w(TAG, "Video is missing duration!"); - } - - callback.durationKnown(duration); - - doExtract(extractor, decoder, outputSurface, outputWidthRotated, outputHeightRotated, duration, thumbnailCount, callback); - } - } catch (Throwable t) { - Log.w(TAG, t); - callback.failed(); - } finally { - if (outputSurface != null) { - outputSurface.release(); - } - if (decoder != null) { - try { - decoder.stop(); - } catch (MediaCodec.CodecException codecException) { - Log.w(TAG, "Decoder stop failed: " + codecException.getDiagnosticInfo(), codecException); - } catch (IllegalStateException ise) { - Log.w(TAG, "Decoder stop failed", ise); - } - decoder.release(); - } - if (extractor != null) { - extractor.release(); - } - } - } - - private static void doExtract(final @NonNull MediaExtractor extractor, - final @NonNull MediaCodec decoder, - final @NonNull OutputSurface outputSurface, - final int outputWidth, int outputHeight, long duration, int thumbnailCount, - final @NonNull Callback callback) - throws TranscodingException - { - - final int TIMEOUT_USEC = 10000; - final ByteBuffer[] decoderInputBuffers = decoder.getInputBuffers(); - final MediaCodec.BufferInfo info = new MediaCodec.BufferInfo(); - - int samplesExtracted = 0; - int thumbnailsCreated = 0; - - Log.i(TAG, "doExtract started"); - final ByteBuffer pixelBuf = ByteBuffer.allocateDirect(outputWidth * outputHeight * 4); - pixelBuf.order(ByteOrder.LITTLE_ENDIAN); - - boolean outputDone = false; - boolean inputDone = false; - while (!outputDone) { - if (!inputDone) { - int inputBufIndex = decoder.dequeueInputBuffer(TIMEOUT_USEC); - if (inputBufIndex >= 0) { - final ByteBuffer inputBuf = decoderInputBuffers[inputBufIndex]; - final int sampleSize = extractor.readSampleData(inputBuf, 0); - if (sampleSize < 0 || samplesExtracted >= thumbnailCount) { - decoder.queueInputBuffer(inputBufIndex, 0, 0, 0L, MediaCodec.BUFFER_FLAG_END_OF_STREAM); - inputDone = true; - Log.i(TAG, "input done"); - } else { - final long presentationTimeUs = extractor.getSampleTime(); - decoder.queueInputBuffer(inputBufIndex, 0, sampleSize, presentationTimeUs, 0 /*flags*/); - samplesExtracted++; - extractor.seekTo(duration * samplesExtracted / thumbnailCount, MediaExtractor.SEEK_TO_CLOSEST_SYNC); - Log.i(TAG, "seek to " + duration * samplesExtracted / thumbnailCount + ", actual " + extractor.getSampleTime()); - } - } - } - - final int outputBufIndex; - try { - outputBufIndex = decoder.dequeueOutputBuffer(info, TIMEOUT_USEC); - } catch (IllegalStateException e) { - Log.w(TAG, "Decoder not in the Executing state, or codec is configured in asynchronous mode.", e); - throw new TranscodingException("Decoder not in the Executing state, or codec is configured in asynchronous mode.", e); - } - - if (outputBufIndex >= 0) { - if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { - outputDone = true; - } - - final boolean shouldRender = (info.size != 0) /*&& (info.presentationTimeUs >= duration * decodeCount / thumbnailCount)*/; - - decoder.releaseOutputBuffer(outputBufIndex, shouldRender); - if (shouldRender) { - outputSurface.awaitNewImage(); - outputSurface.drawImage(); - - if (thumbnailsCreated < thumbnailCount) { - pixelBuf.rewind(); - GLES20.glReadPixels(0, 0, outputWidth, outputHeight, GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, pixelBuf); - - final Bitmap bitmap = Bitmap.createBitmap(outputWidth, outputHeight, Bitmap.Config.ARGB_8888); - pixelBuf.rewind(); - bitmap.copyPixelsFromBuffer(pixelBuf); - - if (!callback.publishProgress(thumbnailsCreated, bitmap)) { - break; - } - Log.i(TAG, "publishProgress for frame " + thumbnailsCreated + " at " + info.presentationTimeUs + " (target " + duration * thumbnailsCreated / thumbnailCount + ")"); - } - thumbnailsCreated++; - } - } - } - Log.i(TAG, "doExtract finished"); - } - +package org.thoughtcrime.securesms.video.videoconverter; + +import android.graphics.Bitmap; +import android.media.MediaCodec; +import android.media.MediaExtractor; +import android.media.MediaFormat; +import android.opengl.GLES20; +import android.os.Build; +import android.os.Bundle; + +import androidx.annotation.NonNull; +import androidx.annotation.RequiresApi; + +import org.signal.core.util.logging.Log; +import org.thoughtcrime.securesms.video.interfaces.MediaInput; +import org.thoughtcrime.securesms.video.videoconverter.utils.MediaCodecCompat; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +@RequiresApi(api = 23) +final class VideoThumbnailsExtractor { + + private static final String TAG = Log.tag(VideoThumbnailsExtractor.class); + + interface Callback { + void durationKnown(long duration); + + boolean publishProgress(int index, Bitmap thumbnail); + + void failed(); + } + + static void extractThumbnails(final @NonNull MediaInput input, + final int thumbnailCount, + final int thumbnailResolution, + final @NonNull Callback callback) + { + MediaExtractor extractor = null; + MediaCodec decoder = null; + OutputSurface outputSurface = null; + try { + extractor = input.createExtractor(); + MediaFormat mediaFormat = null; + for (int index = 0; index < extractor.getTrackCount(); ++index) { + final String mimeType = extractor.getTrackFormat(index).getString(MediaFormat.KEY_MIME); + if (mimeType != null && mimeType.startsWith("video/")) { + extractor.selectTrack(index); + mediaFormat = extractor.getTrackFormat(index); + break; + } + } + if (mediaFormat != null) { + final String mime = mediaFormat.getString(MediaFormat.KEY_MIME); + if (mime == null) { + throw new IllegalArgumentException("Mime type for MediaFormat was null: \t" + mediaFormat); + } + + final int rotation = mediaFormat.containsKey(MediaFormat.KEY_ROTATION) ? mediaFormat.getInteger(MediaFormat.KEY_ROTATION) : 0; + final int width = mediaFormat.getInteger(MediaFormat.KEY_WIDTH); + final int height = mediaFormat.getInteger(MediaFormat.KEY_HEIGHT); + final int outputWidth; + final int outputHeight; + + if (width < height) { + outputWidth = thumbnailResolution; + outputHeight = height * outputWidth / width; + } else { + outputHeight = thumbnailResolution; + outputWidth = width * outputHeight / height; + } + + final int outputWidthRotated; + final int outputHeightRotated; + + if ((rotation % 180 == 90)) { + //noinspection SuspiciousNameCombination + outputWidthRotated = outputHeight; + //noinspection SuspiciousNameCombination + outputHeightRotated = outputWidth; + } else { + outputWidthRotated = outputWidth; + outputHeightRotated = outputHeight; + } + + Log.i(TAG, "video :" + width + "x" + height + " " + rotation); + Log.i(TAG, "output: " + outputWidthRotated + "x" + outputHeightRotated); + + outputSurface = new OutputSurface(outputWidthRotated, outputHeightRotated, true); + + decoder = MediaCodec.createDecoderByType(mime); + final boolean isHdr = MediaCodecCompat.isHdrVideo(mediaFormat); + if (Build.VERSION.SDK_INT >= 31 && isHdr) { + mediaFormat.setInteger(MediaFormat.KEY_COLOR_TRANSFER_REQUEST, MediaFormat.COLOR_TRANSFER_SDR_VIDEO); + } + decoder.configure(mediaFormat, outputSurface.getSurface(), null, 0); + decoder.start(); + if (Build.VERSION.SDK_INT >= 31 && isHdr) { + try { + final String VENDOR_DOLBY_CODEC_TRANSFER_PARAMKEY = "vendor.dolby.codec.transfer.value"; + MediaCodec.ParameterDescriptor descriptor = decoder.getParameterDescriptor(VENDOR_DOLBY_CODEC_TRANSFER_PARAMKEY); + if (descriptor != null) { + Bundle transferBundle = new Bundle(); + transferBundle.putString(VENDOR_DOLBY_CODEC_TRANSFER_PARAMKEY, "transfer.sdr.normal"); + decoder.setParameters(transferBundle); + } + } catch (IllegalStateException e) { + Log.w(TAG, "Failed to set Dolby Vision transfer parameter", e); + } + } + + long duration = 0; + + if (mediaFormat.containsKey(MediaFormat.KEY_DURATION)) { + duration = mediaFormat.getLong(MediaFormat.KEY_DURATION); + } else { + Log.w(TAG, "Video is missing duration!"); + } + + callback.durationKnown(duration); + + doExtract(extractor, decoder, outputSurface, outputWidthRotated, outputHeightRotated, duration, thumbnailCount, callback); + } + } catch (Throwable t) { + Log.w(TAG, t); + callback.failed(); + } finally { + if (outputSurface != null) { + outputSurface.release(); + } + if (decoder != null) { + try { + decoder.stop(); + } catch (MediaCodec.CodecException codecException) { + Log.w(TAG, "Decoder stop failed: " + codecException.getDiagnosticInfo(), codecException); + } catch (IllegalStateException ise) { + Log.w(TAG, "Decoder stop failed", ise); + } + decoder.release(); + } + if (extractor != null) { + extractor.release(); + } + } + } + + private static void doExtract(final @NonNull MediaExtractor extractor, + final @NonNull MediaCodec decoder, + final @NonNull OutputSurface outputSurface, + final int outputWidth, int outputHeight, long duration, int thumbnailCount, + final @NonNull Callback callback) + throws TranscodingException + { + + final int TIMEOUT_USEC = 10000; + final ByteBuffer[] decoderInputBuffers = decoder.getInputBuffers(); + final MediaCodec.BufferInfo info = new MediaCodec.BufferInfo(); + + int samplesExtracted = 0; + int thumbnailsCreated = 0; + + Log.i(TAG, "doExtract started"); + final ByteBuffer pixelBuf = ByteBuffer.allocateDirect(outputWidth * outputHeight * 4); + pixelBuf.order(ByteOrder.LITTLE_ENDIAN); + + boolean outputDone = false; + boolean inputDone = false; + while (!outputDone) { + if (!inputDone) { + int inputBufIndex = decoder.dequeueInputBuffer(TIMEOUT_USEC); + if (inputBufIndex >= 0) { + final ByteBuffer inputBuf = decoderInputBuffers[inputBufIndex]; + final int sampleSize = extractor.readSampleData(inputBuf, 0); + if (sampleSize < 0 || samplesExtracted >= thumbnailCount) { + decoder.queueInputBuffer(inputBufIndex, 0, 0, 0L, MediaCodec.BUFFER_FLAG_END_OF_STREAM); + inputDone = true; + Log.i(TAG, "input done"); + } else { + final long presentationTimeUs = extractor.getSampleTime(); + decoder.queueInputBuffer(inputBufIndex, 0, sampleSize, presentationTimeUs, 0 /*flags*/); + samplesExtracted++; + extractor.seekTo(duration * samplesExtracted / thumbnailCount, MediaExtractor.SEEK_TO_CLOSEST_SYNC); + Log.i(TAG, "seek to " + duration * samplesExtracted / thumbnailCount + ", actual " + extractor.getSampleTime()); + } + } + } + + final int outputBufIndex; + try { + outputBufIndex = decoder.dequeueOutputBuffer(info, TIMEOUT_USEC); + } catch (IllegalStateException e) { + Log.w(TAG, "Decoder not in the Executing state, or codec is configured in asynchronous mode.", e); + throw new TranscodingException("Decoder not in the Executing state, or codec is configured in asynchronous mode.", e); + } + + if (outputBufIndex >= 0) { + if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { + outputDone = true; + } + + final boolean shouldRender = (info.size != 0) /*&& (info.presentationTimeUs >= duration * decodeCount / thumbnailCount)*/; + + decoder.releaseOutputBuffer(outputBufIndex, shouldRender); + if (shouldRender) { + outputSurface.awaitNewImage(); + outputSurface.drawImage(); + + if (thumbnailsCreated < thumbnailCount) { + pixelBuf.rewind(); + GLES20.glReadPixels(0, 0, outputWidth, outputHeight, GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, pixelBuf); + + final Bitmap bitmap = Bitmap.createBitmap(outputWidth, outputHeight, Bitmap.Config.ARGB_8888); + pixelBuf.rewind(); + bitmap.copyPixelsFromBuffer(pixelBuf); + + if (!callback.publishProgress(thumbnailsCreated, bitmap)) { + break; + } + Log.i(TAG, "publishProgress for frame " + thumbnailsCreated + " at " + info.presentationTimeUs + " (target " + duration * thumbnailsCreated / thumbnailCount + ")"); + } + thumbnailsCreated++; + } + } + } + Log.i(TAG, "doExtract finished"); + } + } \ No newline at end of file diff --git a/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/VideoTrackConverter.java b/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/VideoTrackConverter.java index 18624c97e6..c4c1bb7b33 100644 --- a/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/VideoTrackConverter.java +++ b/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/VideoTrackConverter.java @@ -1,703 +1,703 @@ -package org.thoughtcrime.securesms.video.videoconverter; - -import android.media.MediaCodec; -import android.media.MediaCodecInfo; -import android.media.MediaExtractor; -import android.media.MediaFormat; -import android.os.Build; -import android.view.Surface; - -import androidx.annotation.NonNull; -import androidx.annotation.Nullable; - -import org.signal.core.util.logging.Log; -import org.thoughtcrime.securesms.video.interfaces.MediaInput; -import org.thoughtcrime.securesms.video.interfaces.Muxer; -import org.thoughtcrime.securesms.video.videoconverter.exceptions.CodecUnavailableException; -import org.thoughtcrime.securesms.video.videoconverter.exceptions.HdrDecoderUnavailableException; -import org.thoughtcrime.securesms.video.videoconverter.utils.Extensions; -import org.thoughtcrime.securesms.video.videoconverter.utils.MediaCodecCompat; -import org.thoughtcrime.securesms.video.videoconverter.utils.Preconditions; - -import java.io.FileNotFoundException; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.List; -import java.util.Set; -import kotlin.Pair; - -final class VideoTrackConverter { - - private static final String TAG = "media-converter"; - private static final boolean VERBOSE = false; // lots of logging - - private static final int OUTPUT_VIDEO_IFRAME_INTERVAL = 1; // 1 second between I-frames - private static final int OUTPUT_VIDEO_FRAME_RATE = 30; // needed only for MediaFormat.KEY_I_FRAME_INTERVAL to work; the actual frame rate matches the source - - private static final int TIMEOUT_USEC = 10000; - - private static final String MEDIA_FORMAT_KEY_DISPLAY_WIDTH = "display-width"; - private static final String MEDIA_FORMAT_KEY_DISPLAY_HEIGHT = "display-height"; - - private static final float FRAME_RATE_TOLERANCE = 0.05f; // tolerance for transcoding VFR -> CFR - - private boolean mIsHdrInput; - private boolean mToneMapApplied; - private String mDecoderName; - private String mEncoderName; - - private final long mTimeFrom; - private final long mTimeTo; - - final long mInputDuration; - - private final MediaExtractor mVideoExtractor; - private final MediaCodec mVideoDecoder; - private MediaCodec mVideoEncoder; - - private InputSurface mInputSurface; - private final OutputSurface mOutputSurface; - - private final ByteBuffer[] mVideoDecoderInputBuffers; - private ByteBuffer[] mVideoEncoderOutputBuffers; - private final MediaCodec.BufferInfo mVideoDecoderOutputBufferInfo; - private final MediaCodec.BufferInfo mVideoEncoderOutputBufferInfo; - - MediaFormat mEncoderOutputVideoFormat; - - boolean mVideoExtractorDone; - private boolean mVideoDecoderDone; - boolean mVideoEncoderDone; - - private int mOutputVideoTrack = -1; - - long mMuxingVideoPresentationTime; - - private int mVideoExtractedFrameCount; - private int mVideoDecodedFrameCount; - private int mVideoEncodedFrameCount; - - private Muxer mMuxer; - - static @Nullable VideoTrackConverter create( - final @NonNull MediaInput input, - final long timeFrom, - final long timeTo, - final int videoResolution, - final int videoBitrate, - final @NonNull String videoCodec, - final @NonNull Set excludedDecoders) throws IOException, TranscodingException { - - final MediaExtractor videoExtractor = input.createExtractor(); - final int videoInputTrack = getAndSelectVideoTrackIndex(videoExtractor); - if (videoInputTrack == -1) { - videoExtractor.release(); - return null; - } - return new VideoTrackConverter(videoExtractor, videoInputTrack, timeFrom, timeTo, videoResolution, videoBitrate, videoCodec, excludedDecoders); - } - - - private VideoTrackConverter( - final @NonNull MediaExtractor videoExtractor, - final int videoInputTrack, - final long timeFrom, - final long timeTo, - final int videoResolution, - final int videoBitrate, - final @NonNull String videoCodec, - final @NonNull Set excludedDecoders) throws IOException, TranscodingException { - - mTimeFrom = timeFrom; - mTimeTo = timeTo; - mVideoExtractor = videoExtractor; - - final List videoCodecCandidates = MediaConverter.selectCodecs(videoCodec); - if (videoCodecCandidates.isEmpty()) { - // Don't fail CTS if they don't have an AVC codec (not here, anyway). - Log.e(TAG, "Unable to find an appropriate codec for " + videoCodec); - throw new FileNotFoundException(); - } - if (VERBOSE) Log.d(TAG, "video found codecs: " + videoCodecCandidates.size()); - - final MediaFormat inputVideoFormat = mVideoExtractor.getTrackFormat(videoInputTrack); - - mInputDuration = inputVideoFormat.containsKey(MediaFormat.KEY_DURATION) ? inputVideoFormat.getLong(MediaFormat.KEY_DURATION) : 0; - - final int rotation = inputVideoFormat.containsKey(MediaFormat.KEY_ROTATION) ? inputVideoFormat.getInteger(MediaFormat.KEY_ROTATION) : 0; - final int width = inputVideoFormat.containsKey(MEDIA_FORMAT_KEY_DISPLAY_WIDTH) - ? inputVideoFormat.getInteger(MEDIA_FORMAT_KEY_DISPLAY_WIDTH) - : inputVideoFormat.getInteger(MediaFormat.KEY_WIDTH); - final int height = inputVideoFormat.containsKey(MEDIA_FORMAT_KEY_DISPLAY_HEIGHT) - ? inputVideoFormat.getInteger(MEDIA_FORMAT_KEY_DISPLAY_HEIGHT) - : inputVideoFormat.getInteger(MediaFormat.KEY_HEIGHT); - int outputWidth = width; - int outputHeight = height; - if (outputWidth < outputHeight) { - outputWidth = videoResolution; - outputHeight = height * outputWidth / width; - } else { - outputHeight = videoResolution; - outputWidth = width * outputHeight / height; - } - // many encoders do not work when height and width are not multiple of 16 (also, some iPhones do not play some heights) - outputHeight = (outputHeight + 7) & ~0xF; - outputWidth = (outputWidth + 7) & ~0xF; - - final int outputWidthRotated; - final int outputHeightRotated; - if ((rotation % 180 == 90)) { - //noinspection SuspiciousNameCombination - outputWidthRotated = outputHeight; - //noinspection SuspiciousNameCombination - outputHeightRotated = outputWidth; - } else { - outputWidthRotated = outputWidth; - outputHeightRotated = outputHeight; - } - - final MediaFormat outputVideoFormat = MediaFormat.createVideoFormat(videoCodec, outputWidthRotated, outputHeightRotated); - - // Set some properties. Failing to specify some of these can cause the MediaCodec - // configure() call to throw an unhelpful exception. - outputVideoFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface); - outputVideoFormat.setInteger(MediaFormat.KEY_BIT_RATE, videoBitrate); - outputVideoFormat.setInteger(MediaFormat.KEY_BITRATE_MODE, MediaCodecInfo.EncoderCapabilities.BITRATE_MODE_CBR); - outputVideoFormat.setInteger(MediaFormat.KEY_FRAME_RATE, OUTPUT_VIDEO_FRAME_RATE); - outputVideoFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, OUTPUT_VIDEO_IFRAME_INTERVAL); - if (VERBOSE) Log.d(TAG, "video format: " + outputVideoFormat); - - final String fragmentShader = createFragmentShader( - inputVideoFormat.getInteger(MediaFormat.KEY_WIDTH), inputVideoFormat.getInteger(MediaFormat.KEY_HEIGHT), - outputWidth, outputHeight); - - // Create encoder, decoder, and surfaces. The encoder's start() is deferred - // until after the decoder is created, so that the decoder gets first access to - // hardware codec resources on memory-constrained devices. If start() fails - // (e.g. NO_MEMORY on a resource-constrained device), we try the next encoder - // candidate while keeping the same decoder and OutputSurface. - mVideoEncoder = createVideoEncoder(videoCodecCandidates, outputVideoFormat); - mInputSurface = new InputSurface(mVideoEncoder.createInputSurface()); - mInputSurface.makeCurrent(); - mOutputSurface = new OutputSurface(); - mOutputSurface.changeFragmentShader(fragmentShader); - mVideoDecoder = createVideoDecoder(inputVideoFormat, mOutputSurface.getSurface(), excludedDecoders); - startEncoderWithFallback(videoCodecCandidates, outputVideoFormat); - - mVideoDecoderInputBuffers = mVideoDecoder.getInputBuffers(); - mVideoEncoderOutputBuffers = mVideoEncoder.getOutputBuffers(); - mVideoDecoderOutputBufferInfo = new MediaCodec.BufferInfo(); - mVideoEncoderOutputBufferInfo = new MediaCodec.BufferInfo(); - - if (mTimeFrom > 0) { - mVideoExtractor.seekTo(mTimeFrom * 1000, MediaExtractor.SEEK_TO_PREVIOUS_SYNC); - Log.i(TAG, "Seek video:" + mTimeFrom + " " + mVideoExtractor.getSampleTime()); - } - } - - void setMuxer(final @NonNull Muxer muxer) throws IOException { - mMuxer = muxer; - if (mEncoderOutputVideoFormat != null) { - Log.d(TAG, "muxer: adding video track."); - mOutputVideoTrack = muxer.addTrack(mEncoderOutputVideoFormat); - } - } - - void step() throws IOException, TranscodingException { - // Extract video from file and feed to decoder. - // Do not extract video if we have determined the output format but we are not yet - // ready to mux the frames. - while (!mVideoExtractorDone - && (mEncoderOutputVideoFormat == null || mMuxer != null)) { - int decoderInputBufferIndex = mVideoDecoder.dequeueInputBuffer(TIMEOUT_USEC); - if (decoderInputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) { - if (VERBOSE) Log.d(TAG, "no video decoder input buffer"); - break; - } - if (VERBOSE) { - Log.d(TAG, "video decoder: returned input buffer: " + decoderInputBufferIndex); - } - final ByteBuffer decoderInputBuffer = mVideoDecoderInputBuffers[decoderInputBufferIndex]; - final int size = mVideoExtractor.readSampleData(decoderInputBuffer, 0); - final long presentationTime = mVideoExtractor.getSampleTime(); - if (VERBOSE) { - Log.d(TAG, "video extractor: returned buffer of size " + size); - Log.d(TAG, "video extractor: returned buffer for time " + presentationTime); - } - mVideoExtractorDone = size < 0 || (mTimeTo > 0 && presentationTime > mTimeTo * 1000); - - if (mVideoExtractorDone) { - if (VERBOSE) Log.d(TAG, "video extractor: EOS"); - mVideoDecoder.queueInputBuffer( - decoderInputBufferIndex, - 0, - 0, - 0, - MediaCodec.BUFFER_FLAG_END_OF_STREAM); - } else { - mVideoDecoder.queueInputBuffer( - decoderInputBufferIndex, - 0, - size, - presentationTime, - mVideoExtractor.getSampleFlags()); - } - mVideoExtractor.advance(); - mVideoExtractedFrameCount++; - // We extracted a frame, let's try something else next. - break; - } - - // Poll output frames from the video decoder and feed the encoder. - while (!mVideoDecoderDone && (mEncoderOutputVideoFormat == null || mMuxer != null)) { - final int decoderOutputBufferIndex = - mVideoDecoder.dequeueOutputBuffer( - mVideoDecoderOutputBufferInfo, TIMEOUT_USEC); - if (decoderOutputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) { - if (VERBOSE) Log.d(TAG, "no video decoder output buffer"); - break; - } - if (decoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { - if (VERBOSE) Log.d(TAG, "video decoder: output buffers changed"); - break; - } - if (decoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { - if (VERBOSE) { - Log.d(TAG, "video decoder: output format changed: " + mVideoDecoder.getOutputFormat()); - } - break; - } - if (VERBOSE) { - Log.d(TAG, "video decoder: returned output buffer: " - + decoderOutputBufferIndex); - Log.d(TAG, "video decoder: returned buffer of size " - + mVideoDecoderOutputBufferInfo.size); - } - if ((mVideoDecoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) { - if (VERBOSE) Log.d(TAG, "video decoder: codec config buffer"); - mVideoDecoder.releaseOutputBuffer(decoderOutputBufferIndex, false); - break; - } - if (mVideoDecoderOutputBufferInfo.presentationTimeUs < mTimeFrom * 1000 && - (mVideoDecoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) == 0) { - if (VERBOSE) Log.d(TAG, "video decoder: frame prior to " + mVideoDecoderOutputBufferInfo.presentationTimeUs); - mVideoDecoder.releaseOutputBuffer(decoderOutputBufferIndex, false); - break; - } - if (VERBOSE) { - Log.d(TAG, "video decoder: returned buffer for time " + mVideoDecoderOutputBufferInfo.presentationTimeUs); - } - boolean render = mVideoDecoderOutputBufferInfo.size != 0; - mVideoDecoder.releaseOutputBuffer(decoderOutputBufferIndex, render); - if (render) { - if (VERBOSE) Log.d(TAG, "output surface: await new image"); - mOutputSurface.awaitNewImage(); - // Edit the frame and send it to the encoder. - if (VERBOSE) Log.d(TAG, "output surface: draw image"); - mOutputSurface.drawImage(); - mInputSurface.setPresentationTime(mVideoDecoderOutputBufferInfo.presentationTimeUs * 1000); - if (VERBOSE) Log.d(TAG, "input surface: swap buffers"); - mInputSurface.swapBuffers(); - if (VERBOSE) Log.d(TAG, "video encoder: notified of new frame"); - } - if ((mVideoDecoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { - if (VERBOSE) Log.d(TAG, "video decoder: EOS"); - mVideoDecoderDone = true; - mVideoEncoder.signalEndOfInputStream(); - } - mVideoDecodedFrameCount++; - // We extracted a pending frame, let's try something else next. - break; - } - - // Poll frames from the video encoder and send them to the muxer. - while (!mVideoEncoderDone && (mEncoderOutputVideoFormat == null || mMuxer != null)) { - final int encoderOutputBufferIndex = mVideoEncoder.dequeueOutputBuffer(mVideoEncoderOutputBufferInfo, TIMEOUT_USEC); - if (encoderOutputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) { - if (VERBOSE) Log.d(TAG, "no video encoder output buffer"); - if (mVideoDecoderDone) { - // on some devices and encoder stops after signalEndOfInputStream - Log.w(TAG, "mVideoDecoderDone, but didn't get BUFFER_FLAG_END_OF_STREAM"); - mVideoEncodedFrameCount = mVideoDecodedFrameCount; - mVideoEncoderDone = true; - } - break; - } - if (encoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { - if (VERBOSE) Log.d(TAG, "video encoder: output buffers changed"); - mVideoEncoderOutputBuffers = mVideoEncoder.getOutputBuffers(); - break; - } - if (encoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { - if (VERBOSE) Log.d(TAG, "video encoder: output format changed"); - Preconditions.checkState("video encoder changed its output format again?", mOutputVideoTrack < 0); - mEncoderOutputVideoFormat = mVideoEncoder.getOutputFormat(); - break; - } - Preconditions.checkState("should have added track before processing output", mMuxer != null); - if (VERBOSE) { - Log.d(TAG, "video encoder: returned output buffer: " + encoderOutputBufferIndex); - Log.d(TAG, "video encoder: returned buffer of size " + mVideoEncoderOutputBufferInfo.size); - } - final ByteBuffer encoderOutputBuffer = mVideoEncoderOutputBuffers[encoderOutputBufferIndex]; - if ((mVideoEncoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) { - if (VERBOSE) Log.d(TAG, "video encoder: codec config buffer"); - // Simply ignore codec config buffers. - mVideoEncoder.releaseOutputBuffer(encoderOutputBufferIndex, false); - break; - } - if (VERBOSE) { - Log.d(TAG, "video encoder: returned buffer for time " + mVideoEncoderOutputBufferInfo.presentationTimeUs); - } - if (mVideoEncoderOutputBufferInfo.size != 0) { - mMuxer.writeSampleData(mOutputVideoTrack, encoderOutputBuffer, mVideoEncoderOutputBufferInfo); - mMuxingVideoPresentationTime = Math.max(mMuxingVideoPresentationTime, mVideoEncoderOutputBufferInfo.presentationTimeUs); - } - if ((mVideoEncoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { - if (VERBOSE) Log.d(TAG, "video encoder: EOS"); - mVideoEncoderDone = true; - } - mVideoEncoder.releaseOutputBuffer(encoderOutputBufferIndex, false); - mVideoEncodedFrameCount++; - // We enqueued an encoded frame, let's try something else next. - break; - } - } - - void release() throws Exception { - Exception exception = null; - try { - if (mVideoExtractor != null) { - mVideoExtractor.release(); - } - } catch (Exception e) { - Log.e(TAG, "error while releasing mVideoExtractor", e); - exception = e; - } - try { - if (mVideoDecoder != null) { - mVideoDecoder.stop(); - mVideoDecoder.release(); - } - } catch (Exception e) { - Log.e(TAG, "error while releasing mVideoDecoder", e); - if (exception == null) { - exception = e; - } - } - try { - if (mOutputSurface != null) { - mOutputSurface.release(); - } - } catch (Exception e) { - Log.e(TAG, "error while releasing mOutputSurface", e); - if (exception == null) { - exception = e; - } - } - try { - if (mInputSurface != null) { - mInputSurface.release(); - } - } catch (Exception e) { - Log.e(TAG, "error while releasing mInputSurface", e); - if (exception == null) { - exception = e; - } - } - try { - if (mVideoEncoder != null) { - mVideoEncoder.stop(); - mVideoEncoder.release(); - } - } catch (Exception e) { - Log.e(TAG, "error while releasing mVideoEncoder", e); - if (exception == null) { - exception = e; - } - } - if (exception != null) { - throw exception; - } - } - - VideoTrackConverterState dumpState() { - return new VideoTrackConverterState( - mVideoExtractedFrameCount, mVideoExtractorDone, - mVideoDecodedFrameCount, mVideoDecoderDone, - mVideoEncodedFrameCount, mVideoEncoderDone, - mMuxer != null, mOutputVideoTrack); - } - - void verifyEndState() { - Preconditions.checkState("encoded (" + mVideoEncodedFrameCount + ") and decoded (" + mVideoDecodedFrameCount + ") video frame counts should match", Extensions.isWithin(mVideoDecodedFrameCount, mVideoEncodedFrameCount, FRAME_RATE_TOLERANCE)); - Preconditions.checkState("decoded frame count should be less than extracted frame count", mVideoDecodedFrameCount <= mVideoExtractedFrameCount); - } - - boolean isHdrInput() { return mIsHdrInput; } - boolean isToneMapApplied() { return mToneMapApplied; } - String getDecoderName() { return mDecoderName; } - String getEncoderName() { return mEncoderName; } - - private static String createFragmentShader( - final int srcWidth, - final int srcHeight, - final int dstWidth, - final int dstHeight) { - final float kernelSizeX = (float) srcWidth / (float) dstWidth; - final float kernelSizeY = (float) srcHeight / (float) dstHeight; - Log.i(TAG, "kernel " + kernelSizeX + "x" + kernelSizeY); - final String shader; - if (kernelSizeX <= 2 && kernelSizeY <= 2) { - shader = - "#extension GL_OES_EGL_image_external : require\n" + - "precision mediump float;\n" + // highp here doesn't seem to matter - "varying vec2 vTextureCoord;\n" + - "uniform samplerExternalOES sTexture;\n" + - "void main() {\n" + - " gl_FragColor = texture2D(sTexture, vTextureCoord);\n" + - "}\n"; - } else { - final int kernelRadiusX = (int) Math.ceil(kernelSizeX - .1f) / 2; - final int kernelRadiusY = (int) Math.ceil(kernelSizeY - .1f) / 2; - final float stepX = kernelSizeX / (1 + 2 * kernelRadiusX) * (1f / srcWidth); - final float stepY = kernelSizeY / (1 + 2 * kernelRadiusY) * (1f / srcHeight); - final float sum = (1 + 2 * kernelRadiusX) * (1 + 2 * kernelRadiusY); - final StringBuilder colorLoop = new StringBuilder(); - for (int i = -kernelRadiusX; i <=kernelRadiusX; i++) { - for (int j = -kernelRadiusY; j <=kernelRadiusY; j++) { - if (i != 0 || j != 0) { - colorLoop.append(" + texture2D(sTexture, vTextureCoord.xy + vec2(") - .append(i * stepX).append(", ").append(j * stepY).append("))\n"); - } - } - } - shader = - "#extension GL_OES_EGL_image_external : require\n" + - "precision mediump float;\n" + // highp here doesn't seem to matter - "varying vec2 vTextureCoord;\n" + - "uniform samplerExternalOES sTexture;\n" + - "void main() {\n" + - " gl_FragColor = (texture2D(sTexture, vTextureCoord)\n" + - colorLoop + - " ) / " + sum + ";\n" + - "}\n"; - } - Log.i(TAG, shader); - return shader; - } - - private @NonNull - MediaCodec createVideoDecoder( - final @NonNull MediaFormat inputFormat, - final @NonNull Surface surface, - final @NonNull Set excludedDecoders) throws IOException { - final boolean isHdr = MediaCodecCompat.isHdrVideo(inputFormat); - final boolean requestToneMapping = Build.VERSION.SDK_INT >= 31 && isHdr; - final List> allCandidates = MediaCodecCompat.findDecoderCandidates(inputFormat); - final List> candidates = new ArrayList<>(); - for (Pair c : allCandidates) { - if (!excludedDecoders.contains(c.getFirst())) { - candidates.add(c); - } - } - - mIsHdrInput = isHdr; - Exception lastException = null; - - for (int i = 0; i < candidates.size(); i++) { - final Pair candidate = candidates.get(i); - final String codecName = candidate.getFirst(); - final MediaFormat baseFormat = candidate.getSecond(); - MediaCodec decoder = null; - - try { - decoder = MediaCodec.createByCodecName(codecName); - - // For HDR video on API 31+, try requesting SDR tone-mapping. - // Some codecs reject this key, so we catch the error and retry without it. - if (requestToneMapping) { - try { - final MediaFormat toneMapFormat = new MediaFormat(baseFormat); - toneMapFormat.setInteger(MediaFormat.KEY_COLOR_TRANSFER_REQUEST, MediaFormat.COLOR_TRANSFER_SDR_VIDEO); - decoder.configure(toneMapFormat, surface, null, 0); - decoder.start(); - - mToneMapApplied = isToneMapEffective(decoder, codecName); - mDecoderName = codecName; - if (i > 0) { - Log.w(TAG, "Video decoder: succeeded with fallback codec " + codecName + " (attempt " + (i + 1) + " of " + candidates.size() + ")"); - } - return decoder; - } catch (IllegalArgumentException | IllegalStateException e) { - Log.w(TAG, "Video decoder: codec " + codecName + " rejected tone-mapping request, retrying without (attempt " + (i + 1) + " of " + candidates.size() + ")", e); - decoder.release(); - decoder = MediaCodec.createByCodecName(codecName); - } - } - - decoder.configure(baseFormat, surface, null, 0); - decoder.start(); - - mDecoderName = codecName; - if (i > 0 || requestToneMapping) { - Log.w(TAG, "Video decoder: succeeded with codec " + codecName + (requestToneMapping ? " (no tone-mapping)" : "") + " (attempt " + (i + 1) + " of " + candidates.size() + ")"); - } - return decoder; - } catch (IllegalArgumentException | IllegalStateException e) { - Log.w(TAG, "Video decoder: codec " + codecName + " failed (attempt " + (i + 1) + " of " + candidates.size() + ")", e); - lastException = e; - if (decoder != null) { - decoder.release(); - } - } catch (IOException e) { - Log.w(TAG, "Video decoder: codec " + codecName + " failed to create (attempt " + (i + 1) + " of " + candidates.size() + ")", e); - lastException = e; - } - } - - if (mIsHdrInput) { - throw new HdrDecoderUnavailableException("All video decoder codecs failed for HDR video", lastException); - } - throw new CodecUnavailableException("All video decoder codecs failed", lastException); - } - - /** - * Creates and configures a video encoder but does NOT start it. The caller must call - * {@link MediaCodec#createInputSurface()} (between configure and start) and then - * {@link MediaCodec#start()} after the decoder has been created. - */ - private @NonNull - MediaCodec createVideoEncoder( - final @NonNull List codecCandidates, - final @NonNull MediaFormat format) throws IOException { - Exception lastException = null; - - for (int i = 0; i < codecCandidates.size(); i++) { - final MediaCodecInfo codecInfo = codecCandidates.get(i); - MediaCodec encoder = null; - - try { - encoder = MediaCodec.createByCodecName(codecInfo.getName()); - encoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); - mEncoderName = codecInfo.getName(); - if (i > 0) { - Log.w(TAG, "Video encoder: succeeded with fallback codec " + codecInfo.getName() + " (attempt " + (i + 1) + " of " + codecCandidates.size() + ")"); - } - return encoder; - } catch (IllegalArgumentException | IllegalStateException e) { - Log.w(TAG, "Video encoder: codec " + codecInfo.getName() + " failed (attempt " + (i + 1) + " of " + codecCandidates.size() + ")", e); - lastException = e; - if (encoder != null) { - encoder.release(); - } - } - } - - throw new CodecUnavailableException("All video encoder codecs failed", lastException); - } - - /** - * Attempts to start the current encoder ({@link #mVideoEncoder}). If start() fails, - * iterates through the remaining encoder candidates from {@code codecCandidates}, - * replacing the encoder and its {@link InputSurface} on each attempt. The decoder - * and {@link OutputSurface} are independent of the encoder and remain unchanged. - */ - private void startEncoderWithFallback( - final @NonNull List codecCandidates, - final @NonNull MediaFormat format) throws IOException { - Exception lastException = null; - - for (int i = 0; i < codecCandidates.size(); i++) { - final MediaCodecInfo codecInfo = codecCandidates.get(i); - - if (i > 0) { - // Replace the encoder with the next candidate. - mVideoEncoder.release(); - mInputSurface.release(); - - try { - mVideoEncoder = MediaCodec.createByCodecName(codecInfo.getName()); - mVideoEncoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); - mInputSurface = new InputSurface(mVideoEncoder.createInputSurface()); - mInputSurface.makeCurrent(); - mEncoderName = codecInfo.getName(); - } catch (IllegalArgumentException | IllegalStateException | TranscodingException e) { - Log.w(TAG, "Video encoder: codec " + codecInfo.getName() + " failed to configure (attempt " + (i + 1) + " of " + codecCandidates.size() + ")", e); - lastException = e; - continue; - } - } else if (!codecInfo.getName().equals(mEncoderName)) { - // First iteration but createVideoEncoder selected a different codec - // (i.e. the first candidate failed to configure). Skip until we reach - // the one that was actually configured. - continue; - } - - try { - mVideoEncoder.start(); - if (i > 0) { - Log.w(TAG, "Video encoder: succeeded with fallback codec " + codecInfo.getName() + " (attempt " + (i + 1) + " of " + codecCandidates.size() + ")"); - } - return; - } catch (IllegalStateException e) { - Log.w(TAG, "Video encoder: codec " + codecInfo.getName() + " failed to start (attempt " + (i + 1) + " of " + codecCandidates.size() + ")", e); - lastException = e; - } - } - - throw new CodecUnavailableException("All video encoder codecs failed to start", lastException); - } - - private static int getAndSelectVideoTrackIndex(@NonNull MediaExtractor extractor) { - for (int index = 0; index < extractor.getTrackCount(); ++index) { - if (VERBOSE) { - Log.d(TAG, "format for track " + index + " is " + MediaConverter.getMimeTypeFor(extractor.getTrackFormat(index))); - } - if (isVideoFormat(extractor.getTrackFormat(index))) { - extractor.selectTrack(index); - return index; - } - } - return -1; - } - - private static boolean isVideoFormat(final @NonNull MediaFormat format) { - return MediaConverter.getMimeTypeFor(format).startsWith("video/"); - } - - /** - * Checks whether HDR-to-SDR tone-mapping is effective after the decoder has been configured - * and started with {@link MediaFormat#KEY_COLOR_TRANSFER_REQUEST}. Some codecs (especially - * software decoders and some hardware decoders) accept the tone-mapping key without error - * but don't actually perform the conversion. - */ - private static boolean isToneMapEffective(final @NonNull MediaCodec decoder, final @NonNull String codecName) { - // Software codecs never perform HDR→SDR tone-mapping. - String lower = codecName.toLowerCase(java.util.Locale.ROOT); - if (lower.startsWith("omx.google.") || lower.startsWith("c2.android.")) { - Log.w(TAG, "Video decoder: software codec " + codecName + " cannot perform HDR tone-mapping"); - return false; - } - - // For hardware codecs, verify the output format. If the output transfer function - // is still HDR (ST2084 or HLG), the decoder accepted the request but isn't honoring it. - try { - MediaFormat outputFormat = decoder.getOutputFormat(); - if (outputFormat.containsKey(MediaFormat.KEY_COLOR_TRANSFER)) { - int transfer = outputFormat.getInteger(MediaFormat.KEY_COLOR_TRANSFER); - if (transfer == MediaFormat.COLOR_TRANSFER_ST2084 || transfer == MediaFormat.COLOR_TRANSFER_HLG) { - Log.w(TAG, "Video decoder: codec " + codecName + " accepted tone-mapping but output transfer is " + transfer + " (still HDR)"); - return false; - } - } - } catch (Exception e) { - Log.w(TAG, "Video decoder: could not verify tone-mapping for codec " + codecName, e); - } - - return true; - } - -} - +package org.thoughtcrime.securesms.video.videoconverter; + +import android.media.MediaCodec; +import android.media.MediaCodecInfo; +import android.media.MediaExtractor; +import android.media.MediaFormat; +import android.os.Build; +import android.view.Surface; + +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; + +import org.signal.core.util.logging.Log; +import org.thoughtcrime.securesms.video.interfaces.MediaInput; +import org.thoughtcrime.securesms.video.interfaces.Muxer; +import org.thoughtcrime.securesms.video.videoconverter.exceptions.CodecUnavailableException; +import org.thoughtcrime.securesms.video.videoconverter.exceptions.HdrDecoderUnavailableException; +import org.thoughtcrime.securesms.video.videoconverter.utils.Extensions; +import org.thoughtcrime.securesms.video.videoconverter.utils.MediaCodecCompat; +import org.thoughtcrime.securesms.video.videoconverter.utils.Preconditions; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import kotlin.Pair; + +final class VideoTrackConverter { + + private static final String TAG = "media-converter"; + private static final boolean VERBOSE = false; // lots of logging + + private static final int OUTPUT_VIDEO_IFRAME_INTERVAL = 1; // 1 second between I-frames + private static final int OUTPUT_VIDEO_FRAME_RATE = 30; // needed only for MediaFormat.KEY_I_FRAME_INTERVAL to work; the actual frame rate matches the source + + private static final int TIMEOUT_USEC = 10000; + + private static final String MEDIA_FORMAT_KEY_DISPLAY_WIDTH = "display-width"; + private static final String MEDIA_FORMAT_KEY_DISPLAY_HEIGHT = "display-height"; + + private static final float FRAME_RATE_TOLERANCE = 0.05f; // tolerance for transcoding VFR -> CFR + + private boolean mIsHdrInput; + private boolean mToneMapApplied; + private String mDecoderName; + private String mEncoderName; + + private final long mTimeFrom; + private final long mTimeTo; + + final long mInputDuration; + + private final MediaExtractor mVideoExtractor; + private final MediaCodec mVideoDecoder; + private MediaCodec mVideoEncoder; + + private InputSurface mInputSurface; + private final OutputSurface mOutputSurface; + + private final ByteBuffer[] mVideoDecoderInputBuffers; + private ByteBuffer[] mVideoEncoderOutputBuffers; + private final MediaCodec.BufferInfo mVideoDecoderOutputBufferInfo; + private final MediaCodec.BufferInfo mVideoEncoderOutputBufferInfo; + + MediaFormat mEncoderOutputVideoFormat; + + boolean mVideoExtractorDone; + private boolean mVideoDecoderDone; + boolean mVideoEncoderDone; + + private int mOutputVideoTrack = -1; + + long mMuxingVideoPresentationTime; + + private int mVideoExtractedFrameCount; + private int mVideoDecodedFrameCount; + private int mVideoEncodedFrameCount; + + private Muxer mMuxer; + + static @Nullable VideoTrackConverter create( + final @NonNull MediaInput input, + final long timeFrom, + final long timeTo, + final int videoResolution, + final int videoBitrate, + final @NonNull String videoCodec, + final @NonNull Set excludedDecoders) throws IOException, TranscodingException { + + final MediaExtractor videoExtractor = input.createExtractor(); + final int videoInputTrack = getAndSelectVideoTrackIndex(videoExtractor); + if (videoInputTrack == -1) { + videoExtractor.release(); + return null; + } + return new VideoTrackConverter(videoExtractor, videoInputTrack, timeFrom, timeTo, videoResolution, videoBitrate, videoCodec, excludedDecoders); + } + + + private VideoTrackConverter( + final @NonNull MediaExtractor videoExtractor, + final int videoInputTrack, + final long timeFrom, + final long timeTo, + final int videoResolution, + final int videoBitrate, + final @NonNull String videoCodec, + final @NonNull Set excludedDecoders) throws IOException, TranscodingException { + + mTimeFrom = timeFrom; + mTimeTo = timeTo; + mVideoExtractor = videoExtractor; + + final List videoCodecCandidates = MediaConverter.selectCodecs(videoCodec); + if (videoCodecCandidates.isEmpty()) { + // Don't fail CTS if they don't have an AVC codec (not here, anyway). + Log.e(TAG, "Unable to find an appropriate codec for " + videoCodec); + throw new FileNotFoundException(); + } + if (VERBOSE) Log.d(TAG, "video found codecs: " + videoCodecCandidates.size()); + + final MediaFormat inputVideoFormat = mVideoExtractor.getTrackFormat(videoInputTrack); + + mInputDuration = inputVideoFormat.containsKey(MediaFormat.KEY_DURATION) ? inputVideoFormat.getLong(MediaFormat.KEY_DURATION) : 0; + + final int rotation = inputVideoFormat.containsKey(MediaFormat.KEY_ROTATION) ? inputVideoFormat.getInteger(MediaFormat.KEY_ROTATION) : 0; + final int width = inputVideoFormat.containsKey(MEDIA_FORMAT_KEY_DISPLAY_WIDTH) + ? inputVideoFormat.getInteger(MEDIA_FORMAT_KEY_DISPLAY_WIDTH) + : inputVideoFormat.getInteger(MediaFormat.KEY_WIDTH); + final int height = inputVideoFormat.containsKey(MEDIA_FORMAT_KEY_DISPLAY_HEIGHT) + ? inputVideoFormat.getInteger(MEDIA_FORMAT_KEY_DISPLAY_HEIGHT) + : inputVideoFormat.getInteger(MediaFormat.KEY_HEIGHT); + int outputWidth = width; + int outputHeight = height; + if (outputWidth < outputHeight) { + outputWidth = videoResolution; + outputHeight = height * outputWidth / width; + } else { + outputHeight = videoResolution; + outputWidth = width * outputHeight / height; + } + // many encoders do not work when height and width are not multiple of 16 (also, some iPhones do not play some heights) + outputHeight = (outputHeight + 7) & ~0xF; + outputWidth = (outputWidth + 7) & ~0xF; + + final int outputWidthRotated; + final int outputHeightRotated; + if ((rotation % 180 == 90)) { + //noinspection SuspiciousNameCombination + outputWidthRotated = outputHeight; + //noinspection SuspiciousNameCombination + outputHeightRotated = outputWidth; + } else { + outputWidthRotated = outputWidth; + outputHeightRotated = outputHeight; + } + + final MediaFormat outputVideoFormat = MediaFormat.createVideoFormat(videoCodec, outputWidthRotated, outputHeightRotated); + + // Set some properties. Failing to specify some of these can cause the MediaCodec + // configure() call to throw an unhelpful exception. + outputVideoFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface); + outputVideoFormat.setInteger(MediaFormat.KEY_BIT_RATE, videoBitrate); + outputVideoFormat.setInteger(MediaFormat.KEY_BITRATE_MODE, MediaCodecInfo.EncoderCapabilities.BITRATE_MODE_CBR); + outputVideoFormat.setInteger(MediaFormat.KEY_FRAME_RATE, OUTPUT_VIDEO_FRAME_RATE); + outputVideoFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, OUTPUT_VIDEO_IFRAME_INTERVAL); + if (VERBOSE) Log.d(TAG, "video format: " + outputVideoFormat); + + final String fragmentShader = createFragmentShader( + inputVideoFormat.getInteger(MediaFormat.KEY_WIDTH), inputVideoFormat.getInteger(MediaFormat.KEY_HEIGHT), + outputWidth, outputHeight); + + // Create encoder, decoder, and surfaces. The encoder's start() is deferred + // until after the decoder is created, so that the decoder gets first access to + // hardware codec resources on memory-constrained devices. If start() fails + // (e.g. NO_MEMORY on a resource-constrained device), we try the next encoder + // candidate while keeping the same decoder and OutputSurface. + mVideoEncoder = createVideoEncoder(videoCodecCandidates, outputVideoFormat); + mInputSurface = new InputSurface(mVideoEncoder.createInputSurface()); + mInputSurface.makeCurrent(); + mOutputSurface = new OutputSurface(); + mOutputSurface.changeFragmentShader(fragmentShader); + mVideoDecoder = createVideoDecoder(inputVideoFormat, mOutputSurface.getSurface(), excludedDecoders); + startEncoderWithFallback(videoCodecCandidates, outputVideoFormat); + + mVideoDecoderInputBuffers = mVideoDecoder.getInputBuffers(); + mVideoEncoderOutputBuffers = mVideoEncoder.getOutputBuffers(); + mVideoDecoderOutputBufferInfo = new MediaCodec.BufferInfo(); + mVideoEncoderOutputBufferInfo = new MediaCodec.BufferInfo(); + + if (mTimeFrom > 0) { + mVideoExtractor.seekTo(mTimeFrom * 1000, MediaExtractor.SEEK_TO_PREVIOUS_SYNC); + Log.i(TAG, "Seek video:" + mTimeFrom + " " + mVideoExtractor.getSampleTime()); + } + } + + void setMuxer(final @NonNull Muxer muxer) throws IOException { + mMuxer = muxer; + if (mEncoderOutputVideoFormat != null) { + Log.d(TAG, "muxer: adding video track."); + mOutputVideoTrack = muxer.addTrack(mEncoderOutputVideoFormat); + } + } + + void step() throws IOException, TranscodingException { + // Extract video from file and feed to decoder. + // Do not extract video if we have determined the output format but we are not yet + // ready to mux the frames. + while (!mVideoExtractorDone + && (mEncoderOutputVideoFormat == null || mMuxer != null)) { + int decoderInputBufferIndex = mVideoDecoder.dequeueInputBuffer(TIMEOUT_USEC); + if (decoderInputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) { + if (VERBOSE) Log.d(TAG, "no video decoder input buffer"); + break; + } + if (VERBOSE) { + Log.d(TAG, "video decoder: returned input buffer: " + decoderInputBufferIndex); + } + final ByteBuffer decoderInputBuffer = mVideoDecoderInputBuffers[decoderInputBufferIndex]; + final int size = mVideoExtractor.readSampleData(decoderInputBuffer, 0); + final long presentationTime = mVideoExtractor.getSampleTime(); + if (VERBOSE) { + Log.d(TAG, "video extractor: returned buffer of size " + size); + Log.d(TAG, "video extractor: returned buffer for time " + presentationTime); + } + mVideoExtractorDone = size < 0 || (mTimeTo > 0 && presentationTime > mTimeTo * 1000); + + if (mVideoExtractorDone) { + if (VERBOSE) Log.d(TAG, "video extractor: EOS"); + mVideoDecoder.queueInputBuffer( + decoderInputBufferIndex, + 0, + 0, + 0, + MediaCodec.BUFFER_FLAG_END_OF_STREAM); + } else { + mVideoDecoder.queueInputBuffer( + decoderInputBufferIndex, + 0, + size, + presentationTime, + mVideoExtractor.getSampleFlags()); + } + mVideoExtractor.advance(); + mVideoExtractedFrameCount++; + // We extracted a frame, let's try something else next. + break; + } + + // Poll output frames from the video decoder and feed the encoder. + while (!mVideoDecoderDone && (mEncoderOutputVideoFormat == null || mMuxer != null)) { + final int decoderOutputBufferIndex = + mVideoDecoder.dequeueOutputBuffer( + mVideoDecoderOutputBufferInfo, TIMEOUT_USEC); + if (decoderOutputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) { + if (VERBOSE) Log.d(TAG, "no video decoder output buffer"); + break; + } + if (decoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { + if (VERBOSE) Log.d(TAG, "video decoder: output buffers changed"); + break; + } + if (decoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { + if (VERBOSE) { + Log.d(TAG, "video decoder: output format changed: " + mVideoDecoder.getOutputFormat()); + } + break; + } + if (VERBOSE) { + Log.d(TAG, "video decoder: returned output buffer: " + + decoderOutputBufferIndex); + Log.d(TAG, "video decoder: returned buffer of size " + + mVideoDecoderOutputBufferInfo.size); + } + if ((mVideoDecoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) { + if (VERBOSE) Log.d(TAG, "video decoder: codec config buffer"); + mVideoDecoder.releaseOutputBuffer(decoderOutputBufferIndex, false); + break; + } + if (mVideoDecoderOutputBufferInfo.presentationTimeUs < mTimeFrom * 1000 && + (mVideoDecoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) == 0) { + if (VERBOSE) Log.d(TAG, "video decoder: frame prior to " + mVideoDecoderOutputBufferInfo.presentationTimeUs); + mVideoDecoder.releaseOutputBuffer(decoderOutputBufferIndex, false); + break; + } + if (VERBOSE) { + Log.d(TAG, "video decoder: returned buffer for time " + mVideoDecoderOutputBufferInfo.presentationTimeUs); + } + boolean render = mVideoDecoderOutputBufferInfo.size != 0; + mVideoDecoder.releaseOutputBuffer(decoderOutputBufferIndex, render); + if (render) { + if (VERBOSE) Log.d(TAG, "output surface: await new image"); + mOutputSurface.awaitNewImage(); + // Edit the frame and send it to the encoder. + if (VERBOSE) Log.d(TAG, "output surface: draw image"); + mOutputSurface.drawImage(); + mInputSurface.setPresentationTime(mVideoDecoderOutputBufferInfo.presentationTimeUs * 1000); + if (VERBOSE) Log.d(TAG, "input surface: swap buffers"); + mInputSurface.swapBuffers(); + if (VERBOSE) Log.d(TAG, "video encoder: notified of new frame"); + } + if ((mVideoDecoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { + if (VERBOSE) Log.d(TAG, "video decoder: EOS"); + mVideoDecoderDone = true; + mVideoEncoder.signalEndOfInputStream(); + } + mVideoDecodedFrameCount++; + // We extracted a pending frame, let's try something else next. + break; + } + + // Poll frames from the video encoder and send them to the muxer. + while (!mVideoEncoderDone && (mEncoderOutputVideoFormat == null || mMuxer != null)) { + final int encoderOutputBufferIndex = mVideoEncoder.dequeueOutputBuffer(mVideoEncoderOutputBufferInfo, TIMEOUT_USEC); + if (encoderOutputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) { + if (VERBOSE) Log.d(TAG, "no video encoder output buffer"); + if (mVideoDecoderDone) { + // on some devices and encoder stops after signalEndOfInputStream + Log.w(TAG, "mVideoDecoderDone, but didn't get BUFFER_FLAG_END_OF_STREAM"); + mVideoEncodedFrameCount = mVideoDecodedFrameCount; + mVideoEncoderDone = true; + } + break; + } + if (encoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { + if (VERBOSE) Log.d(TAG, "video encoder: output buffers changed"); + mVideoEncoderOutputBuffers = mVideoEncoder.getOutputBuffers(); + break; + } + if (encoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { + if (VERBOSE) Log.d(TAG, "video encoder: output format changed"); + Preconditions.checkState("video encoder changed its output format again?", mOutputVideoTrack < 0); + mEncoderOutputVideoFormat = mVideoEncoder.getOutputFormat(); + break; + } + Preconditions.checkState("should have added track before processing output", mMuxer != null); + if (VERBOSE) { + Log.d(TAG, "video encoder: returned output buffer: " + encoderOutputBufferIndex); + Log.d(TAG, "video encoder: returned buffer of size " + mVideoEncoderOutputBufferInfo.size); + } + final ByteBuffer encoderOutputBuffer = mVideoEncoderOutputBuffers[encoderOutputBufferIndex]; + if ((mVideoEncoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) { + if (VERBOSE) Log.d(TAG, "video encoder: codec config buffer"); + // Simply ignore codec config buffers. + mVideoEncoder.releaseOutputBuffer(encoderOutputBufferIndex, false); + break; + } + if (VERBOSE) { + Log.d(TAG, "video encoder: returned buffer for time " + mVideoEncoderOutputBufferInfo.presentationTimeUs); + } + if (mVideoEncoderOutputBufferInfo.size != 0) { + mMuxer.writeSampleData(mOutputVideoTrack, encoderOutputBuffer, mVideoEncoderOutputBufferInfo); + mMuxingVideoPresentationTime = Math.max(mMuxingVideoPresentationTime, mVideoEncoderOutputBufferInfo.presentationTimeUs); + } + if ((mVideoEncoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { + if (VERBOSE) Log.d(TAG, "video encoder: EOS"); + mVideoEncoderDone = true; + } + mVideoEncoder.releaseOutputBuffer(encoderOutputBufferIndex, false); + mVideoEncodedFrameCount++; + // We enqueued an encoded frame, let's try something else next. + break; + } + } + + void release() throws Exception { + Exception exception = null; + try { + if (mVideoExtractor != null) { + mVideoExtractor.release(); + } + } catch (Exception e) { + Log.e(TAG, "error while releasing mVideoExtractor", e); + exception = e; + } + try { + if (mVideoDecoder != null) { + mVideoDecoder.stop(); + mVideoDecoder.release(); + } + } catch (Exception e) { + Log.e(TAG, "error while releasing mVideoDecoder", e); + if (exception == null) { + exception = e; + } + } + try { + if (mOutputSurface != null) { + mOutputSurface.release(); + } + } catch (Exception e) { + Log.e(TAG, "error while releasing mOutputSurface", e); + if (exception == null) { + exception = e; + } + } + try { + if (mInputSurface != null) { + mInputSurface.release(); + } + } catch (Exception e) { + Log.e(TAG, "error while releasing mInputSurface", e); + if (exception == null) { + exception = e; + } + } + try { + if (mVideoEncoder != null) { + mVideoEncoder.stop(); + mVideoEncoder.release(); + } + } catch (Exception e) { + Log.e(TAG, "error while releasing mVideoEncoder", e); + if (exception == null) { + exception = e; + } + } + if (exception != null) { + throw exception; + } + } + + VideoTrackConverterState dumpState() { + return new VideoTrackConverterState( + mVideoExtractedFrameCount, mVideoExtractorDone, + mVideoDecodedFrameCount, mVideoDecoderDone, + mVideoEncodedFrameCount, mVideoEncoderDone, + mMuxer != null, mOutputVideoTrack); + } + + void verifyEndState() { + Preconditions.checkState("encoded (" + mVideoEncodedFrameCount + ") and decoded (" + mVideoDecodedFrameCount + ") video frame counts should match", Extensions.isWithin(mVideoDecodedFrameCount, mVideoEncodedFrameCount, FRAME_RATE_TOLERANCE)); + Preconditions.checkState("decoded frame count should be less than extracted frame count", mVideoDecodedFrameCount <= mVideoExtractedFrameCount); + } + + boolean isHdrInput() { return mIsHdrInput; } + boolean isToneMapApplied() { return mToneMapApplied; } + String getDecoderName() { return mDecoderName; } + String getEncoderName() { return mEncoderName; } + + private static String createFragmentShader( + final int srcWidth, + final int srcHeight, + final int dstWidth, + final int dstHeight) { + final float kernelSizeX = (float) srcWidth / (float) dstWidth; + final float kernelSizeY = (float) srcHeight / (float) dstHeight; + Log.i(TAG, "kernel " + kernelSizeX + "x" + kernelSizeY); + final String shader; + if (kernelSizeX <= 2 && kernelSizeY <= 2) { + shader = + "#extension GL_OES_EGL_image_external : require\n" + + "precision mediump float;\n" + // highp here doesn't seem to matter + "varying vec2 vTextureCoord;\n" + + "uniform samplerExternalOES sTexture;\n" + + "void main() {\n" + + " gl_FragColor = texture2D(sTexture, vTextureCoord);\n" + + "}\n"; + } else { + final int kernelRadiusX = (int) Math.ceil(kernelSizeX - .1f) / 2; + final int kernelRadiusY = (int) Math.ceil(kernelSizeY - .1f) / 2; + final float stepX = kernelSizeX / (1 + 2 * kernelRadiusX) * (1f / srcWidth); + final float stepY = kernelSizeY / (1 + 2 * kernelRadiusY) * (1f / srcHeight); + final float sum = (1 + 2 * kernelRadiusX) * (1 + 2 * kernelRadiusY); + final StringBuilder colorLoop = new StringBuilder(); + for (int i = -kernelRadiusX; i <=kernelRadiusX; i++) { + for (int j = -kernelRadiusY; j <=kernelRadiusY; j++) { + if (i != 0 || j != 0) { + colorLoop.append(" + texture2D(sTexture, vTextureCoord.xy + vec2(") + .append(i * stepX).append(", ").append(j * stepY).append("))\n"); + } + } + } + shader = + "#extension GL_OES_EGL_image_external : require\n" + + "precision mediump float;\n" + // highp here doesn't seem to matter + "varying vec2 vTextureCoord;\n" + + "uniform samplerExternalOES sTexture;\n" + + "void main() {\n" + + " gl_FragColor = (texture2D(sTexture, vTextureCoord)\n" + + colorLoop + + " ) / " + sum + ";\n" + + "}\n"; + } + Log.i(TAG, shader); + return shader; + } + + private @NonNull + MediaCodec createVideoDecoder( + final @NonNull MediaFormat inputFormat, + final @NonNull Surface surface, + final @NonNull Set excludedDecoders) throws IOException { + final boolean isHdr = MediaCodecCompat.isHdrVideo(inputFormat); + final boolean requestToneMapping = Build.VERSION.SDK_INT >= 31 && isHdr; + final List> allCandidates = MediaCodecCompat.findDecoderCandidates(inputFormat); + final List> candidates = new ArrayList<>(); + for (Pair c : allCandidates) { + if (!excludedDecoders.contains(c.getFirst())) { + candidates.add(c); + } + } + + mIsHdrInput = isHdr; + Exception lastException = null; + + for (int i = 0; i < candidates.size(); i++) { + final Pair candidate = candidates.get(i); + final String codecName = candidate.getFirst(); + final MediaFormat baseFormat = candidate.getSecond(); + MediaCodec decoder = null; + + try { + decoder = MediaCodec.createByCodecName(codecName); + + // For HDR video on API 31+, try requesting SDR tone-mapping. + // Some codecs reject this key, so we catch the error and retry without it. + if (requestToneMapping) { + try { + final MediaFormat toneMapFormat = new MediaFormat(baseFormat); + toneMapFormat.setInteger(MediaFormat.KEY_COLOR_TRANSFER_REQUEST, MediaFormat.COLOR_TRANSFER_SDR_VIDEO); + decoder.configure(toneMapFormat, surface, null, 0); + decoder.start(); + + mToneMapApplied = isToneMapEffective(decoder, codecName); + mDecoderName = codecName; + if (i > 0) { + Log.w(TAG, "Video decoder: succeeded with fallback codec " + codecName + " (attempt " + (i + 1) + " of " + candidates.size() + ")"); + } + return decoder; + } catch (IllegalArgumentException | IllegalStateException e) { + Log.w(TAG, "Video decoder: codec " + codecName + " rejected tone-mapping request, retrying without (attempt " + (i + 1) + " of " + candidates.size() + ")", e); + decoder.release(); + decoder = MediaCodec.createByCodecName(codecName); + } + } + + decoder.configure(baseFormat, surface, null, 0); + decoder.start(); + + mDecoderName = codecName; + if (i > 0 || requestToneMapping) { + Log.w(TAG, "Video decoder: succeeded with codec " + codecName + (requestToneMapping ? " (no tone-mapping)" : "") + " (attempt " + (i + 1) + " of " + candidates.size() + ")"); + } + return decoder; + } catch (IllegalArgumentException | IllegalStateException e) { + Log.w(TAG, "Video decoder: codec " + codecName + " failed (attempt " + (i + 1) + " of " + candidates.size() + ")", e); + lastException = e; + if (decoder != null) { + decoder.release(); + } + } catch (IOException e) { + Log.w(TAG, "Video decoder: codec " + codecName + " failed to create (attempt " + (i + 1) + " of " + candidates.size() + ")", e); + lastException = e; + } + } + + if (mIsHdrInput) { + throw new HdrDecoderUnavailableException("All video decoder codecs failed for HDR video", lastException); + } + throw new CodecUnavailableException("All video decoder codecs failed", lastException); + } + + /** + * Creates and configures a video encoder but does NOT start it. The caller must call + * {@link MediaCodec#createInputSurface()} (between configure and start) and then + * {@link MediaCodec#start()} after the decoder has been created. + */ + private @NonNull + MediaCodec createVideoEncoder( + final @NonNull List codecCandidates, + final @NonNull MediaFormat format) throws IOException { + Exception lastException = null; + + for (int i = 0; i < codecCandidates.size(); i++) { + final MediaCodecInfo codecInfo = codecCandidates.get(i); + MediaCodec encoder = null; + + try { + encoder = MediaCodec.createByCodecName(codecInfo.getName()); + encoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); + mEncoderName = codecInfo.getName(); + if (i > 0) { + Log.w(TAG, "Video encoder: succeeded with fallback codec " + codecInfo.getName() + " (attempt " + (i + 1) + " of " + codecCandidates.size() + ")"); + } + return encoder; + } catch (IllegalArgumentException | IllegalStateException e) { + Log.w(TAG, "Video encoder: codec " + codecInfo.getName() + " failed (attempt " + (i + 1) + " of " + codecCandidates.size() + ")", e); + lastException = e; + if (encoder != null) { + encoder.release(); + } + } + } + + throw new CodecUnavailableException("All video encoder codecs failed", lastException); + } + + /** + * Attempts to start the current encoder ({@link #mVideoEncoder}). If start() fails, + * iterates through the remaining encoder candidates from {@code codecCandidates}, + * replacing the encoder and its {@link InputSurface} on each attempt. The decoder + * and {@link OutputSurface} are independent of the encoder and remain unchanged. + */ + private void startEncoderWithFallback( + final @NonNull List codecCandidates, + final @NonNull MediaFormat format) throws IOException { + Exception lastException = null; + + for (int i = 0; i < codecCandidates.size(); i++) { + final MediaCodecInfo codecInfo = codecCandidates.get(i); + + if (i > 0) { + // Replace the encoder with the next candidate. + mVideoEncoder.release(); + mInputSurface.release(); + + try { + mVideoEncoder = MediaCodec.createByCodecName(codecInfo.getName()); + mVideoEncoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); + mInputSurface = new InputSurface(mVideoEncoder.createInputSurface()); + mInputSurface.makeCurrent(); + mEncoderName = codecInfo.getName(); + } catch (IllegalArgumentException | IllegalStateException | TranscodingException e) { + Log.w(TAG, "Video encoder: codec " + codecInfo.getName() + " failed to configure (attempt " + (i + 1) + " of " + codecCandidates.size() + ")", e); + lastException = e; + continue; + } + } else if (!codecInfo.getName().equals(mEncoderName)) { + // First iteration but createVideoEncoder selected a different codec + // (i.e. the first candidate failed to configure). Skip until we reach + // the one that was actually configured. + continue; + } + + try { + mVideoEncoder.start(); + if (i > 0) { + Log.w(TAG, "Video encoder: succeeded with fallback codec " + codecInfo.getName() + " (attempt " + (i + 1) + " of " + codecCandidates.size() + ")"); + } + return; + } catch (IllegalStateException e) { + Log.w(TAG, "Video encoder: codec " + codecInfo.getName() + " failed to start (attempt " + (i + 1) + " of " + codecCandidates.size() + ")", e); + lastException = e; + } + } + + throw new CodecUnavailableException("All video encoder codecs failed to start", lastException); + } + + private static int getAndSelectVideoTrackIndex(@NonNull MediaExtractor extractor) { + for (int index = 0; index < extractor.getTrackCount(); ++index) { + if (VERBOSE) { + Log.d(TAG, "format for track " + index + " is " + MediaConverter.getMimeTypeFor(extractor.getTrackFormat(index))); + } + if (isVideoFormat(extractor.getTrackFormat(index))) { + extractor.selectTrack(index); + return index; + } + } + return -1; + } + + private static boolean isVideoFormat(final @NonNull MediaFormat format) { + return MediaConverter.getMimeTypeFor(format).startsWith("video/"); + } + + /** + * Checks whether HDR-to-SDR tone-mapping is effective after the decoder has been configured + * and started with {@link MediaFormat#KEY_COLOR_TRANSFER_REQUEST}. Some codecs (especially + * software decoders and some hardware decoders) accept the tone-mapping key without error + * but don't actually perform the conversion. + */ + private static boolean isToneMapEffective(final @NonNull MediaCodec decoder, final @NonNull String codecName) { + // Software codecs never perform HDR→SDR tone-mapping. + String lower = codecName.toLowerCase(java.util.Locale.ROOT); + if (lower.startsWith("omx.google.") || lower.startsWith("c2.android.")) { + Log.w(TAG, "Video decoder: software codec " + codecName + " cannot perform HDR tone-mapping"); + return false; + } + + // For hardware codecs, verify the output format. If the output transfer function + // is still HDR (ST2084 or HLG), the decoder accepted the request but isn't honoring it. + try { + MediaFormat outputFormat = decoder.getOutputFormat(); + if (outputFormat.containsKey(MediaFormat.KEY_COLOR_TRANSFER)) { + int transfer = outputFormat.getInteger(MediaFormat.KEY_COLOR_TRANSFER); + if (transfer == MediaFormat.COLOR_TRANSFER_ST2084 || transfer == MediaFormat.COLOR_TRANSFER_HLG) { + Log.w(TAG, "Video decoder: codec " + codecName + " accepted tone-mapping but output transfer is " + transfer + " (still HDR)"); + return false; + } + } + } catch (Exception e) { + Log.w(TAG, "Video decoder: could not verify tone-mapping for codec " + codecName, e); + } + + return true; + } + +} + diff --git a/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/muxer/AacTrack.java b/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/muxer/AacTrack.java index a37a454bf9..b4182426ff 100644 --- a/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/muxer/AacTrack.java +++ b/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/muxer/AacTrack.java @@ -1,123 +1,123 @@ -package org.thoughtcrime.securesms.video.videoconverter.muxer; - -import android.util.SparseIntArray; - -import org.mp4parser.boxes.iso14496.part1.objectdescriptors.AudioSpecificConfig; -import org.mp4parser.boxes.iso14496.part1.objectdescriptors.DecoderConfigDescriptor; -import org.mp4parser.boxes.iso14496.part1.objectdescriptors.DecoderSpecificInfo; -import org.mp4parser.boxes.iso14496.part1.objectdescriptors.ESDescriptor; -import org.mp4parser.boxes.iso14496.part1.objectdescriptors.SLConfigDescriptor; -import org.mp4parser.boxes.iso14496.part12.SampleDescriptionBox; -import org.mp4parser.boxes.iso14496.part14.ESDescriptorBox; -import org.mp4parser.boxes.sampleentry.AudioSampleEntry; -import org.mp4parser.streaming.extensions.DefaultSampleFlagsTrackExtension; -import org.mp4parser.streaming.input.AbstractStreamingTrack; -import org.mp4parser.streaming.input.StreamingSampleImpl; - -import java.io.IOException; -import java.nio.ByteBuffer; - -import androidx.annotation.Nullable; - -abstract class AacTrack extends AbstractStreamingTrack { - - private static final SparseIntArray SAMPLING_FREQUENCY_INDEX_MAP = new SparseIntArray(); - - static { - SAMPLING_FREQUENCY_INDEX_MAP.put(96000, 0); - SAMPLING_FREQUENCY_INDEX_MAP.put(88200, 1); - SAMPLING_FREQUENCY_INDEX_MAP.put(64000, 2); - SAMPLING_FREQUENCY_INDEX_MAP.put(48000, 3); - SAMPLING_FREQUENCY_INDEX_MAP.put(44100, 4); - SAMPLING_FREQUENCY_INDEX_MAP.put(32000, 5); - SAMPLING_FREQUENCY_INDEX_MAP.put(24000, 6); - SAMPLING_FREQUENCY_INDEX_MAP.put(22050, 7); - SAMPLING_FREQUENCY_INDEX_MAP.put(16000, 8); - SAMPLING_FREQUENCY_INDEX_MAP.put(12000, 9); - SAMPLING_FREQUENCY_INDEX_MAP.put(11025, 10); - SAMPLING_FREQUENCY_INDEX_MAP.put(8000, 11); - } - - private final SampleDescriptionBox stsd; - - private int sampleRate; - - AacTrack(long avgBitrate, long maxBitrate, int sampleRate, int channelCount, int aacProfile, @Nullable DecoderSpecificInfo decoderSpecificInfo) { - this.sampleRate = sampleRate; - - final DefaultSampleFlagsTrackExtension defaultSampleFlagsTrackExtension = new DefaultSampleFlagsTrackExtension(); - defaultSampleFlagsTrackExtension.setIsLeading(2); - defaultSampleFlagsTrackExtension.setSampleDependsOn(2); - defaultSampleFlagsTrackExtension.setSampleIsDependedOn(2); - defaultSampleFlagsTrackExtension.setSampleHasRedundancy(2); - defaultSampleFlagsTrackExtension.setSampleIsNonSyncSample(false); - this.addTrackExtension(defaultSampleFlagsTrackExtension); - - stsd = new SampleDescriptionBox(); - final AudioSampleEntry audioSampleEntry = new AudioSampleEntry("mp4a"); - if (channelCount == 7) { - audioSampleEntry.setChannelCount(8); - } else { - audioSampleEntry.setChannelCount(channelCount); - } - audioSampleEntry.setSampleRate(sampleRate); - audioSampleEntry.setDataReferenceIndex(1); - audioSampleEntry.setSampleSize(16); - - - final ESDescriptorBox esds = new ESDescriptorBox(); - ESDescriptor descriptor = new ESDescriptor(); - descriptor.setEsId(0); - - final SLConfigDescriptor slConfigDescriptor = new SLConfigDescriptor(); - slConfigDescriptor.setPredefined(2); - descriptor.setSlConfigDescriptor(slConfigDescriptor); - - final DecoderConfigDescriptor decoderConfigDescriptor = new DecoderConfigDescriptor(); - decoderConfigDescriptor.setObjectTypeIndication(0x40 /*Audio ISO/IEC 14496-3*/); - decoderConfigDescriptor.setStreamType(5 /*audio stream*/); - decoderConfigDescriptor.setBufferSizeDB(1536); - decoderConfigDescriptor.setMaxBitRate(maxBitrate); - decoderConfigDescriptor.setAvgBitRate(avgBitrate); - - final AudioSpecificConfig audioSpecificConfig = new AudioSpecificConfig(); - audioSpecificConfig.setOriginalAudioObjectType(aacProfile); - audioSpecificConfig.setSamplingFrequencyIndex(SAMPLING_FREQUENCY_INDEX_MAP.get(sampleRate)); - audioSpecificConfig.setChannelConfiguration(channelCount); - decoderConfigDescriptor.setAudioSpecificInfo(audioSpecificConfig); - - if (decoderSpecificInfo != null) { - decoderConfigDescriptor.setDecoderSpecificInfo(decoderSpecificInfo); - } - - descriptor.setDecoderConfigDescriptor(decoderConfigDescriptor); - - esds.setEsDescriptor(descriptor); - - audioSampleEntry.addBox(esds); - stsd.addBox(audioSampleEntry); - } - - public long getTimescale() { - return sampleRate; - } - - public String getHandler() { - return "soun"; - } - - public String getLanguage() { - return "\u0060\u0060\u0060"; // 0 in Iso639 - } - - public synchronized SampleDescriptionBox getSampleDescriptionBox() { - return stsd; - } - - public void close() { - } - - void processSample(ByteBuffer frame) throws IOException { - sampleSink.acceptSample(new StreamingSampleImpl(frame, 1024), this); - } -} +package org.thoughtcrime.securesms.video.videoconverter.muxer; + +import android.util.SparseIntArray; + +import org.mp4parser.boxes.iso14496.part1.objectdescriptors.AudioSpecificConfig; +import org.mp4parser.boxes.iso14496.part1.objectdescriptors.DecoderConfigDescriptor; +import org.mp4parser.boxes.iso14496.part1.objectdescriptors.DecoderSpecificInfo; +import org.mp4parser.boxes.iso14496.part1.objectdescriptors.ESDescriptor; +import org.mp4parser.boxes.iso14496.part1.objectdescriptors.SLConfigDescriptor; +import org.mp4parser.boxes.iso14496.part12.SampleDescriptionBox; +import org.mp4parser.boxes.iso14496.part14.ESDescriptorBox; +import org.mp4parser.boxes.sampleentry.AudioSampleEntry; +import org.mp4parser.streaming.extensions.DefaultSampleFlagsTrackExtension; +import org.mp4parser.streaming.input.AbstractStreamingTrack; +import org.mp4parser.streaming.input.StreamingSampleImpl; + +import java.io.IOException; +import java.nio.ByteBuffer; + +import androidx.annotation.Nullable; + +abstract class AacTrack extends AbstractStreamingTrack { + + private static final SparseIntArray SAMPLING_FREQUENCY_INDEX_MAP = new SparseIntArray(); + + static { + SAMPLING_FREQUENCY_INDEX_MAP.put(96000, 0); + SAMPLING_FREQUENCY_INDEX_MAP.put(88200, 1); + SAMPLING_FREQUENCY_INDEX_MAP.put(64000, 2); + SAMPLING_FREQUENCY_INDEX_MAP.put(48000, 3); + SAMPLING_FREQUENCY_INDEX_MAP.put(44100, 4); + SAMPLING_FREQUENCY_INDEX_MAP.put(32000, 5); + SAMPLING_FREQUENCY_INDEX_MAP.put(24000, 6); + SAMPLING_FREQUENCY_INDEX_MAP.put(22050, 7); + SAMPLING_FREQUENCY_INDEX_MAP.put(16000, 8); + SAMPLING_FREQUENCY_INDEX_MAP.put(12000, 9); + SAMPLING_FREQUENCY_INDEX_MAP.put(11025, 10); + SAMPLING_FREQUENCY_INDEX_MAP.put(8000, 11); + } + + private final SampleDescriptionBox stsd; + + private int sampleRate; + + AacTrack(long avgBitrate, long maxBitrate, int sampleRate, int channelCount, int aacProfile, @Nullable DecoderSpecificInfo decoderSpecificInfo) { + this.sampleRate = sampleRate; + + final DefaultSampleFlagsTrackExtension defaultSampleFlagsTrackExtension = new DefaultSampleFlagsTrackExtension(); + defaultSampleFlagsTrackExtension.setIsLeading(2); + defaultSampleFlagsTrackExtension.setSampleDependsOn(2); + defaultSampleFlagsTrackExtension.setSampleIsDependedOn(2); + defaultSampleFlagsTrackExtension.setSampleHasRedundancy(2); + defaultSampleFlagsTrackExtension.setSampleIsNonSyncSample(false); + this.addTrackExtension(defaultSampleFlagsTrackExtension); + + stsd = new SampleDescriptionBox(); + final AudioSampleEntry audioSampleEntry = new AudioSampleEntry("mp4a"); + if (channelCount == 7) { + audioSampleEntry.setChannelCount(8); + } else { + audioSampleEntry.setChannelCount(channelCount); + } + audioSampleEntry.setSampleRate(sampleRate); + audioSampleEntry.setDataReferenceIndex(1); + audioSampleEntry.setSampleSize(16); + + + final ESDescriptorBox esds = new ESDescriptorBox(); + ESDescriptor descriptor = new ESDescriptor(); + descriptor.setEsId(0); + + final SLConfigDescriptor slConfigDescriptor = new SLConfigDescriptor(); + slConfigDescriptor.setPredefined(2); + descriptor.setSlConfigDescriptor(slConfigDescriptor); + + final DecoderConfigDescriptor decoderConfigDescriptor = new DecoderConfigDescriptor(); + decoderConfigDescriptor.setObjectTypeIndication(0x40 /*Audio ISO/IEC 14496-3*/); + decoderConfigDescriptor.setStreamType(5 /*audio stream*/); + decoderConfigDescriptor.setBufferSizeDB(1536); + decoderConfigDescriptor.setMaxBitRate(maxBitrate); + decoderConfigDescriptor.setAvgBitRate(avgBitrate); + + final AudioSpecificConfig audioSpecificConfig = new AudioSpecificConfig(); + audioSpecificConfig.setOriginalAudioObjectType(aacProfile); + audioSpecificConfig.setSamplingFrequencyIndex(SAMPLING_FREQUENCY_INDEX_MAP.get(sampleRate)); + audioSpecificConfig.setChannelConfiguration(channelCount); + decoderConfigDescriptor.setAudioSpecificInfo(audioSpecificConfig); + + if (decoderSpecificInfo != null) { + decoderConfigDescriptor.setDecoderSpecificInfo(decoderSpecificInfo); + } + + descriptor.setDecoderConfigDescriptor(decoderConfigDescriptor); + + esds.setEsDescriptor(descriptor); + + audioSampleEntry.addBox(esds); + stsd.addBox(audioSampleEntry); + } + + public long getTimescale() { + return sampleRate; + } + + public String getHandler() { + return "soun"; + } + + public String getLanguage() { + return "\u0060\u0060\u0060"; // 0 in Iso639 + } + + public synchronized SampleDescriptionBox getSampleDescriptionBox() { + return stsd; + } + + public void close() { + } + + void processSample(ByteBuffer frame) throws IOException { + sampleSink.acceptSample(new StreamingSampleImpl(frame, 1024), this); + } +} diff --git a/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/muxer/AvcTrack.java b/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/muxer/AvcTrack.java index 737db09c0d..9482816136 100644 --- a/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/muxer/AvcTrack.java +++ b/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/muxer/AvcTrack.java @@ -1,478 +1,478 @@ -package org.thoughtcrime.securesms.video.videoconverter.muxer; - -import androidx.annotation.NonNull; -import androidx.annotation.Nullable; - -import org.mp4parser.boxes.iso14496.part12.SampleDescriptionBox; -import org.mp4parser.boxes.iso14496.part15.AvcConfigurationBox; -import org.mp4parser.boxes.sampleentry.VisualSampleEntry; -import org.mp4parser.streaming.SampleExtension; -import org.mp4parser.streaming.StreamingSample; -import org.mp4parser.streaming.extensions.CompositionTimeSampleExtension; -import org.mp4parser.streaming.extensions.CompositionTimeTrackExtension; -import org.mp4parser.streaming.extensions.DimensionTrackExtension; -import org.mp4parser.streaming.extensions.SampleFlagsSampleExtension; -import org.mp4parser.streaming.input.AbstractStreamingTrack; -import org.mp4parser.streaming.input.StreamingSampleImpl; -import org.mp4parser.streaming.input.h264.H264NalUnitHeader; -import org.mp4parser.streaming.input.h264.H264NalUnitTypes; -import org.mp4parser.streaming.input.h264.spspps.PictureParameterSet; -import org.mp4parser.streaming.input.h264.spspps.SeqParameterSet; -import org.mp4parser.streaming.input.h264.spspps.SliceHeader; -import org.signal.core.util.logging.Log; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.List; - -abstract class AvcTrack extends AbstractStreamingTrack { - - private static final String TAG = "AvcTrack"; - - private int maxDecFrameBuffering = 16; - private final List decFrameBuffer = new ArrayList<>(); - private final List decFrameBuffer2 = new ArrayList<>(); - - private final LinkedHashMap spsIdToSpsBytes = new LinkedHashMap<>(); - private final LinkedHashMap spsIdToSps = new LinkedHashMap<>(); - private final LinkedHashMap ppsIdToPpsBytes = new LinkedHashMap<>(); - private final LinkedHashMap ppsIdToPps = new LinkedHashMap<>(); - - private int timescale = 90000; - private int frametick = 3000; - - private final SampleDescriptionBox stsd; - - private final List bufferedNals = new ArrayList<>(); - private FirstVclNalDetector fvnd; - private H264NalUnitHeader sliceNalUnitHeader; - private long currentPresentationTimeUs; - - AvcTrack(final @NonNull ByteBuffer spsBuffer, final @NonNull ByteBuffer ppsBuffer) { - - handlePPS(ppsBuffer); - - final SeqParameterSet sps = handleSPS(spsBuffer); - - int width = (sps.pic_width_in_mbs_minus1 + 1) * 16; - int mult = 2; - if (sps.frame_mbs_only_flag) { - mult = 1; - } - int height = 16 * (sps.pic_height_in_map_units_minus1 + 1) * mult; - if (sps.frame_cropping_flag) { - int chromaArrayType = 0; - if (!sps.residual_color_transform_flag) { - chromaArrayType = sps.chroma_format_idc.getId(); - } - int cropUnitX = 1; - int cropUnitY = mult; - if (chromaArrayType != 0) { - cropUnitX = sps.chroma_format_idc.getSubWidth(); - cropUnitY = sps.chroma_format_idc.getSubHeight() * mult; - } - - width -= cropUnitX * (sps.frame_crop_left_offset + sps.frame_crop_right_offset); - height -= cropUnitY * (sps.frame_crop_top_offset + sps.frame_crop_bottom_offset); - } - - - final VisualSampleEntry visualSampleEntry = new VisualSampleEntry("avc1"); - visualSampleEntry.setDataReferenceIndex(1); - visualSampleEntry.setDepth(24); - visualSampleEntry.setFrameCount(1); - visualSampleEntry.setHorizresolution(72); - visualSampleEntry.setVertresolution(72); - final DimensionTrackExtension dte = this.getTrackExtension(DimensionTrackExtension.class); - if (dte == null) { - this.addTrackExtension(new DimensionTrackExtension(width, height)); - } - visualSampleEntry.setWidth(width); - visualSampleEntry.setHeight(height); - - visualSampleEntry.setCompressorname("AVC Coding"); - - final AvcConfigurationBox avcConfigurationBox = new AvcConfigurationBox(); - - avcConfigurationBox.setSequenceParameterSets(Collections.singletonList(spsBuffer)); - avcConfigurationBox.setPictureParameterSets(Collections.singletonList(ppsBuffer)); - avcConfigurationBox.setAvcLevelIndication(sps.level_idc); - avcConfigurationBox.setAvcProfileIndication(sps.profile_idc); - avcConfigurationBox.setBitDepthLumaMinus8(sps.bit_depth_luma_minus8); - avcConfigurationBox.setBitDepthChromaMinus8(sps.bit_depth_chroma_minus8); - avcConfigurationBox.setChromaFormat(sps.chroma_format_idc.getId()); - avcConfigurationBox.setConfigurationVersion(1); - avcConfigurationBox.setLengthSizeMinusOne(3); - - - avcConfigurationBox.setProfileCompatibility( - (sps.constraint_set_0_flag ? 128 : 0) + - (sps.constraint_set_1_flag ? 64 : 0) + - (sps.constraint_set_2_flag ? 32 : 0) + - (sps.constraint_set_3_flag ? 16 : 0) + - (sps.constraint_set_4_flag ? 8 : 0) + - (int) (sps.reserved_zero_2bits & 0x3) - ); - - visualSampleEntry.addBox(avcConfigurationBox); - stsd = new SampleDescriptionBox(); - stsd.addBox(visualSampleEntry); - - int _timescale; - int _frametick; - if (sps.vuiParams != null) { - _timescale = sps.vuiParams.time_scale >> 1; // Not sure why, but I found this in several places, and it works... - _frametick = sps.vuiParams.num_units_in_tick; - if (_timescale == 0 || _frametick == 0) { - Log.w(TAG, "vuiParams contain invalid values: time_scale: " + _timescale + " and frame_tick: " + _frametick + ". Setting frame rate to 30fps"); - _timescale = 0; - _frametick = 0; - } - if (_frametick > 0) { - if (_timescale / _frametick > 100) { - Log.w(TAG, "Framerate is " + (_timescale / _frametick) + ". That is suspicious."); - } - } else { - Log.w(TAG, "Frametick is " + _frametick + ". That is suspicious."); - } - if (sps.vuiParams.bitstreamRestriction != null) { - maxDecFrameBuffering = sps.vuiParams.bitstreamRestriction.max_dec_frame_buffering; - } - } else { - Log.w(TAG, "Can't determine frame rate as SPS does not contain vuiParama"); - _timescale = 0; - _frametick = 0; - } - if (_timescale != 0 && _frametick != 0) { - timescale = _timescale; - frametick = _frametick; - } - if (sps.pic_order_cnt_type == 0) { - addTrackExtension(new CompositionTimeTrackExtension()); - } else if (sps.pic_order_cnt_type == 1) { - throw new MuxingException("Have not yet imlemented pic_order_cnt_type 1"); - } - } - - public long getTimescale() { - return timescale; - } - - public String getHandler() { - return "vide"; - } - - public String getLanguage() { - return "\u0060\u0060\u0060"; // 0 in Iso639 - } - - public SampleDescriptionBox getSampleDescriptionBox() { - return stsd; - } - - public void close() { - } - - private static H264NalUnitHeader getNalUnitHeader(@NonNull final ByteBuffer nal) { - final H264NalUnitHeader nalUnitHeader = new H264NalUnitHeader(); - final int type = nal.get(0); - nalUnitHeader.nal_ref_idc = (type >> 5) & 3; - nalUnitHeader.nal_unit_type = type & 0x1f; - return nalUnitHeader; - } - - void consumeNal(@NonNull final ByteBuffer nal, final long presentationTimeUs) throws IOException { - - final H264NalUnitHeader nalUnitHeader = getNalUnitHeader(nal); - switch (nalUnitHeader.nal_unit_type) { - case H264NalUnitTypes.CODED_SLICE_NON_IDR: - case H264NalUnitTypes.CODED_SLICE_DATA_PART_A: - case H264NalUnitTypes.CODED_SLICE_DATA_PART_B: - case H264NalUnitTypes.CODED_SLICE_DATA_PART_C: - case H264NalUnitTypes.CODED_SLICE_IDR: - final FirstVclNalDetector current = new FirstVclNalDetector(nal, nalUnitHeader.nal_ref_idc, nalUnitHeader.nal_unit_type); - if (fvnd != null && fvnd.isFirstInNew(current)) { - pushSample(createSample(bufferedNals, fvnd.sliceHeader, sliceNalUnitHeader, presentationTimeUs - currentPresentationTimeUs), false, false); - bufferedNals.clear(); - } - currentPresentationTimeUs = Math.max(currentPresentationTimeUs, presentationTimeUs); - sliceNalUnitHeader = nalUnitHeader; - fvnd = current; - bufferedNals.add(nal); - break; - - case H264NalUnitTypes.SEI: - case H264NalUnitTypes.AU_UNIT_DELIMITER: - if (fvnd != null) { - pushSample(createSample(bufferedNals, fvnd.sliceHeader, sliceNalUnitHeader, presentationTimeUs - currentPresentationTimeUs), false, false); - bufferedNals.clear(); - fvnd = null; - } - bufferedNals.add(nal); - break; - - case H264NalUnitTypes.SEQ_PARAMETER_SET: - if (fvnd != null) { - pushSample(createSample(bufferedNals, fvnd.sliceHeader, sliceNalUnitHeader, presentationTimeUs - currentPresentationTimeUs), false, false); - bufferedNals.clear(); - fvnd = null; - } - handleSPS(nal); - break; - - case H264NalUnitTypes.PIC_PARAMETER_SET: - if (fvnd != null) { - pushSample(createSample(bufferedNals, fvnd.sliceHeader, sliceNalUnitHeader, presentationTimeUs - currentPresentationTimeUs), false, false); - bufferedNals.clear(); - fvnd = null; - } - handlePPS(nal); - break; - - case H264NalUnitTypes.END_OF_SEQUENCE: - case H264NalUnitTypes.END_OF_STREAM: - return; - - case H264NalUnitTypes.SEQ_PARAMETER_SET_EXT: - throw new IOException("Sequence parameter set extension is not yet handled. Needs TLC."); - - default: - Log.w(TAG, "Unknown NAL unit type: " + nalUnitHeader.nal_unit_type); - - } - } - - void consumeLastNal() throws IOException { - pushSample(createSample(bufferedNals, fvnd.sliceHeader, sliceNalUnitHeader, 0), true, true); - } - - private void pushSample(final StreamingSample ss, final boolean all, final boolean force) throws IOException { - if (ss != null) { - decFrameBuffer.add(ss); - } - if (all) { - while (decFrameBuffer.size() > 0) { - pushSample(null, false, true); - } - } else { - if ((decFrameBuffer.size() - 1 > maxDecFrameBuffering) || force) { - final StreamingSample first = decFrameBuffer.remove(0); - final PictureOrderCountType0SampleExtension poct0se = first.getSampleExtension(PictureOrderCountType0SampleExtension.class); - if (poct0se == null) { - sampleSink.acceptSample(first, this); - } else { - int delay = 0; - for (StreamingSample streamingSample : decFrameBuffer) { - if (poct0se.getPoc() > streamingSample.getSampleExtension(PictureOrderCountType0SampleExtension.class).getPoc()) { - delay++; - } - } - for (StreamingSample streamingSample : decFrameBuffer2) { - if (poct0se.getPoc() < streamingSample.getSampleExtension(PictureOrderCountType0SampleExtension.class).getPoc()) { - delay--; - } - } - decFrameBuffer2.add(first); - if (decFrameBuffer2.size() > maxDecFrameBuffering) { - decFrameBuffer2.remove(0).removeSampleExtension(PictureOrderCountType0SampleExtension.class); - } - - first.addSampleExtension(CompositionTimeSampleExtension.create(delay * frametick)); - sampleSink.acceptSample(first, this); - } - } - } - - } - - private SampleFlagsSampleExtension createSampleFlagsSampleExtension(H264NalUnitHeader nu, SliceHeader sliceHeader) { - final SampleFlagsSampleExtension sampleFlagsSampleExtension = new SampleFlagsSampleExtension(); - if (nu.nal_ref_idc == 0) { - sampleFlagsSampleExtension.setSampleIsDependedOn(2); - } else { - sampleFlagsSampleExtension.setSampleIsDependedOn(1); - } - if ((sliceHeader.slice_type == SliceHeader.SliceType.I) || (sliceHeader.slice_type == SliceHeader.SliceType.SI)) { - sampleFlagsSampleExtension.setSampleDependsOn(2); - } else { - sampleFlagsSampleExtension.setSampleDependsOn(1); - } - sampleFlagsSampleExtension.setSampleIsNonSyncSample(H264NalUnitTypes.CODED_SLICE_IDR != nu.nal_unit_type); - return sampleFlagsSampleExtension; - } - - private PictureOrderCountType0SampleExtension createPictureOrderCountType0SampleExtension(SliceHeader sliceHeader) { - if (sliceHeader.sps.pic_order_cnt_type == 0) { - return new PictureOrderCountType0SampleExtension( - sliceHeader, decFrameBuffer.size() > 0 ? - decFrameBuffer.get(decFrameBuffer.size() - 1).getSampleExtension(PictureOrderCountType0SampleExtension.class) : - null); -/* decFrameBuffer.add(ssi); - if (decFrameBuffer.size() - 1 > maxDecFrameBuffering) { // just added one - drainDecPictureBuffer(false); - }*/ - } else if (sliceHeader.sps.pic_order_cnt_type == 1) { - throw new MuxingException("pic_order_cnt_type == 1 needs to be implemented"); - } else if (sliceHeader.sps.pic_order_cnt_type == 2) { - return null; // no ctts - } - throw new MuxingException("I don't know sliceHeader.sps.pic_order_cnt_type of " + sliceHeader.sps.pic_order_cnt_type); - } - - - private StreamingSample createSample(List nals, SliceHeader sliceHeader, H264NalUnitHeader nu, long sampleDurationNs) { - final long sampleDuration = getTimescale() * Math.max(0, sampleDurationNs) / 1000000L; - final StreamingSample ss = new StreamingSampleImpl(nals, sampleDuration); - ss.addSampleExtension(createSampleFlagsSampleExtension(nu, sliceHeader)); - final SampleExtension pictureOrderCountType0SampleExtension = createPictureOrderCountType0SampleExtension(sliceHeader); - if (pictureOrderCountType0SampleExtension != null) { - ss.addSampleExtension(pictureOrderCountType0SampleExtension); - } - return ss; - } - - private void handlePPS(final @NonNull ByteBuffer nal) { - nal.position(1); - try { - final PictureParameterSet _pictureParameterSet = PictureParameterSet.read(nal); - final ByteBuffer oldPpsSameId = ppsIdToPpsBytes.get(_pictureParameterSet.pic_parameter_set_id); - if (oldPpsSameId != null && !oldPpsSameId.equals(nal)) { - throw new MuxingException("OMG - I got two SPS with same ID but different settings! (AVC3 is the solution)"); - } else { - ppsIdToPpsBytes.put(_pictureParameterSet.pic_parameter_set_id, nal); - ppsIdToPps.put(_pictureParameterSet.pic_parameter_set_id, _pictureParameterSet); - } - } catch (IOException e) { - throw new MuxingException("That's surprising to get IOException when working on ByteArrayInputStream", e); - } - - - } - - private @NonNull SeqParameterSet handleSPS(final @NonNull ByteBuffer nal) { - nal.position(1); - try { - final SeqParameterSet seqParameterSet = SeqParameterSet.read(nal); - final ByteBuffer oldSpsSameId = spsIdToSpsBytes.get(seqParameterSet.seq_parameter_set_id); - if (oldSpsSameId != null && !oldSpsSameId.equals(nal)) { - throw new MuxingException("OMG - I got two SPS with same ID but different settings!"); - } else { - spsIdToSpsBytes.put(seqParameterSet.seq_parameter_set_id, nal); - spsIdToSps.put(seqParameterSet.seq_parameter_set_id, seqParameterSet); - } - return seqParameterSet; - } catch (IOException e) { - throw new MuxingException("That's surprising to get IOException when working on ByteArrayInputStream", e); - } - - } - - class FirstVclNalDetector { - - final SliceHeader sliceHeader; - final int frame_num; - final int pic_parameter_set_id; - final boolean field_pic_flag; - final boolean bottom_field_flag; - final int nal_ref_idc; - final int pic_order_cnt_type; - final int delta_pic_order_cnt_bottom; - final int pic_order_cnt_lsb; - final int delta_pic_order_cnt_0; - final int delta_pic_order_cnt_1; - final int idr_pic_id; - - FirstVclNalDetector(ByteBuffer nal, int nal_ref_idc, int nal_unit_type) { - - SliceHeader sh = new SliceHeader(nal, spsIdToSps, ppsIdToPps, nal_unit_type == 5); - this.sliceHeader = sh; - this.frame_num = sh.frame_num; - this.pic_parameter_set_id = sh.pic_parameter_set_id; - this.field_pic_flag = sh.field_pic_flag; - this.bottom_field_flag = sh.bottom_field_flag; - this.nal_ref_idc = nal_ref_idc; - this.pic_order_cnt_type = spsIdToSps.get(ppsIdToPps.get(sh.pic_parameter_set_id).seq_parameter_set_id).pic_order_cnt_type; - this.delta_pic_order_cnt_bottom = sh.delta_pic_order_cnt_bottom; - this.pic_order_cnt_lsb = sh.pic_order_cnt_lsb; - this.delta_pic_order_cnt_0 = sh.delta_pic_order_cnt_0; - this.delta_pic_order_cnt_1 = sh.delta_pic_order_cnt_1; - this.idr_pic_id = sh.idr_pic_id; - } - - boolean isFirstInNew(FirstVclNalDetector nu) { - if (nu.frame_num != frame_num) { - return true; - } - if (nu.pic_parameter_set_id != pic_parameter_set_id) { - return true; - } - if (nu.field_pic_flag != field_pic_flag) { - return true; - } - if (nu.field_pic_flag) { - if (nu.bottom_field_flag != bottom_field_flag) { - return true; - } - } - if (nu.nal_ref_idc != nal_ref_idc) { - return true; - } - if (nu.pic_order_cnt_type == 0 && pic_order_cnt_type == 0) { - if (nu.pic_order_cnt_lsb != pic_order_cnt_lsb) { - return true; - } - if (nu.delta_pic_order_cnt_bottom != delta_pic_order_cnt_bottom) { - return true; - } - } - if (nu.pic_order_cnt_type == 1 && pic_order_cnt_type == 1) { - if (nu.delta_pic_order_cnt_0 != delta_pic_order_cnt_0) { - return true; - } - if (nu.delta_pic_order_cnt_1 != delta_pic_order_cnt_1) { - return true; - } - } - return false; - } - } - - static class PictureOrderCountType0SampleExtension implements SampleExtension { - int picOrderCntMsb; - int picOrderCountLsb; - - PictureOrderCountType0SampleExtension(final @NonNull SliceHeader currentSlice, final @Nullable PictureOrderCountType0SampleExtension previous) { - int prevPicOrderCntLsb = 0; - int prevPicOrderCntMsb = 0; - if (previous != null) { - prevPicOrderCntLsb = previous.picOrderCountLsb; - prevPicOrderCntMsb = previous.picOrderCntMsb; - } - - final int maxPicOrderCountLsb = (1 << (currentSlice.sps.log2_max_pic_order_cnt_lsb_minus4 + 4)); - // System.out.print(" pic_order_cnt_lsb " + pic_order_cnt_lsb + " " + max_pic_order_count); - picOrderCountLsb = currentSlice.pic_order_cnt_lsb; - picOrderCntMsb = 0; - if ((picOrderCountLsb < prevPicOrderCntLsb) && ((prevPicOrderCntLsb - picOrderCountLsb) >= (maxPicOrderCountLsb / 2))) { - picOrderCntMsb = prevPicOrderCntMsb + maxPicOrderCountLsb; - } else if ((picOrderCountLsb > prevPicOrderCntLsb) && ((picOrderCountLsb - prevPicOrderCntLsb) > (maxPicOrderCountLsb / 2))) { - picOrderCntMsb = prevPicOrderCntMsb - maxPicOrderCountLsb; - } else { - picOrderCntMsb = prevPicOrderCntMsb; - } - } - - int getPoc() { - return picOrderCntMsb + picOrderCountLsb; - } - - @NonNull - @Override - public String toString() { - return "picOrderCntMsb=" + picOrderCntMsb + ", picOrderCountLsb=" + picOrderCountLsb; - } - } -} +package org.thoughtcrime.securesms.video.videoconverter.muxer; + +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; + +import org.mp4parser.boxes.iso14496.part12.SampleDescriptionBox; +import org.mp4parser.boxes.iso14496.part15.AvcConfigurationBox; +import org.mp4parser.boxes.sampleentry.VisualSampleEntry; +import org.mp4parser.streaming.SampleExtension; +import org.mp4parser.streaming.StreamingSample; +import org.mp4parser.streaming.extensions.CompositionTimeSampleExtension; +import org.mp4parser.streaming.extensions.CompositionTimeTrackExtension; +import org.mp4parser.streaming.extensions.DimensionTrackExtension; +import org.mp4parser.streaming.extensions.SampleFlagsSampleExtension; +import org.mp4parser.streaming.input.AbstractStreamingTrack; +import org.mp4parser.streaming.input.StreamingSampleImpl; +import org.mp4parser.streaming.input.h264.H264NalUnitHeader; +import org.mp4parser.streaming.input.h264.H264NalUnitTypes; +import org.mp4parser.streaming.input.h264.spspps.PictureParameterSet; +import org.mp4parser.streaming.input.h264.spspps.SeqParameterSet; +import org.mp4parser.streaming.input.h264.spspps.SliceHeader; +import org.signal.core.util.logging.Log; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; + +abstract class AvcTrack extends AbstractStreamingTrack { + + private static final String TAG = "AvcTrack"; + + private int maxDecFrameBuffering = 16; + private final List decFrameBuffer = new ArrayList<>(); + private final List decFrameBuffer2 = new ArrayList<>(); + + private final LinkedHashMap spsIdToSpsBytes = new LinkedHashMap<>(); + private final LinkedHashMap spsIdToSps = new LinkedHashMap<>(); + private final LinkedHashMap ppsIdToPpsBytes = new LinkedHashMap<>(); + private final LinkedHashMap ppsIdToPps = new LinkedHashMap<>(); + + private int timescale = 90000; + private int frametick = 3000; + + private final SampleDescriptionBox stsd; + + private final List bufferedNals = new ArrayList<>(); + private FirstVclNalDetector fvnd; + private H264NalUnitHeader sliceNalUnitHeader; + private long currentPresentationTimeUs; + + AvcTrack(final @NonNull ByteBuffer spsBuffer, final @NonNull ByteBuffer ppsBuffer) { + + handlePPS(ppsBuffer); + + final SeqParameterSet sps = handleSPS(spsBuffer); + + int width = (sps.pic_width_in_mbs_minus1 + 1) * 16; + int mult = 2; + if (sps.frame_mbs_only_flag) { + mult = 1; + } + int height = 16 * (sps.pic_height_in_map_units_minus1 + 1) * mult; + if (sps.frame_cropping_flag) { + int chromaArrayType = 0; + if (!sps.residual_color_transform_flag) { + chromaArrayType = sps.chroma_format_idc.getId(); + } + int cropUnitX = 1; + int cropUnitY = mult; + if (chromaArrayType != 0) { + cropUnitX = sps.chroma_format_idc.getSubWidth(); + cropUnitY = sps.chroma_format_idc.getSubHeight() * mult; + } + + width -= cropUnitX * (sps.frame_crop_left_offset + sps.frame_crop_right_offset); + height -= cropUnitY * (sps.frame_crop_top_offset + sps.frame_crop_bottom_offset); + } + + + final VisualSampleEntry visualSampleEntry = new VisualSampleEntry("avc1"); + visualSampleEntry.setDataReferenceIndex(1); + visualSampleEntry.setDepth(24); + visualSampleEntry.setFrameCount(1); + visualSampleEntry.setHorizresolution(72); + visualSampleEntry.setVertresolution(72); + final DimensionTrackExtension dte = this.getTrackExtension(DimensionTrackExtension.class); + if (dte == null) { + this.addTrackExtension(new DimensionTrackExtension(width, height)); + } + visualSampleEntry.setWidth(width); + visualSampleEntry.setHeight(height); + + visualSampleEntry.setCompressorname("AVC Coding"); + + final AvcConfigurationBox avcConfigurationBox = new AvcConfigurationBox(); + + avcConfigurationBox.setSequenceParameterSets(Collections.singletonList(spsBuffer)); + avcConfigurationBox.setPictureParameterSets(Collections.singletonList(ppsBuffer)); + avcConfigurationBox.setAvcLevelIndication(sps.level_idc); + avcConfigurationBox.setAvcProfileIndication(sps.profile_idc); + avcConfigurationBox.setBitDepthLumaMinus8(sps.bit_depth_luma_minus8); + avcConfigurationBox.setBitDepthChromaMinus8(sps.bit_depth_chroma_minus8); + avcConfigurationBox.setChromaFormat(sps.chroma_format_idc.getId()); + avcConfigurationBox.setConfigurationVersion(1); + avcConfigurationBox.setLengthSizeMinusOne(3); + + + avcConfigurationBox.setProfileCompatibility( + (sps.constraint_set_0_flag ? 128 : 0) + + (sps.constraint_set_1_flag ? 64 : 0) + + (sps.constraint_set_2_flag ? 32 : 0) + + (sps.constraint_set_3_flag ? 16 : 0) + + (sps.constraint_set_4_flag ? 8 : 0) + + (int) (sps.reserved_zero_2bits & 0x3) + ); + + visualSampleEntry.addBox(avcConfigurationBox); + stsd = new SampleDescriptionBox(); + stsd.addBox(visualSampleEntry); + + int _timescale; + int _frametick; + if (sps.vuiParams != null) { + _timescale = sps.vuiParams.time_scale >> 1; // Not sure why, but I found this in several places, and it works... + _frametick = sps.vuiParams.num_units_in_tick; + if (_timescale == 0 || _frametick == 0) { + Log.w(TAG, "vuiParams contain invalid values: time_scale: " + _timescale + " and frame_tick: " + _frametick + ". Setting frame rate to 30fps"); + _timescale = 0; + _frametick = 0; + } + if (_frametick > 0) { + if (_timescale / _frametick > 100) { + Log.w(TAG, "Framerate is " + (_timescale / _frametick) + ". That is suspicious."); + } + } else { + Log.w(TAG, "Frametick is " + _frametick + ". That is suspicious."); + } + if (sps.vuiParams.bitstreamRestriction != null) { + maxDecFrameBuffering = sps.vuiParams.bitstreamRestriction.max_dec_frame_buffering; + } + } else { + Log.w(TAG, "Can't determine frame rate as SPS does not contain vuiParama"); + _timescale = 0; + _frametick = 0; + } + if (_timescale != 0 && _frametick != 0) { + timescale = _timescale; + frametick = _frametick; + } + if (sps.pic_order_cnt_type == 0) { + addTrackExtension(new CompositionTimeTrackExtension()); + } else if (sps.pic_order_cnt_type == 1) { + throw new MuxingException("Have not yet imlemented pic_order_cnt_type 1"); + } + } + + public long getTimescale() { + return timescale; + } + + public String getHandler() { + return "vide"; + } + + public String getLanguage() { + return "\u0060\u0060\u0060"; // 0 in Iso639 + } + + public SampleDescriptionBox getSampleDescriptionBox() { + return stsd; + } + + public void close() { + } + + private static H264NalUnitHeader getNalUnitHeader(@NonNull final ByteBuffer nal) { + final H264NalUnitHeader nalUnitHeader = new H264NalUnitHeader(); + final int type = nal.get(0); + nalUnitHeader.nal_ref_idc = (type >> 5) & 3; + nalUnitHeader.nal_unit_type = type & 0x1f; + return nalUnitHeader; + } + + void consumeNal(@NonNull final ByteBuffer nal, final long presentationTimeUs) throws IOException { + + final H264NalUnitHeader nalUnitHeader = getNalUnitHeader(nal); + switch (nalUnitHeader.nal_unit_type) { + case H264NalUnitTypes.CODED_SLICE_NON_IDR: + case H264NalUnitTypes.CODED_SLICE_DATA_PART_A: + case H264NalUnitTypes.CODED_SLICE_DATA_PART_B: + case H264NalUnitTypes.CODED_SLICE_DATA_PART_C: + case H264NalUnitTypes.CODED_SLICE_IDR: + final FirstVclNalDetector current = new FirstVclNalDetector(nal, nalUnitHeader.nal_ref_idc, nalUnitHeader.nal_unit_type); + if (fvnd != null && fvnd.isFirstInNew(current)) { + pushSample(createSample(bufferedNals, fvnd.sliceHeader, sliceNalUnitHeader, presentationTimeUs - currentPresentationTimeUs), false, false); + bufferedNals.clear(); + } + currentPresentationTimeUs = Math.max(currentPresentationTimeUs, presentationTimeUs); + sliceNalUnitHeader = nalUnitHeader; + fvnd = current; + bufferedNals.add(nal); + break; + + case H264NalUnitTypes.SEI: + case H264NalUnitTypes.AU_UNIT_DELIMITER: + if (fvnd != null) { + pushSample(createSample(bufferedNals, fvnd.sliceHeader, sliceNalUnitHeader, presentationTimeUs - currentPresentationTimeUs), false, false); + bufferedNals.clear(); + fvnd = null; + } + bufferedNals.add(nal); + break; + + case H264NalUnitTypes.SEQ_PARAMETER_SET: + if (fvnd != null) { + pushSample(createSample(bufferedNals, fvnd.sliceHeader, sliceNalUnitHeader, presentationTimeUs - currentPresentationTimeUs), false, false); + bufferedNals.clear(); + fvnd = null; + } + handleSPS(nal); + break; + + case H264NalUnitTypes.PIC_PARAMETER_SET: + if (fvnd != null) { + pushSample(createSample(bufferedNals, fvnd.sliceHeader, sliceNalUnitHeader, presentationTimeUs - currentPresentationTimeUs), false, false); + bufferedNals.clear(); + fvnd = null; + } + handlePPS(nal); + break; + + case H264NalUnitTypes.END_OF_SEQUENCE: + case H264NalUnitTypes.END_OF_STREAM: + return; + + case H264NalUnitTypes.SEQ_PARAMETER_SET_EXT: + throw new IOException("Sequence parameter set extension is not yet handled. Needs TLC."); + + default: + Log.w(TAG, "Unknown NAL unit type: " + nalUnitHeader.nal_unit_type); + + } + } + + void consumeLastNal() throws IOException { + pushSample(createSample(bufferedNals, fvnd.sliceHeader, sliceNalUnitHeader, 0), true, true); + } + + private void pushSample(final StreamingSample ss, final boolean all, final boolean force) throws IOException { + if (ss != null) { + decFrameBuffer.add(ss); + } + if (all) { + while (decFrameBuffer.size() > 0) { + pushSample(null, false, true); + } + } else { + if ((decFrameBuffer.size() - 1 > maxDecFrameBuffering) || force) { + final StreamingSample first = decFrameBuffer.remove(0); + final PictureOrderCountType0SampleExtension poct0se = first.getSampleExtension(PictureOrderCountType0SampleExtension.class); + if (poct0se == null) { + sampleSink.acceptSample(first, this); + } else { + int delay = 0; + for (StreamingSample streamingSample : decFrameBuffer) { + if (poct0se.getPoc() > streamingSample.getSampleExtension(PictureOrderCountType0SampleExtension.class).getPoc()) { + delay++; + } + } + for (StreamingSample streamingSample : decFrameBuffer2) { + if (poct0se.getPoc() < streamingSample.getSampleExtension(PictureOrderCountType0SampleExtension.class).getPoc()) { + delay--; + } + } + decFrameBuffer2.add(first); + if (decFrameBuffer2.size() > maxDecFrameBuffering) { + decFrameBuffer2.remove(0).removeSampleExtension(PictureOrderCountType0SampleExtension.class); + } + + first.addSampleExtension(CompositionTimeSampleExtension.create(delay * frametick)); + sampleSink.acceptSample(first, this); + } + } + } + + } + + private SampleFlagsSampleExtension createSampleFlagsSampleExtension(H264NalUnitHeader nu, SliceHeader sliceHeader) { + final SampleFlagsSampleExtension sampleFlagsSampleExtension = new SampleFlagsSampleExtension(); + if (nu.nal_ref_idc == 0) { + sampleFlagsSampleExtension.setSampleIsDependedOn(2); + } else { + sampleFlagsSampleExtension.setSampleIsDependedOn(1); + } + if ((sliceHeader.slice_type == SliceHeader.SliceType.I) || (sliceHeader.slice_type == SliceHeader.SliceType.SI)) { + sampleFlagsSampleExtension.setSampleDependsOn(2); + } else { + sampleFlagsSampleExtension.setSampleDependsOn(1); + } + sampleFlagsSampleExtension.setSampleIsNonSyncSample(H264NalUnitTypes.CODED_SLICE_IDR != nu.nal_unit_type); + return sampleFlagsSampleExtension; + } + + private PictureOrderCountType0SampleExtension createPictureOrderCountType0SampleExtension(SliceHeader sliceHeader) { + if (sliceHeader.sps.pic_order_cnt_type == 0) { + return new PictureOrderCountType0SampleExtension( + sliceHeader, decFrameBuffer.size() > 0 ? + decFrameBuffer.get(decFrameBuffer.size() - 1).getSampleExtension(PictureOrderCountType0SampleExtension.class) : + null); +/* decFrameBuffer.add(ssi); + if (decFrameBuffer.size() - 1 > maxDecFrameBuffering) { // just added one + drainDecPictureBuffer(false); + }*/ + } else if (sliceHeader.sps.pic_order_cnt_type == 1) { + throw new MuxingException("pic_order_cnt_type == 1 needs to be implemented"); + } else if (sliceHeader.sps.pic_order_cnt_type == 2) { + return null; // no ctts + } + throw new MuxingException("I don't know sliceHeader.sps.pic_order_cnt_type of " + sliceHeader.sps.pic_order_cnt_type); + } + + + private StreamingSample createSample(List nals, SliceHeader sliceHeader, H264NalUnitHeader nu, long sampleDurationNs) { + final long sampleDuration = getTimescale() * Math.max(0, sampleDurationNs) / 1000000L; + final StreamingSample ss = new StreamingSampleImpl(nals, sampleDuration); + ss.addSampleExtension(createSampleFlagsSampleExtension(nu, sliceHeader)); + final SampleExtension pictureOrderCountType0SampleExtension = createPictureOrderCountType0SampleExtension(sliceHeader); + if (pictureOrderCountType0SampleExtension != null) { + ss.addSampleExtension(pictureOrderCountType0SampleExtension); + } + return ss; + } + + private void handlePPS(final @NonNull ByteBuffer nal) { + nal.position(1); + try { + final PictureParameterSet _pictureParameterSet = PictureParameterSet.read(nal); + final ByteBuffer oldPpsSameId = ppsIdToPpsBytes.get(_pictureParameterSet.pic_parameter_set_id); + if (oldPpsSameId != null && !oldPpsSameId.equals(nal)) { + throw new MuxingException("OMG - I got two SPS with same ID but different settings! (AVC3 is the solution)"); + } else { + ppsIdToPpsBytes.put(_pictureParameterSet.pic_parameter_set_id, nal); + ppsIdToPps.put(_pictureParameterSet.pic_parameter_set_id, _pictureParameterSet); + } + } catch (IOException e) { + throw new MuxingException("That's surprising to get IOException when working on ByteArrayInputStream", e); + } + + + } + + private @NonNull SeqParameterSet handleSPS(final @NonNull ByteBuffer nal) { + nal.position(1); + try { + final SeqParameterSet seqParameterSet = SeqParameterSet.read(nal); + final ByteBuffer oldSpsSameId = spsIdToSpsBytes.get(seqParameterSet.seq_parameter_set_id); + if (oldSpsSameId != null && !oldSpsSameId.equals(nal)) { + throw new MuxingException("OMG - I got two SPS with same ID but different settings!"); + } else { + spsIdToSpsBytes.put(seqParameterSet.seq_parameter_set_id, nal); + spsIdToSps.put(seqParameterSet.seq_parameter_set_id, seqParameterSet); + } + return seqParameterSet; + } catch (IOException e) { + throw new MuxingException("That's surprising to get IOException when working on ByteArrayInputStream", e); + } + + } + + class FirstVclNalDetector { + + final SliceHeader sliceHeader; + final int frame_num; + final int pic_parameter_set_id; + final boolean field_pic_flag; + final boolean bottom_field_flag; + final int nal_ref_idc; + final int pic_order_cnt_type; + final int delta_pic_order_cnt_bottom; + final int pic_order_cnt_lsb; + final int delta_pic_order_cnt_0; + final int delta_pic_order_cnt_1; + final int idr_pic_id; + + FirstVclNalDetector(ByteBuffer nal, int nal_ref_idc, int nal_unit_type) { + + SliceHeader sh = new SliceHeader(nal, spsIdToSps, ppsIdToPps, nal_unit_type == 5); + this.sliceHeader = sh; + this.frame_num = sh.frame_num; + this.pic_parameter_set_id = sh.pic_parameter_set_id; + this.field_pic_flag = sh.field_pic_flag; + this.bottom_field_flag = sh.bottom_field_flag; + this.nal_ref_idc = nal_ref_idc; + this.pic_order_cnt_type = spsIdToSps.get(ppsIdToPps.get(sh.pic_parameter_set_id).seq_parameter_set_id).pic_order_cnt_type; + this.delta_pic_order_cnt_bottom = sh.delta_pic_order_cnt_bottom; + this.pic_order_cnt_lsb = sh.pic_order_cnt_lsb; + this.delta_pic_order_cnt_0 = sh.delta_pic_order_cnt_0; + this.delta_pic_order_cnt_1 = sh.delta_pic_order_cnt_1; + this.idr_pic_id = sh.idr_pic_id; + } + + boolean isFirstInNew(FirstVclNalDetector nu) { + if (nu.frame_num != frame_num) { + return true; + } + if (nu.pic_parameter_set_id != pic_parameter_set_id) { + return true; + } + if (nu.field_pic_flag != field_pic_flag) { + return true; + } + if (nu.field_pic_flag) { + if (nu.bottom_field_flag != bottom_field_flag) { + return true; + } + } + if (nu.nal_ref_idc != nal_ref_idc) { + return true; + } + if (nu.pic_order_cnt_type == 0 && pic_order_cnt_type == 0) { + if (nu.pic_order_cnt_lsb != pic_order_cnt_lsb) { + return true; + } + if (nu.delta_pic_order_cnt_bottom != delta_pic_order_cnt_bottom) { + return true; + } + } + if (nu.pic_order_cnt_type == 1 && pic_order_cnt_type == 1) { + if (nu.delta_pic_order_cnt_0 != delta_pic_order_cnt_0) { + return true; + } + if (nu.delta_pic_order_cnt_1 != delta_pic_order_cnt_1) { + return true; + } + } + return false; + } + } + + static class PictureOrderCountType0SampleExtension implements SampleExtension { + int picOrderCntMsb; + int picOrderCountLsb; + + PictureOrderCountType0SampleExtension(final @NonNull SliceHeader currentSlice, final @Nullable PictureOrderCountType0SampleExtension previous) { + int prevPicOrderCntLsb = 0; + int prevPicOrderCntMsb = 0; + if (previous != null) { + prevPicOrderCntLsb = previous.picOrderCountLsb; + prevPicOrderCntMsb = previous.picOrderCntMsb; + } + + final int maxPicOrderCountLsb = (1 << (currentSlice.sps.log2_max_pic_order_cnt_lsb_minus4 + 4)); + // System.out.print(" pic_order_cnt_lsb " + pic_order_cnt_lsb + " " + max_pic_order_count); + picOrderCountLsb = currentSlice.pic_order_cnt_lsb; + picOrderCntMsb = 0; + if ((picOrderCountLsb < prevPicOrderCntLsb) && ((prevPicOrderCntLsb - picOrderCountLsb) >= (maxPicOrderCountLsb / 2))) { + picOrderCntMsb = prevPicOrderCntMsb + maxPicOrderCountLsb; + } else if ((picOrderCountLsb > prevPicOrderCntLsb) && ((picOrderCountLsb - prevPicOrderCntLsb) > (maxPicOrderCountLsb / 2))) { + picOrderCntMsb = prevPicOrderCntMsb - maxPicOrderCountLsb; + } else { + picOrderCntMsb = prevPicOrderCntMsb; + } + } + + int getPoc() { + return picOrderCntMsb + picOrderCountLsb; + } + + @NonNull + @Override + public String toString() { + return "picOrderCntMsb=" + picOrderCntMsb + ", picOrderCountLsb=" + picOrderCountLsb; + } + } +} diff --git a/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/muxer/H264Utils.java b/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/muxer/H264Utils.java index f401cdbd07..c4e26cd291 100644 --- a/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/muxer/H264Utils.java +++ b/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/muxer/H264Utils.java @@ -1,99 +1,99 @@ -/* - * Copyright 2008-2019 JCodecProject - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. Redistributions in binary form - * must reproduce the above copyright notice, this list of conditions and the - * following disclaimer in the documentation and/or other materials provided with - * the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR - * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON - * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * https://github.com/jcodec/jcodec/blob/master/src/main/java/org/jcodec/codecs/h264/H264Utils.java - * - * This file has been modified by Signal. - */ -package org.thoughtcrime.securesms.video.videoconverter.muxer; - -import androidx.annotation.NonNull; - -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.util.ArrayList; -import java.util.List; - -final class H264Utils { - - private H264Utils() {} - - static @NonNull List getNals(ByteBuffer buffer) { - final List nals = new ArrayList<>(); - ByteBuffer nal; - while ((nal = nextNALUnit(buffer)) != null) { - nals.add(nal); - } - return nals; - } - - static ByteBuffer nextNALUnit(ByteBuffer buf) { - skipToNALUnit(buf); - return gotoNALUnit(buf); - } - - static void skipToNALUnit(ByteBuffer buf) { - if (!buf.hasRemaining()) - return; - - int val = 0xffffffff; - while (buf.hasRemaining()) { - val <<= 8; - val |= (buf.get() & 0xff); - if ((val & 0xffffff) == 1) { - buf.position(buf.position()); - break; - } - } - } - - /** - * Finds next Nth H.264 bitstream NAL unit (0x00000001) and returns the data - * that preceeds it as a ByteBuffer slice - *

- * Segment byte order is always little endian - *

- * TODO: emulation prevention - */ - static ByteBuffer gotoNALUnit(ByteBuffer buf) { - - if (!buf.hasRemaining()) - return null; - - int from = buf.position(); - ByteBuffer result = buf.slice(); - result.order(ByteOrder.BIG_ENDIAN); - - int val = 0xffffffff; - while (buf.hasRemaining()) { - val <<= 8; - val |= (buf.get() & 0xff); - if ((val & 0xffffff) == 1) { - buf.position(buf.position() - (val == 1 ? 4 : 3)); - result.limit(buf.position() - from); - break; - } - } - return result; - } -} +/* + * Copyright 2008-2019 JCodecProject + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. Redistributions in binary form + * must reproduce the above copyright notice, this list of conditions and the + * following disclaimer in the documentation and/or other materials provided with + * the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * https://github.com/jcodec/jcodec/blob/master/src/main/java/org/jcodec/codecs/h264/H264Utils.java + * + * This file has been modified by Signal. + */ +package org.thoughtcrime.securesms.video.videoconverter.muxer; + +import androidx.annotation.NonNull; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.ArrayList; +import java.util.List; + +final class H264Utils { + + private H264Utils() {} + + static @NonNull List getNals(ByteBuffer buffer) { + final List nals = new ArrayList<>(); + ByteBuffer nal; + while ((nal = nextNALUnit(buffer)) != null) { + nals.add(nal); + } + return nals; + } + + static ByteBuffer nextNALUnit(ByteBuffer buf) { + skipToNALUnit(buf); + return gotoNALUnit(buf); + } + + static void skipToNALUnit(ByteBuffer buf) { + if (!buf.hasRemaining()) + return; + + int val = 0xffffffff; + while (buf.hasRemaining()) { + val <<= 8; + val |= (buf.get() & 0xff); + if ((val & 0xffffff) == 1) { + buf.position(buf.position()); + break; + } + } + } + + /** + * Finds next Nth H.264 bitstream NAL unit (0x00000001) and returns the data + * that preceeds it as a ByteBuffer slice + *

+ * Segment byte order is always little endian + *

+ * TODO: emulation prevention + */ + static ByteBuffer gotoNALUnit(ByteBuffer buf) { + + if (!buf.hasRemaining()) + return null; + + int from = buf.position(); + ByteBuffer result = buf.slice(); + result.order(ByteOrder.BIG_ENDIAN); + + int val = 0xffffffff; + while (buf.hasRemaining()) { + val <<= 8; + val |= (buf.get() & 0xff); + if ((val & 0xffffff) == 1) { + buf.position(buf.position() - (val == 1 ? 4 : 3)); + result.limit(buf.position() - from); + break; + } + } + return result; + } +} diff --git a/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/muxer/HevcTrack.java b/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/muxer/HevcTrack.java index 94cf850a89..870f3f06c3 100644 --- a/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/muxer/HevcTrack.java +++ b/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/muxer/HevcTrack.java @@ -1,261 +1,261 @@ -package org.thoughtcrime.securesms.video.videoconverter.muxer; - -import androidx.annotation.NonNull; -import androidx.annotation.Nullable; - -import org.mp4parser.boxes.iso14496.part12.SampleDescriptionBox; -import org.mp4parser.boxes.iso14496.part15.HevcConfigurationBox; -import org.mp4parser.boxes.iso14496.part15.HevcDecoderConfigurationRecord; -import org.mp4parser.boxes.sampleentry.VisualSampleEntry; -import org.mp4parser.muxer.tracks.CleanInputStream; -import org.mp4parser.muxer.tracks.h265.H265NalUnitHeader; -import org.mp4parser.muxer.tracks.h265.H265NalUnitTypes; -import org.mp4parser.muxer.tracks.h265.SequenceParameterSetRbsp; -import org.mp4parser.streaming.StreamingSample; -import org.mp4parser.streaming.extensions.DimensionTrackExtension; -import org.mp4parser.streaming.extensions.SampleFlagsSampleExtension; -import org.mp4parser.streaming.input.AbstractStreamingTrack; -import org.mp4parser.streaming.input.StreamingSampleImpl; -import org.mp4parser.tools.ByteBufferByteChannel; -import org.mp4parser.tools.IsoTypeReader; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.channels.Channels; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -abstract class HevcTrack extends AbstractStreamingTrack implements H265NalUnitTypes { - - private final ArrayList bufferedNals = new ArrayList<>(); - private boolean vclNalUnitSeenInAU; - private boolean isIdr = true; - private long currentPresentationTimeUs; - private final SampleDescriptionBox stsd; - - HevcTrack(final @NonNull List csd) throws IOException { - final ArrayList sps = new ArrayList<>(); - final ArrayList pps = new ArrayList<>(); - final ArrayList vps = new ArrayList<>(); - SequenceParameterSetRbsp spsStruct = null; - for (ByteBuffer nal : csd) { - final H265NalUnitHeader unitHeader = getNalUnitHeader(nal); - nal.position(0); - // collect sps/vps/pps - switch (unitHeader.nalUnitType) { - case NAL_TYPE_PPS_NUT: - pps.add(nal.duplicate()); - break; - case NAL_TYPE_VPS_NUT: - vps.add(nal.duplicate()); - break; - case NAL_TYPE_SPS_NUT: - sps.add(nal.duplicate()); - nal.position(2); - spsStruct = new SequenceParameterSetRbsp(new CleanInputStream(Channels.newInputStream(new ByteBufferByteChannel(nal.slice())))); - break; - case NAL_TYPE_PREFIX_SEI_NUT: - //new SEIMessage(new BitReaderBuffer(nal.slice())); - break; - } - } - - stsd = new SampleDescriptionBox(); - stsd.addBox(createSampleEntry(sps, pps, vps, spsStruct)); - - } - - @Override - public long getTimescale() { - return 90000; - } - - @Override - public String getHandler() { - return "vide"; - } - - @Override - public String getLanguage() { - return "\u0060\u0060\u0060"; // 0 in Iso639 - } - - @Override - public SampleDescriptionBox getSampleDescriptionBox() { - return stsd; - } - - @Override - public void close() { - } - - void consumeLastNal() throws IOException { - wrapUp(bufferedNals, currentPresentationTimeUs); - } - - void consumeNal(final @NonNull ByteBuffer nal, final long presentationTimeUs) throws IOException { - - final H265NalUnitHeader unitHeader = getNalUnitHeader(nal); - final boolean isVcl = isVcl(unitHeader); - // - if (vclNalUnitSeenInAU) { // we need at least 1 VCL per AU - // This branch checks if we encountered the start of a samples/AU - if (isVcl) { - if ((nal.get(2) & -128) != 0) { // this is: first_slice_segment_in_pic_flag u(1) - wrapUp(bufferedNals, presentationTimeUs); - } - } else { - switch (unitHeader.nalUnitType) { - case NAL_TYPE_PREFIX_SEI_NUT: - case NAL_TYPE_AUD_NUT: - case NAL_TYPE_PPS_NUT: - case NAL_TYPE_VPS_NUT: - case NAL_TYPE_SPS_NUT: - case NAL_TYPE_RSV_NVCL41: - case NAL_TYPE_RSV_NVCL42: - case NAL_TYPE_RSV_NVCL43: - case NAL_TYPE_RSV_NVCL44: - case NAL_TYPE_UNSPEC48: - case NAL_TYPE_UNSPEC49: - case NAL_TYPE_UNSPEC50: - case NAL_TYPE_UNSPEC51: - case NAL_TYPE_UNSPEC52: - case NAL_TYPE_UNSPEC53: - case NAL_TYPE_UNSPEC54: - case NAL_TYPE_UNSPEC55: - - case NAL_TYPE_EOB_NUT: // a bit special but also causes a sample to be formed - case NAL_TYPE_EOS_NUT: - wrapUp(bufferedNals, presentationTimeUs); - break; - } - } - } - - - switch (unitHeader.nalUnitType) { - case NAL_TYPE_SPS_NUT: - case NAL_TYPE_VPS_NUT: - case NAL_TYPE_PPS_NUT: - case NAL_TYPE_EOB_NUT: - case NAL_TYPE_EOS_NUT: - case NAL_TYPE_AUD_NUT: - case NAL_TYPE_FD_NUT: - // ignore these - break; - default: - bufferedNals.add(nal); - break; - } - - if (isVcl) { - isIdr = unitHeader.nalUnitType == NAL_TYPE_IDR_W_RADL || unitHeader.nalUnitType == NAL_TYPE_IDR_N_LP; - vclNalUnitSeenInAU = true; - } - } - - private void wrapUp(final @NonNull List nals, final long presentationTimeUs) throws IOException { - - final long duration = presentationTimeUs - currentPresentationTimeUs; - currentPresentationTimeUs = presentationTimeUs; - - final StreamingSample sample = new StreamingSampleImpl( - nals, getTimescale() * Math.max(0, duration) / 1000000L); - - final SampleFlagsSampleExtension sampleFlagsSampleExtension = new SampleFlagsSampleExtension(); - sampleFlagsSampleExtension.setSampleIsNonSyncSample(!isIdr); - - sample.addSampleExtension(sampleFlagsSampleExtension); - - sampleSink.acceptSample(sample, this); - - vclNalUnitSeenInAU = false; - isIdr = true; - nals.clear(); - } - - private static @NonNull H265NalUnitHeader getNalUnitHeader(final @NonNull ByteBuffer nal) { - nal.position(0); - final int nalUnitHeaderValue = IsoTypeReader.readUInt16(nal); - final H265NalUnitHeader nalUnitHeader = new H265NalUnitHeader(); - nalUnitHeader.forbiddenZeroFlag = (nalUnitHeaderValue & 0x8000) >> 15; - nalUnitHeader.nalUnitType = (nalUnitHeaderValue & 0x7E00) >> 9; - nalUnitHeader.nuhLayerId = (nalUnitHeaderValue & 0x1F8) >> 3; - nalUnitHeader.nuhTemporalIdPlusOne = (nalUnitHeaderValue & 0x7); - return nalUnitHeader; - } - - private @NonNull VisualSampleEntry createSampleEntry( - final @NonNull ArrayList sps, - final @NonNull ArrayList pps, - final @NonNull ArrayList vps, - final @Nullable SequenceParameterSetRbsp spsStruct) - { - final VisualSampleEntry visualSampleEntry = new VisualSampleEntry("hvc1"); - visualSampleEntry.setDataReferenceIndex(1); - visualSampleEntry.setDepth(24); - visualSampleEntry.setFrameCount(1); - visualSampleEntry.setHorizresolution(72); - visualSampleEntry.setVertresolution(72); - visualSampleEntry.setCompressorname("HEVC Coding"); - - final HevcConfigurationBox hevcConfigurationBox = new HevcConfigurationBox(); - hevcConfigurationBox.getHevcDecoderConfigurationRecord().setConfigurationVersion(1); - - if (spsStruct != null) { - visualSampleEntry.setWidth(spsStruct.pic_width_in_luma_samples); - visualSampleEntry.setHeight(spsStruct.pic_height_in_luma_samples); - final DimensionTrackExtension dte = this.getTrackExtension(DimensionTrackExtension.class); - if (dte == null) { - this.addTrackExtension(new DimensionTrackExtension(spsStruct.pic_width_in_luma_samples, spsStruct.pic_height_in_luma_samples)); - } - final HevcDecoderConfigurationRecord hevcDecoderConfigurationRecord = hevcConfigurationBox.getHevcDecoderConfigurationRecord(); - hevcDecoderConfigurationRecord.setChromaFormat(spsStruct.chroma_format_idc); - hevcDecoderConfigurationRecord.setGeneral_profile_idc(spsStruct.general_profile_idc); - hevcDecoderConfigurationRecord.setGeneral_profile_compatibility_flags(spsStruct.general_profile_compatibility_flags); - hevcDecoderConfigurationRecord.setGeneral_constraint_indicator_flags(spsStruct.general_constraint_indicator_flags); - hevcDecoderConfigurationRecord.setGeneral_level_idc(spsStruct.general_level_idc); - hevcDecoderConfigurationRecord.setGeneral_tier_flag(spsStruct.general_tier_flag); - hevcDecoderConfigurationRecord.setGeneral_profile_space(spsStruct.general_profile_space); - hevcDecoderConfigurationRecord.setBitDepthChromaMinus8(spsStruct.bit_depth_chroma_minus8); - hevcDecoderConfigurationRecord.setBitDepthLumaMinus8(spsStruct.bit_depth_luma_minus8); - hevcDecoderConfigurationRecord.setTemporalIdNested(spsStruct.sps_temporal_id_nesting_flag); - } - - hevcConfigurationBox.getHevcDecoderConfigurationRecord().setLengthSizeMinusOne(3); - - final HevcDecoderConfigurationRecord.Array vpsArray = new HevcDecoderConfigurationRecord.Array(); - vpsArray.array_completeness = false; - vpsArray.nal_unit_type = NAL_TYPE_VPS_NUT; - vpsArray.nalUnits = new ArrayList<>(); - for (ByteBuffer vp : vps) { - vpsArray.nalUnits.add(Utils.toArray(vp)); - } - - final HevcDecoderConfigurationRecord.Array spsArray = new HevcDecoderConfigurationRecord.Array(); - spsArray.array_completeness = false; - spsArray.nal_unit_type = NAL_TYPE_SPS_NUT; - spsArray.nalUnits = new ArrayList<>(); - for (ByteBuffer sp : sps) { - spsArray.nalUnits.add(Utils.toArray(sp)); - } - - final HevcDecoderConfigurationRecord.Array ppsArray = new HevcDecoderConfigurationRecord.Array(); - ppsArray.array_completeness = false; - ppsArray.nal_unit_type = NAL_TYPE_PPS_NUT; - ppsArray.nalUnits = new ArrayList<>(); - for (ByteBuffer pp : pps) { - ppsArray.nalUnits.add(Utils.toArray(pp)); - } - - hevcConfigurationBox.getArrays().addAll(Arrays.asList(spsArray, vpsArray, ppsArray)); - - visualSampleEntry.addBox(hevcConfigurationBox); - return visualSampleEntry; - } - - private boolean isVcl(final @NonNull H265NalUnitHeader nalUnitHeader) { - return nalUnitHeader.nalUnitType >= 0 && nalUnitHeader.nalUnitType <= 31; - } -} +package org.thoughtcrime.securesms.video.videoconverter.muxer; + +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; + +import org.mp4parser.boxes.iso14496.part12.SampleDescriptionBox; +import org.mp4parser.boxes.iso14496.part15.HevcConfigurationBox; +import org.mp4parser.boxes.iso14496.part15.HevcDecoderConfigurationRecord; +import org.mp4parser.boxes.sampleentry.VisualSampleEntry; +import org.mp4parser.muxer.tracks.CleanInputStream; +import org.mp4parser.muxer.tracks.h265.H265NalUnitHeader; +import org.mp4parser.muxer.tracks.h265.H265NalUnitTypes; +import org.mp4parser.muxer.tracks.h265.SequenceParameterSetRbsp; +import org.mp4parser.streaming.StreamingSample; +import org.mp4parser.streaming.extensions.DimensionTrackExtension; +import org.mp4parser.streaming.extensions.SampleFlagsSampleExtension; +import org.mp4parser.streaming.input.AbstractStreamingTrack; +import org.mp4parser.streaming.input.StreamingSampleImpl; +import org.mp4parser.tools.ByteBufferByteChannel; +import org.mp4parser.tools.IsoTypeReader; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.Channels; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +abstract class HevcTrack extends AbstractStreamingTrack implements H265NalUnitTypes { + + private final ArrayList bufferedNals = new ArrayList<>(); + private boolean vclNalUnitSeenInAU; + private boolean isIdr = true; + private long currentPresentationTimeUs; + private final SampleDescriptionBox stsd; + + HevcTrack(final @NonNull List csd) throws IOException { + final ArrayList sps = new ArrayList<>(); + final ArrayList pps = new ArrayList<>(); + final ArrayList vps = new ArrayList<>(); + SequenceParameterSetRbsp spsStruct = null; + for (ByteBuffer nal : csd) { + final H265NalUnitHeader unitHeader = getNalUnitHeader(nal); + nal.position(0); + // collect sps/vps/pps + switch (unitHeader.nalUnitType) { + case NAL_TYPE_PPS_NUT: + pps.add(nal.duplicate()); + break; + case NAL_TYPE_VPS_NUT: + vps.add(nal.duplicate()); + break; + case NAL_TYPE_SPS_NUT: + sps.add(nal.duplicate()); + nal.position(2); + spsStruct = new SequenceParameterSetRbsp(new CleanInputStream(Channels.newInputStream(new ByteBufferByteChannel(nal.slice())))); + break; + case NAL_TYPE_PREFIX_SEI_NUT: + //new SEIMessage(new BitReaderBuffer(nal.slice())); + break; + } + } + + stsd = new SampleDescriptionBox(); + stsd.addBox(createSampleEntry(sps, pps, vps, spsStruct)); + + } + + @Override + public long getTimescale() { + return 90000; + } + + @Override + public String getHandler() { + return "vide"; + } + + @Override + public String getLanguage() { + return "\u0060\u0060\u0060"; // 0 in Iso639 + } + + @Override + public SampleDescriptionBox getSampleDescriptionBox() { + return stsd; + } + + @Override + public void close() { + } + + void consumeLastNal() throws IOException { + wrapUp(bufferedNals, currentPresentationTimeUs); + } + + void consumeNal(final @NonNull ByteBuffer nal, final long presentationTimeUs) throws IOException { + + final H265NalUnitHeader unitHeader = getNalUnitHeader(nal); + final boolean isVcl = isVcl(unitHeader); + // + if (vclNalUnitSeenInAU) { // we need at least 1 VCL per AU + // This branch checks if we encountered the start of a samples/AU + if (isVcl) { + if ((nal.get(2) & -128) != 0) { // this is: first_slice_segment_in_pic_flag u(1) + wrapUp(bufferedNals, presentationTimeUs); + } + } else { + switch (unitHeader.nalUnitType) { + case NAL_TYPE_PREFIX_SEI_NUT: + case NAL_TYPE_AUD_NUT: + case NAL_TYPE_PPS_NUT: + case NAL_TYPE_VPS_NUT: + case NAL_TYPE_SPS_NUT: + case NAL_TYPE_RSV_NVCL41: + case NAL_TYPE_RSV_NVCL42: + case NAL_TYPE_RSV_NVCL43: + case NAL_TYPE_RSV_NVCL44: + case NAL_TYPE_UNSPEC48: + case NAL_TYPE_UNSPEC49: + case NAL_TYPE_UNSPEC50: + case NAL_TYPE_UNSPEC51: + case NAL_TYPE_UNSPEC52: + case NAL_TYPE_UNSPEC53: + case NAL_TYPE_UNSPEC54: + case NAL_TYPE_UNSPEC55: + + case NAL_TYPE_EOB_NUT: // a bit special but also causes a sample to be formed + case NAL_TYPE_EOS_NUT: + wrapUp(bufferedNals, presentationTimeUs); + break; + } + } + } + + + switch (unitHeader.nalUnitType) { + case NAL_TYPE_SPS_NUT: + case NAL_TYPE_VPS_NUT: + case NAL_TYPE_PPS_NUT: + case NAL_TYPE_EOB_NUT: + case NAL_TYPE_EOS_NUT: + case NAL_TYPE_AUD_NUT: + case NAL_TYPE_FD_NUT: + // ignore these + break; + default: + bufferedNals.add(nal); + break; + } + + if (isVcl) { + isIdr = unitHeader.nalUnitType == NAL_TYPE_IDR_W_RADL || unitHeader.nalUnitType == NAL_TYPE_IDR_N_LP; + vclNalUnitSeenInAU = true; + } + } + + private void wrapUp(final @NonNull List nals, final long presentationTimeUs) throws IOException { + + final long duration = presentationTimeUs - currentPresentationTimeUs; + currentPresentationTimeUs = presentationTimeUs; + + final StreamingSample sample = new StreamingSampleImpl( + nals, getTimescale() * Math.max(0, duration) / 1000000L); + + final SampleFlagsSampleExtension sampleFlagsSampleExtension = new SampleFlagsSampleExtension(); + sampleFlagsSampleExtension.setSampleIsNonSyncSample(!isIdr); + + sample.addSampleExtension(sampleFlagsSampleExtension); + + sampleSink.acceptSample(sample, this); + + vclNalUnitSeenInAU = false; + isIdr = true; + nals.clear(); + } + + private static @NonNull H265NalUnitHeader getNalUnitHeader(final @NonNull ByteBuffer nal) { + nal.position(0); + final int nalUnitHeaderValue = IsoTypeReader.readUInt16(nal); + final H265NalUnitHeader nalUnitHeader = new H265NalUnitHeader(); + nalUnitHeader.forbiddenZeroFlag = (nalUnitHeaderValue & 0x8000) >> 15; + nalUnitHeader.nalUnitType = (nalUnitHeaderValue & 0x7E00) >> 9; + nalUnitHeader.nuhLayerId = (nalUnitHeaderValue & 0x1F8) >> 3; + nalUnitHeader.nuhTemporalIdPlusOne = (nalUnitHeaderValue & 0x7); + return nalUnitHeader; + } + + private @NonNull VisualSampleEntry createSampleEntry( + final @NonNull ArrayList sps, + final @NonNull ArrayList pps, + final @NonNull ArrayList vps, + final @Nullable SequenceParameterSetRbsp spsStruct) + { + final VisualSampleEntry visualSampleEntry = new VisualSampleEntry("hvc1"); + visualSampleEntry.setDataReferenceIndex(1); + visualSampleEntry.setDepth(24); + visualSampleEntry.setFrameCount(1); + visualSampleEntry.setHorizresolution(72); + visualSampleEntry.setVertresolution(72); + visualSampleEntry.setCompressorname("HEVC Coding"); + + final HevcConfigurationBox hevcConfigurationBox = new HevcConfigurationBox(); + hevcConfigurationBox.getHevcDecoderConfigurationRecord().setConfigurationVersion(1); + + if (spsStruct != null) { + visualSampleEntry.setWidth(spsStruct.pic_width_in_luma_samples); + visualSampleEntry.setHeight(spsStruct.pic_height_in_luma_samples); + final DimensionTrackExtension dte = this.getTrackExtension(DimensionTrackExtension.class); + if (dte == null) { + this.addTrackExtension(new DimensionTrackExtension(spsStruct.pic_width_in_luma_samples, spsStruct.pic_height_in_luma_samples)); + } + final HevcDecoderConfigurationRecord hevcDecoderConfigurationRecord = hevcConfigurationBox.getHevcDecoderConfigurationRecord(); + hevcDecoderConfigurationRecord.setChromaFormat(spsStruct.chroma_format_idc); + hevcDecoderConfigurationRecord.setGeneral_profile_idc(spsStruct.general_profile_idc); + hevcDecoderConfigurationRecord.setGeneral_profile_compatibility_flags(spsStruct.general_profile_compatibility_flags); + hevcDecoderConfigurationRecord.setGeneral_constraint_indicator_flags(spsStruct.general_constraint_indicator_flags); + hevcDecoderConfigurationRecord.setGeneral_level_idc(spsStruct.general_level_idc); + hevcDecoderConfigurationRecord.setGeneral_tier_flag(spsStruct.general_tier_flag); + hevcDecoderConfigurationRecord.setGeneral_profile_space(spsStruct.general_profile_space); + hevcDecoderConfigurationRecord.setBitDepthChromaMinus8(spsStruct.bit_depth_chroma_minus8); + hevcDecoderConfigurationRecord.setBitDepthLumaMinus8(spsStruct.bit_depth_luma_minus8); + hevcDecoderConfigurationRecord.setTemporalIdNested(spsStruct.sps_temporal_id_nesting_flag); + } + + hevcConfigurationBox.getHevcDecoderConfigurationRecord().setLengthSizeMinusOne(3); + + final HevcDecoderConfigurationRecord.Array vpsArray = new HevcDecoderConfigurationRecord.Array(); + vpsArray.array_completeness = false; + vpsArray.nal_unit_type = NAL_TYPE_VPS_NUT; + vpsArray.nalUnits = new ArrayList<>(); + for (ByteBuffer vp : vps) { + vpsArray.nalUnits.add(Utils.toArray(vp)); + } + + final HevcDecoderConfigurationRecord.Array spsArray = new HevcDecoderConfigurationRecord.Array(); + spsArray.array_completeness = false; + spsArray.nal_unit_type = NAL_TYPE_SPS_NUT; + spsArray.nalUnits = new ArrayList<>(); + for (ByteBuffer sp : sps) { + spsArray.nalUnits.add(Utils.toArray(sp)); + } + + final HevcDecoderConfigurationRecord.Array ppsArray = new HevcDecoderConfigurationRecord.Array(); + ppsArray.array_completeness = false; + ppsArray.nal_unit_type = NAL_TYPE_PPS_NUT; + ppsArray.nalUnits = new ArrayList<>(); + for (ByteBuffer pp : pps) { + ppsArray.nalUnits.add(Utils.toArray(pp)); + } + + hevcConfigurationBox.getArrays().addAll(Arrays.asList(spsArray, vpsArray, ppsArray)); + + visualSampleEntry.addBox(hevcConfigurationBox); + return visualSampleEntry; + } + + private boolean isVcl(final @NonNull H265NalUnitHeader nalUnitHeader) { + return nalUnitHeader.nalUnitType >= 0 && nalUnitHeader.nalUnitType <= 31; + } +} diff --git a/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/muxer/Mp4Writer.java b/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/muxer/Mp4Writer.java index 2d7484d73d..5b85f9ff64 100644 --- a/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/muxer/Mp4Writer.java +++ b/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/muxer/Mp4Writer.java @@ -1,523 +1,523 @@ -/* - * Copyright (C) https://github.com/sannies/mp4parser/blob/master/LICENSE - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * https://github.com/sannies/mp4parser/blob/4ed724754cde751c3f27fdda51f288df4f4c5db5/streaming/src/main/java/org/mp4parser/streaming/output/mp4/StandardMp4Writer.java - * - * This file has been modified by Signal. - */ -package org.thoughtcrime.securesms.video.videoconverter.muxer; - -import androidx.annotation.NonNull; - -import org.mp4parser.Box; -import org.mp4parser.boxes.iso14496.part12.ChunkOffsetBox; -import org.mp4parser.boxes.iso14496.part12.CompositionTimeToSample; -import org.mp4parser.boxes.iso14496.part12.FileTypeBox; -import org.mp4parser.boxes.iso14496.part12.MediaHeaderBox; -import org.mp4parser.boxes.iso14496.part12.MovieBox; -import org.mp4parser.boxes.iso14496.part12.MovieHeaderBox; -import org.mp4parser.boxes.iso14496.part12.SampleSizeBox; -import org.mp4parser.boxes.iso14496.part12.SampleTableBox; -import org.mp4parser.boxes.iso14496.part12.SampleToChunkBox; -import org.mp4parser.boxes.iso14496.part12.SyncSampleBox; -import org.mp4parser.boxes.iso14496.part12.TimeToSampleBox; -import org.mp4parser.boxes.iso14496.part12.TrackBox; -import org.mp4parser.boxes.iso14496.part12.TrackHeaderBox; -import org.mp4parser.streaming.StreamingSample; -import org.mp4parser.streaming.StreamingTrack; -import org.mp4parser.streaming.extensions.CompositionTimeSampleExtension; -import org.mp4parser.streaming.extensions.CompositionTimeTrackExtension; -import org.mp4parser.streaming.extensions.SampleFlagsSampleExtension; -import org.mp4parser.streaming.extensions.TrackIdTrackExtension; -import org.mp4parser.streaming.output.SampleSink; -import org.mp4parser.streaming.output.mp4.DefaultBoxes; -import org.mp4parser.tools.Mp4Arrays; -import org.mp4parser.tools.Mp4Math; -import org.mp4parser.tools.Path; -import org.signal.core.util.logging.Log; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.channels.WritableByteChannel; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Queue; -import java.util.concurrent.ConcurrentHashMap; - -import static org.mp4parser.tools.CastUtils.l2i; - -/** - * Creates an MP4 file with ftyp, mdat+, moov order. - * A very special property of this variant is that it written sequentially. You can start transferring the - * data while the sink receives it. (in contrast to typical implementations which need random - * access to write length fields at the beginning of the file) - */ -final class Mp4Writer extends DefaultBoxes implements SampleSink { - - private static final String TAG = "Mp4Writer"; - private static final Long UInt32_MAX = (1L << 32) - 1; - - private final WritableByteChannel sink; - private final List source; - private final Date creationTime = new Date(); - - private boolean hasWrittenMdat = false; - - /** - * Contains the start time of the next segment in line that will be created. - */ - private final Map nextChunkCreateStartTime = new ConcurrentHashMap<>(); - /** - * Contains the start time of the next segment in line that will be written. - */ - private final Map nextChunkWriteStartTime = new ConcurrentHashMap<>(); - /** - * Contains the next sample's start time. - */ - private final Map nextSampleStartTime = new HashMap<>(); - /** - * Buffers the samples per track until there are enough samples to form a Segment. - */ - private final Map> sampleBuffers = new HashMap<>(); - private final Map trackBoxes = new HashMap<>(); - /** - * Buffers segments until it's time for a segment to be written. - */ - private final Map> chunkBuffers = new ConcurrentHashMap<>(); - private final Map chunkNumbers = new HashMap<>(); - private final Map sampleNumbers = new HashMap<>(); - private long bytesWritten = 0; - - private long mMDatTotalContentLength = 0; - - Mp4Writer(final @NonNull List source, final @NonNull WritableByteChannel sink) throws IOException { - this.source = new ArrayList<>(source); - this.sink = sink; - - final HashSet trackIds = new HashSet<>(); - for (StreamingTrack streamingTrack : source) { - streamingTrack.setSampleSink(this); - chunkNumbers.put(streamingTrack, 1L); - sampleNumbers.put(streamingTrack, 1L); - nextSampleStartTime.put(streamingTrack, 0L); - nextChunkCreateStartTime.put(streamingTrack, 0L); - nextChunkWriteStartTime.put(streamingTrack, 0L); - sampleBuffers.put(streamingTrack, new ArrayList<>()); - chunkBuffers.put(streamingTrack, new LinkedList<>()); - if (streamingTrack.getTrackExtension(TrackIdTrackExtension.class) != null) { - final TrackIdTrackExtension trackIdTrackExtension = streamingTrack.getTrackExtension(TrackIdTrackExtension.class); - if (trackIds.contains(trackIdTrackExtension.getTrackId())) { - throw new MuxingException("There may not be two tracks with the same trackID within one file"); - } - trackIds.add(trackIdTrackExtension.getTrackId()); - } - } - for (StreamingTrack streamingTrack : source) { - if (streamingTrack.getTrackExtension(TrackIdTrackExtension.class) == null) { - long maxTrackId = 0; - for (Long trackId : trackIds) { - maxTrackId = Math.max(trackId, maxTrackId); - } - final TrackIdTrackExtension tiExt = new TrackIdTrackExtension(maxTrackId + 1); - trackIds.add(tiExt.getTrackId()); - streamingTrack.addTrackExtension(tiExt); - } - } - - final List minorBrands = new LinkedList<>(); - minorBrands.add("isom"); - minorBrands.add("mp42"); - write(sink, new FileTypeBox("mp42", 0, minorBrands)); - } - - public void close() throws IOException { - for (StreamingTrack streamingTrack : source) { - writeChunkContainer(createChunkContainer(streamingTrack)); - streamingTrack.close(); - } - write(sink, createMoov()); - hasWrittenMdat = false; - } - - public long getTotalMdatContentLength() { - return mMDatTotalContentLength; - } - - private Box createMoov() { - final MovieBox movieBox = new MovieBox(); - - final MovieHeaderBox mvhd = createMvhd(); - movieBox.addBox(mvhd); - - // update durations - for (StreamingTrack streamingTrack : source) { - final TrackBox tb = trackBoxes.get(streamingTrack); - final MediaHeaderBox mdhd = Path.getPath(tb, "mdia[0]/mdhd[0]"); - mdhd.setCreationTime(creationTime); - mdhd.setModificationTime(creationTime); - final Long mediaHeaderDuration = Objects.requireNonNull(nextSampleStartTime.get(streamingTrack)); - if (mediaHeaderDuration >= UInt32_MAX) { - mdhd.setVersion(1); - } - mdhd.setDuration(mediaHeaderDuration); - mdhd.setTimescale(streamingTrack.getTimescale()); - mdhd.setLanguage(streamingTrack.getLanguage()); - movieBox.addBox(tb); - - final TrackHeaderBox tkhd = Path.getPath(tb, "tkhd[0]"); - final double duration = (double) mediaHeaderDuration / streamingTrack.getTimescale(); - tkhd.setCreationTime(creationTime); - tkhd.setModificationTime(creationTime); - final long trackHeaderDuration = (long) (mvhd.getTimescale() * duration); - if (trackHeaderDuration >= UInt32_MAX) { - tkhd.setVersion(1); - } - tkhd.setDuration(trackHeaderDuration); - } - - // metadata here - return movieBox; - } - - private void sortTracks() { - Collections.sort(source, (o1, o2) -> { - // compare times and account for timestamps! - final long a = Objects.requireNonNull(nextChunkWriteStartTime.get(o1)) * o2.getTimescale(); - final long b = Objects.requireNonNull(nextChunkWriteStartTime.get(o2)) * o1.getTimescale(); - return (int) Math.signum(a - b); - }); - } - - @Override - protected MovieHeaderBox createMvhd() { - final MovieHeaderBox mvhd = new MovieHeaderBox(); - mvhd.setVersion(1); - mvhd.setCreationTime(creationTime); - mvhd.setModificationTime(creationTime); - - - long[] timescales = new long[0]; - long maxTrackId = 0; - double duration = 0; - for (StreamingTrack streamingTrack : source) { - duration = Math.max((double) Objects.requireNonNull(nextSampleStartTime.get(streamingTrack)) / streamingTrack.getTimescale(), duration); - timescales = Mp4Arrays.copyOfAndAppend(timescales, streamingTrack.getTimescale()); - maxTrackId = Math.max(streamingTrack.getTrackExtension(TrackIdTrackExtension.class).getTrackId(), maxTrackId); - } - - - long chosenTimescale = Mp4Math.lcm(timescales); - Log.d(TAG, "chosenTimescale = " + chosenTimescale); - final long MAX_UNSIGNED_INT = 0xFFFFFFFFL; - if (chosenTimescale > MAX_UNSIGNED_INT) { - int nRatio = (int)(chosenTimescale / MAX_UNSIGNED_INT); - Log.d(TAG, "chosenTimescale exceeds 32-bit range " + nRatio + " times !"); - int nDownscaleFactor = 1; - if (nRatio < 10) { - nDownscaleFactor = 10; - } else if (nRatio < 100) { - nDownscaleFactor = 100; - } else if (nRatio < 1000) { - nDownscaleFactor = 1000; - } else if (nRatio < 10000) { - nDownscaleFactor = 10000; - } - chosenTimescale /= nDownscaleFactor; - Log.d(TAG, "chosenTimescale is scaled down by factor of " + nDownscaleFactor + " to value " + chosenTimescale); - } - - double fDurationTicks = chosenTimescale * duration; - Log.d(TAG, "fDurationTicks = chosenTimescale * duration = " + fDurationTicks); - final double MAX_UNSIGNED_64_BIT_VALUE = 18446744073709551615.0; - if (fDurationTicks > MAX_UNSIGNED_64_BIT_VALUE) { - // Highly unlikely, as duration (number of seconds) - // would need to be larger than MAX_UNSIGNED_INT - // to produce fDuration = chosenTimescale * duration - // which whould exceed 64-bit storage - Log.d(TAG, "Numeric overflow !!!"); - } - mvhd.setTimescale(chosenTimescale); - mvhd.setDuration((long) (fDurationTicks)); - // find the next available trackId - mvhd.setNextTrackId(maxTrackId + 1); - return mvhd; - } - - private void write(final @NonNull WritableByteChannel out, Box... boxes) throws IOException { - for (Box box1 : boxes) { - box1.getBox(out); - bytesWritten += box1.getSize(); - } - } - - /** - * Tests if the currently received samples for a given track - * are already a 'chunk' as we want to have it. The next - * sample will not be part of the chunk - * will be added to the fragment buffer later. - * - * @param streamingTrack track to test - * @param next the lastest samples - * @return true if a chunk is to b e created. - */ - private boolean isChunkReady(StreamingTrack streamingTrack, StreamingSample next) { - final long ts = Objects.requireNonNull(nextSampleStartTime.get(streamingTrack)); - final long cfst = Objects.requireNonNull(nextChunkCreateStartTime.get(streamingTrack)); - - return (ts >= cfst + 2 * streamingTrack.getTimescale()); - // chunk interleave of 2 seconds - } - - private void writeChunkContainer(ChunkContainer chunkContainer) throws IOException { - final TrackBox tb = trackBoxes.get(chunkContainer.streamingTrack); - final ChunkOffsetBox stco = Objects.requireNonNull(Path.getPath(tb, "mdia[0]/minf[0]/stbl[0]/stco[0]")); - final int extraChunkOffset = hasWrittenMdat ? 0 : 8; - stco.setChunkOffsets(Mp4Arrays.copyOfAndAppend(stco.getChunkOffsets(), bytesWritten + extraChunkOffset)); - chunkContainer.mdat.includeHeader = !hasWrittenMdat; - write(sink, chunkContainer.mdat); - - mMDatTotalContentLength += chunkContainer.mdat.getSize(); - - if (!hasWrittenMdat) { - hasWrittenMdat = true; - } - } - - public void acceptSample( - final @NonNull StreamingSample streamingSample, - final @NonNull StreamingTrack streamingTrack) throws IOException - { - if (streamingSample.getContent().limit() == 0) { - // - // For currently unknown reason, the STSZ table of AAC audio stream - // related to the very last chunk comes with the extra table elements - // whose value is zero. - // - // The ISO MP4 spec does not absolutely prohibit such a case, but strongly - // stipulates that the stream has to have the inner logic to support - // the zero length audio frames (QCELP happens to be one such example). - // - // Spec excerpt: - // ---------------------------------------------------------------------- - // 8.7.3 Sample Size Boxes - // 8.7.3.1 Definition - // ... - // NOTE A sample size of zero is not prohibited in general, but it - // must be valid and defined for the coding system, as defined by - // the sample entry, that the sample belongs to - // ---------------------------------------------------------------------- - // - // In all other cases, having zero STSZ table values is very illogical - // and may pose the problems down the road. Here we will eliminate such - // samples from all the related bookkeeping - // - Log.d(TAG, "skipping zero-sized sample"); - return; - } - TrackBox tb = trackBoxes.get(streamingTrack); - if (tb == null) { - tb = new TrackBox(); - tb.addBox(createTkhd(streamingTrack)); - tb.addBox(createMdia(streamingTrack)); - trackBoxes.put(streamingTrack, tb); - } - - if (isChunkReady(streamingTrack, streamingSample)) { - - final ChunkContainer chunkContainer = createChunkContainer(streamingTrack); - //System.err.println("Creating fragment for " + streamingTrack); - Objects.requireNonNull(sampleBuffers.get(streamingTrack)).clear(); - nextChunkCreateStartTime.put(streamingTrack, Objects.requireNonNull(nextChunkCreateStartTime.get(streamingTrack)) + chunkContainer.duration); - final Queue chunkQueue = Objects.requireNonNull(chunkBuffers.get(streamingTrack)); - chunkQueue.add(chunkContainer); - if (source.get(0) == streamingTrack) { - - Queue tracksFragmentQueue; - StreamingTrack currentStreamingTrack; - // This will write AT LEAST the currently created fragment and possibly a few more - while (!(tracksFragmentQueue = chunkBuffers.get((currentStreamingTrack = this.source.get(0)))).isEmpty()) { - final ChunkContainer currentFragmentContainer = tracksFragmentQueue.remove(); - writeChunkContainer(currentFragmentContainer); - Log.d(TAG, "write chunk " + currentStreamingTrack.getHandler() + ". duration " + (double) currentFragmentContainer.duration / currentStreamingTrack.getTimescale()); - final long ts = Objects.requireNonNull(nextChunkWriteStartTime.get(currentStreamingTrack)) + currentFragmentContainer.duration; - nextChunkWriteStartTime.put(currentStreamingTrack, ts); - Log.d(TAG, currentStreamingTrack.getHandler() + " track advanced to " + (double) ts / currentStreamingTrack.getTimescale()); - sortTracks(); - } - } else { - Log.d(TAG, streamingTrack.getHandler() + " track delayed, queue size is " + chunkQueue.size()); - } - } - - Objects.requireNonNull(sampleBuffers.get(streamingTrack)).add(streamingSample); - nextSampleStartTime.put(streamingTrack, Objects.requireNonNull(nextSampleStartTime.get(streamingTrack)) + streamingSample.getDuration()); - - } - - private ChunkContainer createChunkContainer(final @NonNull StreamingTrack streamingTrack) { - - final List samples = Objects.requireNonNull(sampleBuffers.get(streamingTrack)); - final long chunkNumber = Objects.requireNonNull(chunkNumbers.get(streamingTrack)); - chunkNumbers.put(streamingTrack, chunkNumber + 1); - final ChunkContainer cc = new ChunkContainer(); - cc.streamingTrack = streamingTrack; - cc.mdat = new Mdat(samples); - cc.duration = Objects.requireNonNull(nextSampleStartTime.get(streamingTrack)) - Objects.requireNonNull(nextChunkCreateStartTime.get(streamingTrack)); - final TrackBox tb = trackBoxes.get(streamingTrack); - final SampleTableBox stbl = Objects.requireNonNull(Path.getPath(tb, "mdia[0]/minf[0]/stbl[0]")); - final SampleToChunkBox stsc = Objects.requireNonNull(Path.getPath(stbl, "stsc[0]")); - if (stsc.getEntries().isEmpty()) { - final List entries = new ArrayList<>(); - stsc.setEntries(entries); - entries.add(new SampleToChunkBox.Entry(chunkNumber, samples.size(), 1)); - } else { - final SampleToChunkBox.Entry e = stsc.getEntries().get(stsc.getEntries().size() - 1); - if (e.getSamplesPerChunk() != samples.size()) { - stsc.getEntries().add(new SampleToChunkBox.Entry(chunkNumber, samples.size(), 1)); - } - } - long sampleNumber = Objects.requireNonNull(sampleNumbers.get(streamingTrack)); - - final SampleSizeBox stsz = Objects.requireNonNull(Path.getPath(stbl, "stsz[0]")); - final TimeToSampleBox stts = Objects.requireNonNull(Path.getPath(stbl, "stts[0]")); - SyncSampleBox stss = Path.getPath(stbl, "stss[0]"); - CompositionTimeToSample ctts = Path.getPath(stbl, "ctts[0]"); - if (streamingTrack.getTrackExtension(CompositionTimeTrackExtension.class) != null) { - if (ctts == null) { - ctts = new CompositionTimeToSample(); - ctts.setEntries(new ArrayList<>()); - - final ArrayList bs = new ArrayList<>(stbl.getBoxes()); - bs.add(bs.indexOf(stts), ctts); - } - } - - final long[] sampleSizes = new long[samples.size()]; - int i = 0; - for (StreamingSample sample : samples) { - sampleSizes[i++] = sample.getContent().limit(); - - if (ctts != null) { - ctts.getEntries().add(new CompositionTimeToSample.Entry(1, l2i(sample.getSampleExtension(CompositionTimeSampleExtension.class).getCompositionTimeOffset()))); - } - - if (stts.getEntries().isEmpty()) { - final ArrayList entries = new ArrayList<>(stts.getEntries()); - entries.add(new TimeToSampleBox.Entry(1, sample.getDuration())); - stts.setEntries(entries); - } else { - final TimeToSampleBox.Entry sttsEntry = stts.getEntries().get(stts.getEntries().size() - 1); - if (sttsEntry.getDelta() == sample.getDuration()) { - sttsEntry.setCount(sttsEntry.getCount() + 1); - } else { - stts.getEntries().add(new TimeToSampleBox.Entry(1, sample.getDuration())); - } - } - final SampleFlagsSampleExtension sampleFlagsSampleExtension = sample.getSampleExtension(SampleFlagsSampleExtension.class); - if (sampleFlagsSampleExtension != null && sampleFlagsSampleExtension.isSyncSample()) { - if (stss == null) { - stss = new SyncSampleBox(); - stbl.addBox(stss); - } - stss.setSampleNumber(Mp4Arrays.copyOfAndAppend(stss.getSampleNumber(), sampleNumber)); - } - sampleNumber++; - - } - stsz.setSampleSizes(Mp4Arrays.copyOfAndAppend(stsz.getSampleSizes(), sampleSizes)); - - sampleNumbers.put(streamingTrack, sampleNumber); - samples.clear(); - Log.d(TAG, "chunk container created for " + streamingTrack.getHandler() + ". mdat size: " + cc.mdat.size + ". chunk duration is " + (double) cc.duration / streamingTrack.getTimescale()); - return cc; - } - - protected @NonNull Box createMdhd(final @NonNull StreamingTrack streamingTrack) { - final MediaHeaderBox mdhd = new MediaHeaderBox(); - mdhd.setCreationTime(creationTime); - mdhd.setModificationTime(creationTime); - //mdhd.setDuration(nextSampleStartTime.get(streamingTrack)); will update at the end, in createMoov - mdhd.setTimescale(streamingTrack.getTimescale()); - mdhd.setLanguage(streamingTrack.getLanguage()); - return mdhd; - } - - @Override - protected Box createTkhd(StreamingTrack streamingTrack) { - TrackHeaderBox tkhd = (TrackHeaderBox) super.createTkhd(streamingTrack); - tkhd.setEnabled(true); - tkhd.setInMovie(true); - return tkhd; - } - - private class Mdat implements Box { - final ArrayList samples; - long size; - - boolean includeHeader; - - Mdat(final @NonNull List samples) { - this.samples = new ArrayList<>(samples); - size = 8; - - for (StreamingSample sample : samples) { - size += sample.getContent().limit(); - } - } - - @Override - public String getType() { - return "mdat"; - } - - @Override - public long getSize() { - if (includeHeader) { - return size; - } else { - return size - 8; - } - } - - @Override - public void getBox(WritableByteChannel writableByteChannel) throws IOException { - if (includeHeader) { - // When we include the header, we specify the declared size as 1, indicating the size is from here until the end of the file - writableByteChannel.write(ByteBuffer.wrap(new byte[] { - 0, 0, 0, 0, // size (4 bytes) - 109, 100, 97, 116, // 'm' 'd' 'a' 't' - })); - } - - for (StreamingSample sample : samples) { - writableByteChannel.write((ByteBuffer) sample.getContent().rewind()); - } - } - } - - private class ChunkContainer { - Mdat mdat; - StreamingTrack streamingTrack; - long duration; - } -} +/* + * Copyright (C) https://github.com/sannies/mp4parser/blob/master/LICENSE + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * https://github.com/sannies/mp4parser/blob/4ed724754cde751c3f27fdda51f288df4f4c5db5/streaming/src/main/java/org/mp4parser/streaming/output/mp4/StandardMp4Writer.java + * + * This file has been modified by Signal. + */ +package org.thoughtcrime.securesms.video.videoconverter.muxer; + +import androidx.annotation.NonNull; + +import org.mp4parser.Box; +import org.mp4parser.boxes.iso14496.part12.ChunkOffsetBox; +import org.mp4parser.boxes.iso14496.part12.CompositionTimeToSample; +import org.mp4parser.boxes.iso14496.part12.FileTypeBox; +import org.mp4parser.boxes.iso14496.part12.MediaHeaderBox; +import org.mp4parser.boxes.iso14496.part12.MovieBox; +import org.mp4parser.boxes.iso14496.part12.MovieHeaderBox; +import org.mp4parser.boxes.iso14496.part12.SampleSizeBox; +import org.mp4parser.boxes.iso14496.part12.SampleTableBox; +import org.mp4parser.boxes.iso14496.part12.SampleToChunkBox; +import org.mp4parser.boxes.iso14496.part12.SyncSampleBox; +import org.mp4parser.boxes.iso14496.part12.TimeToSampleBox; +import org.mp4parser.boxes.iso14496.part12.TrackBox; +import org.mp4parser.boxes.iso14496.part12.TrackHeaderBox; +import org.mp4parser.streaming.StreamingSample; +import org.mp4parser.streaming.StreamingTrack; +import org.mp4parser.streaming.extensions.CompositionTimeSampleExtension; +import org.mp4parser.streaming.extensions.CompositionTimeTrackExtension; +import org.mp4parser.streaming.extensions.SampleFlagsSampleExtension; +import org.mp4parser.streaming.extensions.TrackIdTrackExtension; +import org.mp4parser.streaming.output.SampleSink; +import org.mp4parser.streaming.output.mp4.DefaultBoxes; +import org.mp4parser.tools.Mp4Arrays; +import org.mp4parser.tools.Mp4Math; +import org.mp4parser.tools.Path; +import org.signal.core.util.logging.Log; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.WritableByteChannel; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Queue; +import java.util.concurrent.ConcurrentHashMap; + +import static org.mp4parser.tools.CastUtils.l2i; + +/** + * Creates an MP4 file with ftyp, mdat+, moov order. + * A very special property of this variant is that it written sequentially. You can start transferring the + * data while the sink receives it. (in contrast to typical implementations which need random + * access to write length fields at the beginning of the file) + */ +final class Mp4Writer extends DefaultBoxes implements SampleSink { + + private static final String TAG = "Mp4Writer"; + private static final Long UInt32_MAX = (1L << 32) - 1; + + private final WritableByteChannel sink; + private final List source; + private final Date creationTime = new Date(); + + private boolean hasWrittenMdat = false; + + /** + * Contains the start time of the next segment in line that will be created. + */ + private final Map nextChunkCreateStartTime = new ConcurrentHashMap<>(); + /** + * Contains the start time of the next segment in line that will be written. + */ + private final Map nextChunkWriteStartTime = new ConcurrentHashMap<>(); + /** + * Contains the next sample's start time. + */ + private final Map nextSampleStartTime = new HashMap<>(); + /** + * Buffers the samples per track until there are enough samples to form a Segment. + */ + private final Map> sampleBuffers = new HashMap<>(); + private final Map trackBoxes = new HashMap<>(); + /** + * Buffers segments until it's time for a segment to be written. + */ + private final Map> chunkBuffers = new ConcurrentHashMap<>(); + private final Map chunkNumbers = new HashMap<>(); + private final Map sampleNumbers = new HashMap<>(); + private long bytesWritten = 0; + + private long mMDatTotalContentLength = 0; + + Mp4Writer(final @NonNull List source, final @NonNull WritableByteChannel sink) throws IOException { + this.source = new ArrayList<>(source); + this.sink = sink; + + final HashSet trackIds = new HashSet<>(); + for (StreamingTrack streamingTrack : source) { + streamingTrack.setSampleSink(this); + chunkNumbers.put(streamingTrack, 1L); + sampleNumbers.put(streamingTrack, 1L); + nextSampleStartTime.put(streamingTrack, 0L); + nextChunkCreateStartTime.put(streamingTrack, 0L); + nextChunkWriteStartTime.put(streamingTrack, 0L); + sampleBuffers.put(streamingTrack, new ArrayList<>()); + chunkBuffers.put(streamingTrack, new LinkedList<>()); + if (streamingTrack.getTrackExtension(TrackIdTrackExtension.class) != null) { + final TrackIdTrackExtension trackIdTrackExtension = streamingTrack.getTrackExtension(TrackIdTrackExtension.class); + if (trackIds.contains(trackIdTrackExtension.getTrackId())) { + throw new MuxingException("There may not be two tracks with the same trackID within one file"); + } + trackIds.add(trackIdTrackExtension.getTrackId()); + } + } + for (StreamingTrack streamingTrack : source) { + if (streamingTrack.getTrackExtension(TrackIdTrackExtension.class) == null) { + long maxTrackId = 0; + for (Long trackId : trackIds) { + maxTrackId = Math.max(trackId, maxTrackId); + } + final TrackIdTrackExtension tiExt = new TrackIdTrackExtension(maxTrackId + 1); + trackIds.add(tiExt.getTrackId()); + streamingTrack.addTrackExtension(tiExt); + } + } + + final List minorBrands = new LinkedList<>(); + minorBrands.add("isom"); + minorBrands.add("mp42"); + write(sink, new FileTypeBox("mp42", 0, minorBrands)); + } + + public void close() throws IOException { + for (StreamingTrack streamingTrack : source) { + writeChunkContainer(createChunkContainer(streamingTrack)); + streamingTrack.close(); + } + write(sink, createMoov()); + hasWrittenMdat = false; + } + + public long getTotalMdatContentLength() { + return mMDatTotalContentLength; + } + + private Box createMoov() { + final MovieBox movieBox = new MovieBox(); + + final MovieHeaderBox mvhd = createMvhd(); + movieBox.addBox(mvhd); + + // update durations + for (StreamingTrack streamingTrack : source) { + final TrackBox tb = trackBoxes.get(streamingTrack); + final MediaHeaderBox mdhd = Path.getPath(tb, "mdia[0]/mdhd[0]"); + mdhd.setCreationTime(creationTime); + mdhd.setModificationTime(creationTime); + final Long mediaHeaderDuration = Objects.requireNonNull(nextSampleStartTime.get(streamingTrack)); + if (mediaHeaderDuration >= UInt32_MAX) { + mdhd.setVersion(1); + } + mdhd.setDuration(mediaHeaderDuration); + mdhd.setTimescale(streamingTrack.getTimescale()); + mdhd.setLanguage(streamingTrack.getLanguage()); + movieBox.addBox(tb); + + final TrackHeaderBox tkhd = Path.getPath(tb, "tkhd[0]"); + final double duration = (double) mediaHeaderDuration / streamingTrack.getTimescale(); + tkhd.setCreationTime(creationTime); + tkhd.setModificationTime(creationTime); + final long trackHeaderDuration = (long) (mvhd.getTimescale() * duration); + if (trackHeaderDuration >= UInt32_MAX) { + tkhd.setVersion(1); + } + tkhd.setDuration(trackHeaderDuration); + } + + // metadata here + return movieBox; + } + + private void sortTracks() { + Collections.sort(source, (o1, o2) -> { + // compare times and account for timestamps! + final long a = Objects.requireNonNull(nextChunkWriteStartTime.get(o1)) * o2.getTimescale(); + final long b = Objects.requireNonNull(nextChunkWriteStartTime.get(o2)) * o1.getTimescale(); + return (int) Math.signum(a - b); + }); + } + + @Override + protected MovieHeaderBox createMvhd() { + final MovieHeaderBox mvhd = new MovieHeaderBox(); + mvhd.setVersion(1); + mvhd.setCreationTime(creationTime); + mvhd.setModificationTime(creationTime); + + + long[] timescales = new long[0]; + long maxTrackId = 0; + double duration = 0; + for (StreamingTrack streamingTrack : source) { + duration = Math.max((double) Objects.requireNonNull(nextSampleStartTime.get(streamingTrack)) / streamingTrack.getTimescale(), duration); + timescales = Mp4Arrays.copyOfAndAppend(timescales, streamingTrack.getTimescale()); + maxTrackId = Math.max(streamingTrack.getTrackExtension(TrackIdTrackExtension.class).getTrackId(), maxTrackId); + } + + + long chosenTimescale = Mp4Math.lcm(timescales); + Log.d(TAG, "chosenTimescale = " + chosenTimescale); + final long MAX_UNSIGNED_INT = 0xFFFFFFFFL; + if (chosenTimescale > MAX_UNSIGNED_INT) { + int nRatio = (int)(chosenTimescale / MAX_UNSIGNED_INT); + Log.d(TAG, "chosenTimescale exceeds 32-bit range " + nRatio + " times !"); + int nDownscaleFactor = 1; + if (nRatio < 10) { + nDownscaleFactor = 10; + } else if (nRatio < 100) { + nDownscaleFactor = 100; + } else if (nRatio < 1000) { + nDownscaleFactor = 1000; + } else if (nRatio < 10000) { + nDownscaleFactor = 10000; + } + chosenTimescale /= nDownscaleFactor; + Log.d(TAG, "chosenTimescale is scaled down by factor of " + nDownscaleFactor + " to value " + chosenTimescale); + } + + double fDurationTicks = chosenTimescale * duration; + Log.d(TAG, "fDurationTicks = chosenTimescale * duration = " + fDurationTicks); + final double MAX_UNSIGNED_64_BIT_VALUE = 18446744073709551615.0; + if (fDurationTicks > MAX_UNSIGNED_64_BIT_VALUE) { + // Highly unlikely, as duration (number of seconds) + // would need to be larger than MAX_UNSIGNED_INT + // to produce fDuration = chosenTimescale * duration + // which whould exceed 64-bit storage + Log.d(TAG, "Numeric overflow !!!"); + } + mvhd.setTimescale(chosenTimescale); + mvhd.setDuration((long) (fDurationTicks)); + // find the next available trackId + mvhd.setNextTrackId(maxTrackId + 1); + return mvhd; + } + + private void write(final @NonNull WritableByteChannel out, Box... boxes) throws IOException { + for (Box box1 : boxes) { + box1.getBox(out); + bytesWritten += box1.getSize(); + } + } + + /** + * Tests if the currently received samples for a given track + * are already a 'chunk' as we want to have it. The next + * sample will not be part of the chunk + * will be added to the fragment buffer later. + * + * @param streamingTrack track to test + * @param next the lastest samples + * @return true if a chunk is to b e created. + */ + private boolean isChunkReady(StreamingTrack streamingTrack, StreamingSample next) { + final long ts = Objects.requireNonNull(nextSampleStartTime.get(streamingTrack)); + final long cfst = Objects.requireNonNull(nextChunkCreateStartTime.get(streamingTrack)); + + return (ts >= cfst + 2 * streamingTrack.getTimescale()); + // chunk interleave of 2 seconds + } + + private void writeChunkContainer(ChunkContainer chunkContainer) throws IOException { + final TrackBox tb = trackBoxes.get(chunkContainer.streamingTrack); + final ChunkOffsetBox stco = Objects.requireNonNull(Path.getPath(tb, "mdia[0]/minf[0]/stbl[0]/stco[0]")); + final int extraChunkOffset = hasWrittenMdat ? 0 : 8; + stco.setChunkOffsets(Mp4Arrays.copyOfAndAppend(stco.getChunkOffsets(), bytesWritten + extraChunkOffset)); + chunkContainer.mdat.includeHeader = !hasWrittenMdat; + write(sink, chunkContainer.mdat); + + mMDatTotalContentLength += chunkContainer.mdat.getSize(); + + if (!hasWrittenMdat) { + hasWrittenMdat = true; + } + } + + public void acceptSample( + final @NonNull StreamingSample streamingSample, + final @NonNull StreamingTrack streamingTrack) throws IOException + { + if (streamingSample.getContent().limit() == 0) { + // + // For currently unknown reason, the STSZ table of AAC audio stream + // related to the very last chunk comes with the extra table elements + // whose value is zero. + // + // The ISO MP4 spec does not absolutely prohibit such a case, but strongly + // stipulates that the stream has to have the inner logic to support + // the zero length audio frames (QCELP happens to be one such example). + // + // Spec excerpt: + // ---------------------------------------------------------------------- + // 8.7.3 Sample Size Boxes + // 8.7.3.1 Definition + // ... + // NOTE A sample size of zero is not prohibited in general, but it + // must be valid and defined for the coding system, as defined by + // the sample entry, that the sample belongs to + // ---------------------------------------------------------------------- + // + // In all other cases, having zero STSZ table values is very illogical + // and may pose the problems down the road. Here we will eliminate such + // samples from all the related bookkeeping + // + Log.d(TAG, "skipping zero-sized sample"); + return; + } + TrackBox tb = trackBoxes.get(streamingTrack); + if (tb == null) { + tb = new TrackBox(); + tb.addBox(createTkhd(streamingTrack)); + tb.addBox(createMdia(streamingTrack)); + trackBoxes.put(streamingTrack, tb); + } + + if (isChunkReady(streamingTrack, streamingSample)) { + + final ChunkContainer chunkContainer = createChunkContainer(streamingTrack); + //System.err.println("Creating fragment for " + streamingTrack); + Objects.requireNonNull(sampleBuffers.get(streamingTrack)).clear(); + nextChunkCreateStartTime.put(streamingTrack, Objects.requireNonNull(nextChunkCreateStartTime.get(streamingTrack)) + chunkContainer.duration); + final Queue chunkQueue = Objects.requireNonNull(chunkBuffers.get(streamingTrack)); + chunkQueue.add(chunkContainer); + if (source.get(0) == streamingTrack) { + + Queue tracksFragmentQueue; + StreamingTrack currentStreamingTrack; + // This will write AT LEAST the currently created fragment and possibly a few more + while (!(tracksFragmentQueue = chunkBuffers.get((currentStreamingTrack = this.source.get(0)))).isEmpty()) { + final ChunkContainer currentFragmentContainer = tracksFragmentQueue.remove(); + writeChunkContainer(currentFragmentContainer); + Log.d(TAG, "write chunk " + currentStreamingTrack.getHandler() + ". duration " + (double) currentFragmentContainer.duration / currentStreamingTrack.getTimescale()); + final long ts = Objects.requireNonNull(nextChunkWriteStartTime.get(currentStreamingTrack)) + currentFragmentContainer.duration; + nextChunkWriteStartTime.put(currentStreamingTrack, ts); + Log.d(TAG, currentStreamingTrack.getHandler() + " track advanced to " + (double) ts / currentStreamingTrack.getTimescale()); + sortTracks(); + } + } else { + Log.d(TAG, streamingTrack.getHandler() + " track delayed, queue size is " + chunkQueue.size()); + } + } + + Objects.requireNonNull(sampleBuffers.get(streamingTrack)).add(streamingSample); + nextSampleStartTime.put(streamingTrack, Objects.requireNonNull(nextSampleStartTime.get(streamingTrack)) + streamingSample.getDuration()); + + } + + private ChunkContainer createChunkContainer(final @NonNull StreamingTrack streamingTrack) { + + final List samples = Objects.requireNonNull(sampleBuffers.get(streamingTrack)); + final long chunkNumber = Objects.requireNonNull(chunkNumbers.get(streamingTrack)); + chunkNumbers.put(streamingTrack, chunkNumber + 1); + final ChunkContainer cc = new ChunkContainer(); + cc.streamingTrack = streamingTrack; + cc.mdat = new Mdat(samples); + cc.duration = Objects.requireNonNull(nextSampleStartTime.get(streamingTrack)) - Objects.requireNonNull(nextChunkCreateStartTime.get(streamingTrack)); + final TrackBox tb = trackBoxes.get(streamingTrack); + final SampleTableBox stbl = Objects.requireNonNull(Path.getPath(tb, "mdia[0]/minf[0]/stbl[0]")); + final SampleToChunkBox stsc = Objects.requireNonNull(Path.getPath(stbl, "stsc[0]")); + if (stsc.getEntries().isEmpty()) { + final List entries = new ArrayList<>(); + stsc.setEntries(entries); + entries.add(new SampleToChunkBox.Entry(chunkNumber, samples.size(), 1)); + } else { + final SampleToChunkBox.Entry e = stsc.getEntries().get(stsc.getEntries().size() - 1); + if (e.getSamplesPerChunk() != samples.size()) { + stsc.getEntries().add(new SampleToChunkBox.Entry(chunkNumber, samples.size(), 1)); + } + } + long sampleNumber = Objects.requireNonNull(sampleNumbers.get(streamingTrack)); + + final SampleSizeBox stsz = Objects.requireNonNull(Path.getPath(stbl, "stsz[0]")); + final TimeToSampleBox stts = Objects.requireNonNull(Path.getPath(stbl, "stts[0]")); + SyncSampleBox stss = Path.getPath(stbl, "stss[0]"); + CompositionTimeToSample ctts = Path.getPath(stbl, "ctts[0]"); + if (streamingTrack.getTrackExtension(CompositionTimeTrackExtension.class) != null) { + if (ctts == null) { + ctts = new CompositionTimeToSample(); + ctts.setEntries(new ArrayList<>()); + + final ArrayList bs = new ArrayList<>(stbl.getBoxes()); + bs.add(bs.indexOf(stts), ctts); + } + } + + final long[] sampleSizes = new long[samples.size()]; + int i = 0; + for (StreamingSample sample : samples) { + sampleSizes[i++] = sample.getContent().limit(); + + if (ctts != null) { + ctts.getEntries().add(new CompositionTimeToSample.Entry(1, l2i(sample.getSampleExtension(CompositionTimeSampleExtension.class).getCompositionTimeOffset()))); + } + + if (stts.getEntries().isEmpty()) { + final ArrayList entries = new ArrayList<>(stts.getEntries()); + entries.add(new TimeToSampleBox.Entry(1, sample.getDuration())); + stts.setEntries(entries); + } else { + final TimeToSampleBox.Entry sttsEntry = stts.getEntries().get(stts.getEntries().size() - 1); + if (sttsEntry.getDelta() == sample.getDuration()) { + sttsEntry.setCount(sttsEntry.getCount() + 1); + } else { + stts.getEntries().add(new TimeToSampleBox.Entry(1, sample.getDuration())); + } + } + final SampleFlagsSampleExtension sampleFlagsSampleExtension = sample.getSampleExtension(SampleFlagsSampleExtension.class); + if (sampleFlagsSampleExtension != null && sampleFlagsSampleExtension.isSyncSample()) { + if (stss == null) { + stss = new SyncSampleBox(); + stbl.addBox(stss); + } + stss.setSampleNumber(Mp4Arrays.copyOfAndAppend(stss.getSampleNumber(), sampleNumber)); + } + sampleNumber++; + + } + stsz.setSampleSizes(Mp4Arrays.copyOfAndAppend(stsz.getSampleSizes(), sampleSizes)); + + sampleNumbers.put(streamingTrack, sampleNumber); + samples.clear(); + Log.d(TAG, "chunk container created for " + streamingTrack.getHandler() + ". mdat size: " + cc.mdat.size + ". chunk duration is " + (double) cc.duration / streamingTrack.getTimescale()); + return cc; + } + + protected @NonNull Box createMdhd(final @NonNull StreamingTrack streamingTrack) { + final MediaHeaderBox mdhd = new MediaHeaderBox(); + mdhd.setCreationTime(creationTime); + mdhd.setModificationTime(creationTime); + //mdhd.setDuration(nextSampleStartTime.get(streamingTrack)); will update at the end, in createMoov + mdhd.setTimescale(streamingTrack.getTimescale()); + mdhd.setLanguage(streamingTrack.getLanguage()); + return mdhd; + } + + @Override + protected Box createTkhd(StreamingTrack streamingTrack) { + TrackHeaderBox tkhd = (TrackHeaderBox) super.createTkhd(streamingTrack); + tkhd.setEnabled(true); + tkhd.setInMovie(true); + return tkhd; + } + + private class Mdat implements Box { + final ArrayList samples; + long size; + + boolean includeHeader; + + Mdat(final @NonNull List samples) { + this.samples = new ArrayList<>(samples); + size = 8; + + for (StreamingSample sample : samples) { + size += sample.getContent().limit(); + } + } + + @Override + public String getType() { + return "mdat"; + } + + @Override + public long getSize() { + if (includeHeader) { + return size; + } else { + return size - 8; + } + } + + @Override + public void getBox(WritableByteChannel writableByteChannel) throws IOException { + if (includeHeader) { + // When we include the header, we specify the declared size as 1, indicating the size is from here until the end of the file + writableByteChannel.write(ByteBuffer.wrap(new byte[] { + 0, 0, 0, 0, // size (4 bytes) + 109, 100, 97, 116, // 'm' 'd' 'a' 't' + })); + } + + for (StreamingSample sample : samples) { + writableByteChannel.write((ByteBuffer) sample.getContent().rewind()); + } + } + } + + private class ChunkContainer { + Mdat mdat; + StreamingTrack streamingTrack; + long duration; + } +} diff --git a/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/muxer/MuxingException.java b/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/muxer/MuxingException.java index b72a002b03..ed2d161ce7 100644 --- a/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/muxer/MuxingException.java +++ b/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/muxer/MuxingException.java @@ -1,12 +1,12 @@ -package org.thoughtcrime.securesms.video.videoconverter.muxer; - -final class MuxingException extends RuntimeException { - - public MuxingException(String message) { - super(message); - } - - public MuxingException(String message, Throwable cause) { - super(message, cause); - } -} +package org.thoughtcrime.securesms.video.videoconverter.muxer; + +final class MuxingException extends RuntimeException { + + public MuxingException(String message) { + super(message); + } + + public MuxingException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/muxer/StreamingMuxer.java b/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/muxer/StreamingMuxer.java index b3f339dddb..828fc8a2e1 100644 --- a/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/muxer/StreamingMuxer.java +++ b/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/muxer/StreamingMuxer.java @@ -1,190 +1,190 @@ -package org.thoughtcrime.securesms.video.videoconverter.muxer; - -import android.media.MediaCodec; -import android.media.MediaFormat; - -import androidx.annotation.NonNull; -import androidx.annotation.Nullable; - -import org.mp4parser.boxes.iso14496.part1.objectdescriptors.DecoderSpecificInfo; -import org.mp4parser.streaming.StreamingTrack; -import org.signal.core.util.logging.Log; -import org.thoughtcrime.securesms.video.interfaces.Muxer; -import org.thoughtcrime.securesms.video.videoconverter.utils.MediaCodecCompat; - -import java.io.IOException; -import java.io.OutputStream; -import java.nio.ByteBuffer; -import java.nio.channels.Channels; -import java.util.ArrayList; -import java.util.List; - -public final class StreamingMuxer implements Muxer { - private static final String TAG = Log.tag(StreamingMuxer.class); - private final OutputStream outputStream; - private final List tracks = new ArrayList<>(); - private Mp4Writer mp4Writer; - - public StreamingMuxer(OutputStream outputStream) { - this.outputStream = outputStream; - } - - @Override - public void start() throws IOException { - final List source = new ArrayList<>(); - for (MediaCodecTrack track : tracks) { - source.add((StreamingTrack) track); - } - mp4Writer = new Mp4Writer(source, Channels.newChannel(outputStream)); - } - - @Override - public long stop() throws IOException { - if (mp4Writer == null) { - throw new IllegalStateException("calling stop prior to start"); - } - for (MediaCodecTrack track : tracks) { - track.finish(); - } - mp4Writer.close(); - long mdatLength = mp4Writer.getTotalMdatContentLength(); - - mp4Writer = null; - - return mdatLength; - } - - @Override - public int addTrack(@NonNull MediaFormat format) throws IOException { - - final String mime = format.getString(MediaFormat.KEY_MIME); - switch (mime) { - case "video/avc": - tracks.add(new MediaCodecAvcTrack(format)); - break; - case "audio/mp4a-latm": - tracks.add(MediaCodecAacTrack.create(format)); - break; - case "video/hevc": - tracks.add(new MediaCodecHevcTrack(format)); - break; - default: - throw new IllegalArgumentException("unknown track format"); - } - return tracks.size() - 1; - } - - @Override - public void writeSampleData(int trackIndex, @NonNull ByteBuffer byteBuf, @NonNull MediaCodec.BufferInfo bufferInfo) throws IOException { - tracks.get(trackIndex).writeSampleData(byteBuf, bufferInfo); - } - - @Override - public void release() { - } - - @Override - public boolean supportsAudioRemux() { - return true; - } - - interface MediaCodecTrack { - void writeSampleData(@NonNull ByteBuffer byteBuf, @NonNull MediaCodec.BufferInfo bufferInfo) throws IOException; - - void finish() throws IOException; - } - - static class MediaCodecAvcTrack extends AvcTrack implements MediaCodecTrack { - - MediaCodecAvcTrack(@NonNull MediaFormat format) { - super(Utils.subBuffer(format.getByteBuffer("csd-0"), 4), Utils.subBuffer(format.getByteBuffer("csd-1"), 4)); - } - - @Override - public void writeSampleData(@NonNull ByteBuffer byteBuf, @NonNull MediaCodec.BufferInfo bufferInfo) throws IOException { - final List nals = H264Utils.getNals(byteBuf); - for (ByteBuffer nal : nals) { - consumeNal(Utils.clone(nal), bufferInfo.presentationTimeUs); - } - } - - @Override - public void finish() throws IOException { - consumeLastNal(); - } - } - - static class MediaCodecHevcTrack extends HevcTrack implements MediaCodecTrack { - - MediaCodecHevcTrack(@NonNull MediaFormat format) throws IOException { - super(H264Utils.getNals(format.getByteBuffer("csd-0"))); - } - - @Override - public void writeSampleData(@NonNull ByteBuffer byteBuf, @NonNull MediaCodec.BufferInfo bufferInfo) throws IOException { - final List nals = H264Utils.getNals(byteBuf); - for (ByteBuffer nal : nals) { - consumeNal(Utils.clone(nal), bufferInfo.presentationTimeUs); - } - } - - @Override - public void finish() throws IOException { - consumeLastNal(); - } - } - - static class MediaCodecAacTrack extends AacTrack implements MediaCodecTrack { - - private MediaCodecAacTrack(long avgBitrate, long maxBitrate, int sampleRate, int channelCount, int aacProfile, @Nullable DecoderSpecificInfo decoderSpecificInfo) { - super(avgBitrate, maxBitrate, sampleRate, channelCount, aacProfile, decoderSpecificInfo); - } - - public static MediaCodecAacTrack create(@NonNull MediaFormat format) { - final int bitrate = format.getInteger(MediaFormat.KEY_BIT_RATE); - final int maxBitrate; - if (format.containsKey(MediaCodecCompat.MEDIA_FORMAT_KEY_MAX_BIT_RATE)) { - maxBitrate = format.getInteger(MediaCodecCompat.MEDIA_FORMAT_KEY_MAX_BIT_RATE); - } else { - maxBitrate = bitrate; - } - - final DecoderSpecificInfo filledDecoderSpecificInfo; - if (format.containsKey(MediaCodecCompat.MEDIA_FORMAT_KEY_MAX_BIT_RATE)) { - final ByteBuffer csd = format.getByteBuffer(MediaCodecCompat.MEDIA_FORMAT_KEY_CODEC_SPECIFIC_DATA_0); - - DecoderSpecificInfo decoderSpecificInfo = new DecoderSpecificInfo(); - boolean parseSuccess = false; - try { - decoderSpecificInfo.parseDetail(csd); - parseSuccess = true; - } catch (IOException e) { - Log.w(TAG, "Could not parse AAC codec-specific data!", e); - } - if (parseSuccess) { - filledDecoderSpecificInfo = decoderSpecificInfo; - } else { - filledDecoderSpecificInfo = null; - } - } else { - filledDecoderSpecificInfo = null; - } - - return new MediaCodecAacTrack(bitrate, maxBitrate, - format.getInteger(MediaFormat.KEY_SAMPLE_RATE), format.getInteger(MediaFormat.KEY_CHANNEL_COUNT), - format.getInteger(MediaFormat.KEY_AAC_PROFILE), filledDecoderSpecificInfo); - } - - @Override - public void writeSampleData(@NonNull ByteBuffer byteBuf, @NonNull MediaCodec.BufferInfo bufferInfo) throws IOException { - final byte[] buffer = new byte[bufferInfo.size]; - byteBuf.position(bufferInfo.offset); - byteBuf.get(buffer, 0, bufferInfo.size); - processSample(ByteBuffer.wrap(buffer)); - } - - @Override - public void finish() { - } - } -} +package org.thoughtcrime.securesms.video.videoconverter.muxer; + +import android.media.MediaCodec; +import android.media.MediaFormat; + +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; + +import org.mp4parser.boxes.iso14496.part1.objectdescriptors.DecoderSpecificInfo; +import org.mp4parser.streaming.StreamingTrack; +import org.signal.core.util.logging.Log; +import org.thoughtcrime.securesms.video.interfaces.Muxer; +import org.thoughtcrime.securesms.video.videoconverter.utils.MediaCodecCompat; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.nio.channels.Channels; +import java.util.ArrayList; +import java.util.List; + +public final class StreamingMuxer implements Muxer { + private static final String TAG = Log.tag(StreamingMuxer.class); + private final OutputStream outputStream; + private final List tracks = new ArrayList<>(); + private Mp4Writer mp4Writer; + + public StreamingMuxer(OutputStream outputStream) { + this.outputStream = outputStream; + } + + @Override + public void start() throws IOException { + final List source = new ArrayList<>(); + for (MediaCodecTrack track : tracks) { + source.add((StreamingTrack) track); + } + mp4Writer = new Mp4Writer(source, Channels.newChannel(outputStream)); + } + + @Override + public long stop() throws IOException { + if (mp4Writer == null) { + throw new IllegalStateException("calling stop prior to start"); + } + for (MediaCodecTrack track : tracks) { + track.finish(); + } + mp4Writer.close(); + long mdatLength = mp4Writer.getTotalMdatContentLength(); + + mp4Writer = null; + + return mdatLength; + } + + @Override + public int addTrack(@NonNull MediaFormat format) throws IOException { + + final String mime = format.getString(MediaFormat.KEY_MIME); + switch (mime) { + case "video/avc": + tracks.add(new MediaCodecAvcTrack(format)); + break; + case "audio/mp4a-latm": + tracks.add(MediaCodecAacTrack.create(format)); + break; + case "video/hevc": + tracks.add(new MediaCodecHevcTrack(format)); + break; + default: + throw new IllegalArgumentException("unknown track format"); + } + return tracks.size() - 1; + } + + @Override + public void writeSampleData(int trackIndex, @NonNull ByteBuffer byteBuf, @NonNull MediaCodec.BufferInfo bufferInfo) throws IOException { + tracks.get(trackIndex).writeSampleData(byteBuf, bufferInfo); + } + + @Override + public void release() { + } + + @Override + public boolean supportsAudioRemux() { + return true; + } + + interface MediaCodecTrack { + void writeSampleData(@NonNull ByteBuffer byteBuf, @NonNull MediaCodec.BufferInfo bufferInfo) throws IOException; + + void finish() throws IOException; + } + + static class MediaCodecAvcTrack extends AvcTrack implements MediaCodecTrack { + + MediaCodecAvcTrack(@NonNull MediaFormat format) { + super(Utils.subBuffer(format.getByteBuffer("csd-0"), 4), Utils.subBuffer(format.getByteBuffer("csd-1"), 4)); + } + + @Override + public void writeSampleData(@NonNull ByteBuffer byteBuf, @NonNull MediaCodec.BufferInfo bufferInfo) throws IOException { + final List nals = H264Utils.getNals(byteBuf); + for (ByteBuffer nal : nals) { + consumeNal(Utils.clone(nal), bufferInfo.presentationTimeUs); + } + } + + @Override + public void finish() throws IOException { + consumeLastNal(); + } + } + + static class MediaCodecHevcTrack extends HevcTrack implements MediaCodecTrack { + + MediaCodecHevcTrack(@NonNull MediaFormat format) throws IOException { + super(H264Utils.getNals(format.getByteBuffer("csd-0"))); + } + + @Override + public void writeSampleData(@NonNull ByteBuffer byteBuf, @NonNull MediaCodec.BufferInfo bufferInfo) throws IOException { + final List nals = H264Utils.getNals(byteBuf); + for (ByteBuffer nal : nals) { + consumeNal(Utils.clone(nal), bufferInfo.presentationTimeUs); + } + } + + @Override + public void finish() throws IOException { + consumeLastNal(); + } + } + + static class MediaCodecAacTrack extends AacTrack implements MediaCodecTrack { + + private MediaCodecAacTrack(long avgBitrate, long maxBitrate, int sampleRate, int channelCount, int aacProfile, @Nullable DecoderSpecificInfo decoderSpecificInfo) { + super(avgBitrate, maxBitrate, sampleRate, channelCount, aacProfile, decoderSpecificInfo); + } + + public static MediaCodecAacTrack create(@NonNull MediaFormat format) { + final int bitrate = format.getInteger(MediaFormat.KEY_BIT_RATE); + final int maxBitrate; + if (format.containsKey(MediaCodecCompat.MEDIA_FORMAT_KEY_MAX_BIT_RATE)) { + maxBitrate = format.getInteger(MediaCodecCompat.MEDIA_FORMAT_KEY_MAX_BIT_RATE); + } else { + maxBitrate = bitrate; + } + + final DecoderSpecificInfo filledDecoderSpecificInfo; + if (format.containsKey(MediaCodecCompat.MEDIA_FORMAT_KEY_MAX_BIT_RATE)) { + final ByteBuffer csd = format.getByteBuffer(MediaCodecCompat.MEDIA_FORMAT_KEY_CODEC_SPECIFIC_DATA_0); + + DecoderSpecificInfo decoderSpecificInfo = new DecoderSpecificInfo(); + boolean parseSuccess = false; + try { + decoderSpecificInfo.parseDetail(csd); + parseSuccess = true; + } catch (IOException e) { + Log.w(TAG, "Could not parse AAC codec-specific data!", e); + } + if (parseSuccess) { + filledDecoderSpecificInfo = decoderSpecificInfo; + } else { + filledDecoderSpecificInfo = null; + } + } else { + filledDecoderSpecificInfo = null; + } + + return new MediaCodecAacTrack(bitrate, maxBitrate, + format.getInteger(MediaFormat.KEY_SAMPLE_RATE), format.getInteger(MediaFormat.KEY_CHANNEL_COUNT), + format.getInteger(MediaFormat.KEY_AAC_PROFILE), filledDecoderSpecificInfo); + } + + @Override + public void writeSampleData(@NonNull ByteBuffer byteBuf, @NonNull MediaCodec.BufferInfo bufferInfo) throws IOException { + final byte[] buffer = new byte[bufferInfo.size]; + byteBuf.position(bufferInfo.offset); + byteBuf.get(buffer, 0, bufferInfo.size); + processSample(ByteBuffer.wrap(buffer)); + } + + @Override + public void finish() { + } + } +} diff --git a/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/muxer/Utils.java b/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/muxer/Utils.java index a850fdae3c..c050d114b6 100644 --- a/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/muxer/Utils.java +++ b/lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/muxer/Utils.java @@ -1,44 +1,44 @@ -package org.thoughtcrime.securesms.video.videoconverter.muxer; - -import androidx.annotation.NonNull; - -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.util.ArrayList; -import java.util.List; - -/** - * Based on https://github.com/jcodec/jcodec/blob/master/src/main/java/org/jcodec/codecs/h264/H264Utils.java - */ -final class Utils { - - private Utils() {} - - static byte[] toArray(final @NonNull ByteBuffer buf) { - final ByteBuffer newBuf = buf.duplicate(); - byte[] bytes = new byte[newBuf.remaining()]; - newBuf.get(bytes, 0, bytes.length); - return bytes; - } - - public static ByteBuffer clone(final @NonNull ByteBuffer original) { - final ByteBuffer clone = ByteBuffer.allocate(original.capacity()); - original.rewind(); - clone.put(original); - original.rewind(); - clone.flip(); - return clone; - } - - static @NonNull ByteBuffer subBuffer(final @NonNull ByteBuffer buf, final int start) { - return subBuffer(buf, start, buf.limit() - start); - } - - static @NonNull ByteBuffer subBuffer(final @NonNull ByteBuffer buf, final int start, final int count) { - final ByteBuffer newBuf = buf.duplicate(); - byte[] bytes = new byte[count]; - newBuf.position(start); - newBuf.get(bytes, 0, bytes.length); - return ByteBuffer.wrap(bytes); - } -} +package org.thoughtcrime.securesms.video.videoconverter.muxer; + +import androidx.annotation.NonNull; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.ArrayList; +import java.util.List; + +/** + * Based on https://github.com/jcodec/jcodec/blob/master/src/main/java/org/jcodec/codecs/h264/H264Utils.java + */ +final class Utils { + + private Utils() {} + + static byte[] toArray(final @NonNull ByteBuffer buf) { + final ByteBuffer newBuf = buf.duplicate(); + byte[] bytes = new byte[newBuf.remaining()]; + newBuf.get(bytes, 0, bytes.length); + return bytes; + } + + public static ByteBuffer clone(final @NonNull ByteBuffer original) { + final ByteBuffer clone = ByteBuffer.allocate(original.capacity()); + original.rewind(); + clone.put(original); + original.rewind(); + clone.flip(); + return clone; + } + + static @NonNull ByteBuffer subBuffer(final @NonNull ByteBuffer buf, final int start) { + return subBuffer(buf, start, buf.limit() - start); + } + + static @NonNull ByteBuffer subBuffer(final @NonNull ByteBuffer buf, final int start, final int count) { + final ByteBuffer newBuf = buf.duplicate(); + byte[] bytes = new byte[count]; + newBuf.position(start); + newBuf.get(bytes, 0, bytes.length); + return ByteBuffer.wrap(bytes); + } +}