Signal/Android
1
0
Fork 0
mirror of https://github.com/signalapp/Signal-Android.git synced 2026-04-17 07:23:21 +01:00
Code Issues Packages Projects Releases Wiki Activity

Convert Windows line endings to Unix format.

Browse Source 
Closes signalapp/Signal-Android#14632
This commit is contained in:
Jesse Weinstein
2026-02-27 19:11:54 +00:00
committed by jeffrey-signal
parent 330a5aece2
commit 48cd1c1da0
13 changed files with 3484 additions and 3484 deletions
Download Patch File Download Diff File Expand all files Collapse all files

516
app/src/main/java/org/thoughtcrime/securesms/video/videoconverter/VideoThumbnailsView.java
View File

@@ -1,258 +1,258 @@
package org.thoughtcrime.securesms.video.videoconverter;
import android.content.Context;
import android.graphics.Bitmap;
import android.graphics.Canvas;
import android.graphics.Paint;
import android.graphics.Path;
import android.graphics.Rect;
import android.graphics.RectF;
import android.net.Uri;
import android.os.AsyncTask;
import android.util.AttributeSet;
import android.view.View;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import androidx.annotation.RequiresApi;
import org.signal.core.util.logging.Log;
import org.thoughtcrime.securesms.media.DecryptableUriMediaInput;
import org.thoughtcrime.securesms.util.ViewUtil;
import org.thoughtcrime.securesms.video.interfaces.MediaInput;
import java.io.IOException;
import java.lang.ref.WeakReference;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
@RequiresApi(api = 23)
abstract public class VideoThumbnailsView extends View {
private static final String TAG = Log.tag(VideoThumbnailsView.class);
private static final int CORNER_RADIUS = ViewUtil.dpToPx(8);
protected Uri currentUri;
private MediaInput input;
private volatile ArrayList<Bitmap> thumbnails;
private AsyncTask<Void, Bitmap, Void> thumbnailsTask;
private final Paint paint = new Paint(Paint.ANTI_ALIAS_FLAG);
private final RectF tempRect = new RectF();
private final Rect drawRect = new Rect();
private final Rect tempDrawRect = new Rect();
private long duration = 0;
protected final Path clippingPath = new Path();
public VideoThumbnailsView(final Context context) {
super(context);
}
public VideoThumbnailsView(final Context context, final @Nullable AttributeSet attrs) {
super(context, attrs);
}
public VideoThumbnailsView(final Context context, final @Nullable AttributeSet attrs, final int defStyleAttr) {
super(context, attrs, defStyleAttr);
}
/**
* @return Whether or not the current URI was changed.
*/
public boolean setInput(@NonNull Uri uri) throws IOException {
if (uri.equals(this.currentUri)) {
return false;
}
this.currentUri = uri;
this.input = DecryptableUriMediaInput.createForUri(getContext(), uri);
this.thumbnails = null;
if (thumbnailsTask != null) {
thumbnailsTask.cancel(true);
thumbnailsTask = null;
}
invalidate();
return true;
}
@Override
protected void onDetachedFromWindow() {
super.onDetachedFromWindow();
thumbnails = null;
if (thumbnailsTask != null) {
thumbnailsTask.cancel(true);
thumbnailsTask = null;
}
if (input != null) {
try {
input.close();
} catch (IOException e) {
Log.w(TAG, e);
}
}
}
@Override
protected void onDraw(final Canvas canvas) {
super.onDraw(canvas);
if (input == null) {
return;
}
final int left = getPaddingLeft();
final int top = getPaddingTop();
final int right = getWidth() - getPaddingRight();
final int bottom = getHeight() - getPaddingBottom();
clippingPath.reset();
clippingPath.addRoundRect(left, top, right, bottom, CORNER_RADIUS, CORNER_RADIUS, Path.Direction.CW);
tempDrawRect.set(left, top, right, bottom);
if (!drawRect.equals(tempDrawRect)) {
drawRect.set(tempDrawRect);
thumbnails = null;
if (thumbnailsTask != null) {
thumbnailsTask.cancel(true);
thumbnailsTask = null;
}
}
if (thumbnails == null) {
if (thumbnailsTask == null) {
final int thumbnailCount = drawRect.width() / drawRect.height();
final float thumbnailWidth = (float) drawRect.width() / thumbnailCount;
final float thumbnailHeight = drawRect.height();
thumbnails = new ArrayList<>(thumbnailCount);
thumbnailsTask = new ThumbnailsTask(this, input, thumbnailWidth, thumbnailHeight, thumbnailCount);
thumbnailsTask.execute();
}
} else {
final int thumbnailCount = drawRect.width() / drawRect.height();
final float thumbnailWidth = (float) drawRect.width() / thumbnailCount;
final float thumbnailHeight = drawRect.height();
tempRect.top = drawRect.top;
tempRect.bottom = drawRect.bottom;
canvas.save();
canvas.clipPath(clippingPath);
for (int i = 0; i < thumbnails.size(); i++) {
tempRect.left = drawRect.left + i * thumbnailWidth;
tempRect.right = tempRect.left + thumbnailWidth;
final Bitmap thumbnailBitmap = thumbnails.get(i);
if (thumbnailBitmap != null) {
canvas.save();
canvas.rotate(180, tempRect.centerX(), tempRect.centerY());
tempDrawRect.set(0, 0, thumbnailBitmap.getWidth(), thumbnailBitmap.getHeight());
if (tempDrawRect.width() * thumbnailHeight > tempDrawRect.height() * thumbnailWidth) {
float w = tempDrawRect.height() * thumbnailWidth / thumbnailHeight;
tempDrawRect.left = tempDrawRect.centerX() - (int) (w / 2);
tempDrawRect.right = tempDrawRect.left + (int) w;
} else {
float h = tempDrawRect.width() * thumbnailHeight / thumbnailWidth;
tempDrawRect.top = tempDrawRect.centerY() - (int) (h / 2);
tempDrawRect.bottom = tempDrawRect.top + (int) h;
}
canvas.drawBitmap(thumbnailBitmap, tempDrawRect, tempRect, paint);
canvas.restore();
}
}
canvas.restore();
}
}
private void setDuration(long duration) {
if (this.duration != duration) {
this.duration = duration;
afterDurationChange(duration);
}
}
abstract void afterDurationChange(long duration);
public long getDuration() {
return duration;
}
private static class ThumbnailsTask extends AsyncTask<Void, Bitmap, Void> {
final WeakReference<VideoThumbnailsView> viewReference;
final MediaInput input;
final float thumbnailWidth;
final float thumbnailHeight;
final int thumbnailCount;
long duration;
ThumbnailsTask(final @NonNull VideoThumbnailsView view, final @NonNull MediaInput input, final float thumbnailWidth, final float thumbnailHeight, final int thumbnailCount) {
this.viewReference = new WeakReference<>(view);
this.input = input;
this.thumbnailWidth = thumbnailWidth;
this.thumbnailHeight = thumbnailHeight;
this.thumbnailCount = thumbnailCount;
}
@Override
protected Void doInBackground(Void... params) {
Log.i(TAG, "generate " + thumbnailCount + " thumbnails " + thumbnailWidth + "x" + thumbnailHeight);
VideoThumbnailsExtractor.extractThumbnails(input, thumbnailCount, (int) thumbnailHeight, new VideoThumbnailsExtractor.Callback() {
@Override
public void durationKnown(long duration) {
ThumbnailsTask.this.duration = duration;
}
@Override
public boolean publishProgress(int index, Bitmap thumbnail) {
boolean notCanceled = !isCancelled();
if (notCanceled) {
ThumbnailsTask.this.publishProgress(thumbnail);
}
return notCanceled;
}
@Override
public void failed() {
Log.w(TAG, "Thumbnail extraction failed");
}
});
return null;
}
@Override
protected void onProgressUpdate(Bitmap... values) {
if (isCancelled()) {
return;
}
VideoThumbnailsView view = viewReference.get();
List<Bitmap> thumbnails = view != null ? view.thumbnails : null;
if (thumbnails != null) {
thumbnails.addAll(Arrays.asList(values));
view.invalidate();
}
}
@Override
protected void onPostExecute(Void result) {
VideoThumbnailsView view = viewReference.get();
List<Bitmap> thumbnails = view != null ? view.thumbnails : null;
if (view != null) {
view.setDuration(ThumbnailsTask.this.duration);
view.invalidate();
Log.i(TAG, "onPostExecute, we have " + (thumbnails != null ? thumbnails.size() : "null") + " thumbs");
}
}
}
}
package org.thoughtcrime.securesms.video.videoconverter;
import android.content.Context;
import android.graphics.Bitmap;
import android.graphics.Canvas;
import android.graphics.Paint;
import android.graphics.Path;
import android.graphics.Rect;
import android.graphics.RectF;
import android.net.Uri;
import android.os.AsyncTask;
import android.util.AttributeSet;
import android.view.View;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import androidx.annotation.RequiresApi;
import org.signal.core.util.logging.Log;
import org.thoughtcrime.securesms.media.DecryptableUriMediaInput;
import org.thoughtcrime.securesms.util.ViewUtil;
import org.thoughtcrime.securesms.video.interfaces.MediaInput;
import java.io.IOException;
import java.lang.ref.WeakReference;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
@RequiresApi(api = 23)
abstract public class VideoThumbnailsView extends View {
private static final String TAG = Log.tag(VideoThumbnailsView.class);
private static final int CORNER_RADIUS = ViewUtil.dpToPx(8);
protected Uri currentUri;
private MediaInput input;
private volatile ArrayList<Bitmap> thumbnails;
private AsyncTask<Void, Bitmap, Void> thumbnailsTask;
private final Paint paint = new Paint(Paint.ANTI_ALIAS_FLAG);
private final RectF tempRect = new RectF();
private final Rect drawRect = new Rect();
private final Rect tempDrawRect = new Rect();
private long duration = 0;
protected final Path clippingPath = new Path();
public VideoThumbnailsView(final Context context) {
super(context);
}
public VideoThumbnailsView(final Context context, final @Nullable AttributeSet attrs) {
super(context, attrs);
}
public VideoThumbnailsView(final Context context, final @Nullable AttributeSet attrs, final int defStyleAttr) {
super(context, attrs, defStyleAttr);
}
/**
* @return Whether or not the current URI was changed.
*/
public boolean setInput(@NonNull Uri uri) throws IOException {
if (uri.equals(this.currentUri)) {
return false;
}
this.currentUri = uri;
this.input = DecryptableUriMediaInput.createForUri(getContext(), uri);
this.thumbnails = null;
if (thumbnailsTask != null) {
thumbnailsTask.cancel(true);
thumbnailsTask = null;
}
invalidate();
return true;
}
@Override
protected void onDetachedFromWindow() {
super.onDetachedFromWindow();
thumbnails = null;
if (thumbnailsTask != null) {
thumbnailsTask.cancel(true);
thumbnailsTask = null;
}
if (input != null) {
try {
input.close();
} catch (IOException e) {
Log.w(TAG, e);
}
}
}
@Override
protected void onDraw(final Canvas canvas) {
super.onDraw(canvas);
if (input == null) {
return;
}
final int left = getPaddingLeft();
final int top = getPaddingTop();
final int right = getWidth() - getPaddingRight();
final int bottom = getHeight() - getPaddingBottom();
clippingPath.reset();
clippingPath.addRoundRect(left, top, right, bottom, CORNER_RADIUS, CORNER_RADIUS, Path.Direction.CW);
tempDrawRect.set(left, top, right, bottom);
if (!drawRect.equals(tempDrawRect)) {
drawRect.set(tempDrawRect);
thumbnails = null;
if (thumbnailsTask != null) {
thumbnailsTask.cancel(true);
thumbnailsTask = null;
}
}
if (thumbnails == null) {
if (thumbnailsTask == null) {
final int thumbnailCount = drawRect.width() / drawRect.height();
final float thumbnailWidth = (float) drawRect.width() / thumbnailCount;
final float thumbnailHeight = drawRect.height();
thumbnails = new ArrayList<>(thumbnailCount);
thumbnailsTask = new ThumbnailsTask(this, input, thumbnailWidth, thumbnailHeight, thumbnailCount);
thumbnailsTask.execute();
}
} else {
final int thumbnailCount = drawRect.width() / drawRect.height();
final float thumbnailWidth = (float) drawRect.width() / thumbnailCount;
final float thumbnailHeight = drawRect.height();
tempRect.top = drawRect.top;
tempRect.bottom = drawRect.bottom;
canvas.save();
canvas.clipPath(clippingPath);
for (int i = 0; i < thumbnails.size(); i++) {
tempRect.left = drawRect.left + i * thumbnailWidth;
tempRect.right = tempRect.left + thumbnailWidth;
final Bitmap thumbnailBitmap = thumbnails.get(i);
if (thumbnailBitmap != null) {
canvas.save();
canvas.rotate(180, tempRect.centerX(), tempRect.centerY());
tempDrawRect.set(0, 0, thumbnailBitmap.getWidth(), thumbnailBitmap.getHeight());
if (tempDrawRect.width() * thumbnailHeight > tempDrawRect.height() * thumbnailWidth) {
float w = tempDrawRect.height() * thumbnailWidth / thumbnailHeight;
tempDrawRect.left = tempDrawRect.centerX() - (int) (w / 2);
tempDrawRect.right = tempDrawRect.left + (int) w;
} else {
float h = tempDrawRect.width() * thumbnailHeight / thumbnailWidth;
tempDrawRect.top = tempDrawRect.centerY() - (int) (h / 2);
tempDrawRect.bottom = tempDrawRect.top + (int) h;
}
canvas.drawBitmap(thumbnailBitmap, tempDrawRect, tempRect, paint);
canvas.restore();
}
}
canvas.restore();
}
}
private void setDuration(long duration) {
if (this.duration != duration) {
this.duration = duration;
afterDurationChange(duration);
}
}
abstract void afterDurationChange(long duration);
public long getDuration() {
return duration;
}
private static class ThumbnailsTask extends AsyncTask<Void, Bitmap, Void> {
final WeakReference<VideoThumbnailsView> viewReference;
final MediaInput input;
final float thumbnailWidth;
final float thumbnailHeight;
final int thumbnailCount;
long duration;
ThumbnailsTask(final @NonNull VideoThumbnailsView view, final @NonNull MediaInput input, final float thumbnailWidth, final float thumbnailHeight, final int thumbnailCount) {
this.viewReference = new WeakReference<>(view);
this.input = input;
this.thumbnailWidth = thumbnailWidth;
this.thumbnailHeight = thumbnailHeight;
this.thumbnailCount = thumbnailCount;
}
@Override
protected Void doInBackground(Void... params) {
Log.i(TAG, "generate " + thumbnailCount + " thumbnails " + thumbnailWidth + "x" + thumbnailHeight);
VideoThumbnailsExtractor.extractThumbnails(input, thumbnailCount, (int) thumbnailHeight, new VideoThumbnailsExtractor.Callback() {
@Override
public void durationKnown(long duration) {
ThumbnailsTask.this.duration = duration;
}
@Override
public boolean publishProgress(int index, Bitmap thumbnail) {
boolean notCanceled = !isCancelled();
if (notCanceled) {
ThumbnailsTask.this.publishProgress(thumbnail);
}
return notCanceled;
}
@Override
public void failed() {
Log.w(TAG, "Thumbnail extraction failed");
}
});
return null;
}
@Override
protected void onProgressUpdate(Bitmap... values) {
if (isCancelled()) {
return;
}
VideoThumbnailsView view = viewReference.get();
List<Bitmap> thumbnails = view != null ? view.thumbnails : null;
if (thumbnails != null) {
thumbnails.addAll(Arrays.asList(values));
view.invalidate();
}
}
@Override
protected void onPostExecute(Void result) {
VideoThumbnailsView view = viewReference.get();
List<Bitmap> thumbnails = view != null ? view.thumbnails : null;
if (view != null) {
view.setDuration(ThumbnailsTask.this.duration);
view.invalidate();
Log.i(TAG, "onPostExecute, we have " + (thumbnails != null ? thumbnails.size() : "null") + " thumbs");
}
}
}
}

120
lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/AndroidMuxer.java
View File

@@ -1,60 +1,60 @@
package org.thoughtcrime.securesms.video.videoconverter;
import android.media.MediaCodec;
import android.media.MediaFormat;
import android.media.MediaMuxer;
import androidx.annotation.NonNull;
import androidx.annotation.RequiresApi;
import org.thoughtcrime.securesms.video.interfaces.Muxer;
import java.io.File;
import java.io.FileDescriptor;
import java.io.IOException;
import java.nio.ByteBuffer;
final class AndroidMuxer implements Muxer {
private final MediaMuxer muxer;
AndroidMuxer(final @NonNull File file) throws IOException {
muxer = new MediaMuxer(file.getAbsolutePath(), MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4);
}
@RequiresApi(26)
AndroidMuxer(final @NonNull FileDescriptor fileDescriptor) throws IOException {
muxer = new MediaMuxer(fileDescriptor, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4);
}
@Override
public void start() {
muxer.start();
}
@Override
public long stop() {
muxer.stop();
return 0;
}
@Override
public int addTrack(final @NonNull MediaFormat format) {
return muxer.addTrack(format);
}
@Override
public void writeSampleData(final int trackIndex, final @NonNull ByteBuffer byteBuf, final @NonNull MediaCodec.BufferInfo bufferInfo) {
muxer.writeSampleData(trackIndex, byteBuf, bufferInfo);
}
@Override
public void release() {
muxer.release();
}
@Override
public boolean supportsAudioRemux() {
return false;
}
}
package org.thoughtcrime.securesms.video.videoconverter;
import android.media.MediaCodec;
import android.media.MediaFormat;
import android.media.MediaMuxer;
import androidx.annotation.NonNull;
import androidx.annotation.RequiresApi;
import org.thoughtcrime.securesms.video.interfaces.Muxer;
import java.io.File;
import java.io.FileDescriptor;
import java.io.IOException;
import java.nio.ByteBuffer;
final class AndroidMuxer implements Muxer {
private final MediaMuxer muxer;
AndroidMuxer(final @NonNull File file) throws IOException {
muxer = new MediaMuxer(file.getAbsolutePath(), MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4);
}
@RequiresApi(26)
AndroidMuxer(final @NonNull FileDescriptor fileDescriptor) throws IOException {
muxer = new MediaMuxer(fileDescriptor, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4);
}
@Override
public void start() {
muxer.start();
}
@Override
public long stop() {
muxer.stop();
return 0;
}
@Override
public int addTrack(final @NonNull MediaFormat format) {
return muxer.addTrack(format);
}
@Override
public void writeSampleData(final int trackIndex, final @NonNull ByteBuffer byteBuf, final @NonNull MediaCodec.BufferInfo bufferInfo) {
muxer.writeSampleData(trackIndex, byteBuf, bufferInfo);
}
@Override
public void release() {
muxer.release();
}
@Override
public boolean supportsAudioRemux() {
return false;
}
}

1012
lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/AudioTrackConverter.java
View File

File diff suppressed because it is too large Load Diff

454
lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/VideoThumbnailsExtractor.java
View File

@@ -1,228 +1,228 @@
package org.thoughtcrime.securesms.video.videoconverter;
import android.graphics.Bitmap;
import android.media.MediaCodec;
import android.media.MediaExtractor;
import android.media.MediaFormat;
import android.opengl.GLES20;
import android.os.Build;
import android.os.Bundle;
import androidx.annotation.NonNull;
import androidx.annotation.RequiresApi;
import org.signal.core.util.logging.Log;
import org.thoughtcrime.securesms.video.interfaces.MediaInput;
import org.thoughtcrime.securesms.video.videoconverter.utils.MediaCodecCompat;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
@RequiresApi(api = 23)
final class VideoThumbnailsExtractor {
private static final String TAG = Log.tag(VideoThumbnailsExtractor.class);
interface Callback {
void durationKnown(long duration);
boolean publishProgress(int index, Bitmap thumbnail);
void failed();
}
static void extractThumbnails(final @NonNull MediaInput input,
final int thumbnailCount,
final int thumbnailResolution,
final @NonNull Callback callback)
{
MediaExtractor extractor = null;
MediaCodec decoder = null;
OutputSurface outputSurface = null;
try {
extractor = input.createExtractor();
MediaFormat mediaFormat = null;
for (int index = 0; index < extractor.getTrackCount(); ++index) {
final String mimeType = extractor.getTrackFormat(index).getString(MediaFormat.KEY_MIME);
if (mimeType != null && mimeType.startsWith("video/")) {
extractor.selectTrack(index);
mediaFormat = extractor.getTrackFormat(index);
break;
}
}
if (mediaFormat != null) {
final String mime = mediaFormat.getString(MediaFormat.KEY_MIME);
if (mime == null) {
throw new IllegalArgumentException("Mime type for MediaFormat was null: \t" + mediaFormat);
}
final int rotation = mediaFormat.containsKey(MediaFormat.KEY_ROTATION) ? mediaFormat.getInteger(MediaFormat.KEY_ROTATION) : 0;
final int width = mediaFormat.getInteger(MediaFormat.KEY_WIDTH);
final int height = mediaFormat.getInteger(MediaFormat.KEY_HEIGHT);
final int outputWidth;
final int outputHeight;
if (width < height) {
outputWidth = thumbnailResolution;
outputHeight = height * outputWidth / width;
} else {
outputHeight = thumbnailResolution;
outputWidth = width * outputHeight / height;
}
final int outputWidthRotated;
final int outputHeightRotated;
if ((rotation % 180 == 90)) {
//noinspection SuspiciousNameCombination
outputWidthRotated = outputHeight;
//noinspection SuspiciousNameCombination
outputHeightRotated = outputWidth;
} else {
outputWidthRotated = outputWidth;
outputHeightRotated = outputHeight;
}
Log.i(TAG, "video :" + width + "x" + height + " " + rotation);
Log.i(TAG, "output: " + outputWidthRotated + "x" + outputHeightRotated);
outputSurface = new OutputSurface(outputWidthRotated, outputHeightRotated, true);
decoder = MediaCodec.createDecoderByType(mime);
final boolean isHdr = MediaCodecCompat.isHdrVideo(mediaFormat);
if (Build.VERSION.SDK_INT >= 31 && isHdr) {
mediaFormat.setInteger(MediaFormat.KEY_COLOR_TRANSFER_REQUEST, MediaFormat.COLOR_TRANSFER_SDR_VIDEO);
}
decoder.configure(mediaFormat, outputSurface.getSurface(), null, 0);
decoder.start();
if (Build.VERSION.SDK_INT >= 31 && isHdr) {
try {
final String VENDOR_DOLBY_CODEC_TRANSFER_PARAMKEY = "vendor.dolby.codec.transfer.value";
MediaCodec.ParameterDescriptor descriptor = decoder.getParameterDescriptor(VENDOR_DOLBY_CODEC_TRANSFER_PARAMKEY);
if (descriptor != null) {
Bundle transferBundle = new Bundle();
transferBundle.putString(VENDOR_DOLBY_CODEC_TRANSFER_PARAMKEY, "transfer.sdr.normal");
decoder.setParameters(transferBundle);
}
} catch (IllegalStateException e) {
Log.w(TAG, "Failed to set Dolby Vision transfer parameter", e);
}
}
long duration = 0;
if (mediaFormat.containsKey(MediaFormat.KEY_DURATION)) {
duration = mediaFormat.getLong(MediaFormat.KEY_DURATION);
} else {
Log.w(TAG, "Video is missing duration!");
}
callback.durationKnown(duration);
doExtract(extractor, decoder, outputSurface, outputWidthRotated, outputHeightRotated, duration, thumbnailCount, callback);
}
} catch (Throwable t) {
Log.w(TAG, t);
callback.failed();
} finally {
if (outputSurface != null) {
outputSurface.release();
}
if (decoder != null) {
try {
decoder.stop();
} catch (MediaCodec.CodecException codecException) {
Log.w(TAG, "Decoder stop failed: " + codecException.getDiagnosticInfo(), codecException);
} catch (IllegalStateException ise) {
Log.w(TAG, "Decoder stop failed", ise);
}
decoder.release();
}
if (extractor != null) {
extractor.release();
}
}
}
private static void doExtract(final @NonNull MediaExtractor extractor,
final @NonNull MediaCodec decoder,
final @NonNull OutputSurface outputSurface,
final int outputWidth, int outputHeight, long duration, int thumbnailCount,
final @NonNull Callback callback)
throws TranscodingException
{
final int TIMEOUT_USEC = 10000;
final ByteBuffer[] decoderInputBuffers = decoder.getInputBuffers();
final MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
int samplesExtracted = 0;
int thumbnailsCreated = 0;
Log.i(TAG, "doExtract started");
final ByteBuffer pixelBuf = ByteBuffer.allocateDirect(outputWidth * outputHeight * 4);
pixelBuf.order(ByteOrder.LITTLE_ENDIAN);
boolean outputDone = false;
boolean inputDone = false;
while (!outputDone) {
if (!inputDone) {
int inputBufIndex = decoder.dequeueInputBuffer(TIMEOUT_USEC);
if (inputBufIndex >= 0) {
final ByteBuffer inputBuf = decoderInputBuffers[inputBufIndex];
final int sampleSize = extractor.readSampleData(inputBuf, 0);
if (sampleSize < 0 || samplesExtracted >= thumbnailCount) {
decoder.queueInputBuffer(inputBufIndex, 0, 0, 0L, MediaCodec.BUFFER_FLAG_END_OF_STREAM);
inputDone = true;
Log.i(TAG, "input done");
} else {
final long presentationTimeUs = extractor.getSampleTime();
decoder.queueInputBuffer(inputBufIndex, 0, sampleSize, presentationTimeUs, 0 /*flags*/);
samplesExtracted++;
extractor.seekTo(duration * samplesExtracted / thumbnailCount, MediaExtractor.SEEK_TO_CLOSEST_SYNC);
Log.i(TAG, "seek to " + duration * samplesExtracted / thumbnailCount + ", actual " + extractor.getSampleTime());
}
}
}
final int outputBufIndex;
try {
outputBufIndex = decoder.dequeueOutputBuffer(info, TIMEOUT_USEC);
} catch (IllegalStateException e) {
Log.w(TAG, "Decoder not in the Executing state, or codec is configured in asynchronous mode.", e);
throw new TranscodingException("Decoder not in the Executing state, or codec is configured in asynchronous mode.", e);
}
if (outputBufIndex >= 0) {
if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
outputDone = true;
}
final boolean shouldRender = (info.size != 0) /*&& (info.presentationTimeUs >= duration * decodeCount / thumbnailCount)*/;
decoder.releaseOutputBuffer(outputBufIndex, shouldRender);
if (shouldRender) {
outputSurface.awaitNewImage();
outputSurface.drawImage();
if (thumbnailsCreated < thumbnailCount) {
pixelBuf.rewind();
GLES20.glReadPixels(0, 0, outputWidth, outputHeight, GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, pixelBuf);
final Bitmap bitmap = Bitmap.createBitmap(outputWidth, outputHeight, Bitmap.Config.ARGB_8888);
pixelBuf.rewind();
bitmap.copyPixelsFromBuffer(pixelBuf);
if (!callback.publishProgress(thumbnailsCreated, bitmap)) {
break;
}
Log.i(TAG, "publishProgress for frame " + thumbnailsCreated + " at " + info.presentationTimeUs + " (target " + duration * thumbnailsCreated / thumbnailCount + ")");
}
thumbnailsCreated++;
}
}
}
Log.i(TAG, "doExtract finished");
}
package org.thoughtcrime.securesms.video.videoconverter;
import android.graphics.Bitmap;
import android.media.MediaCodec;
import android.media.MediaExtractor;
import android.media.MediaFormat;
import android.opengl.GLES20;
import android.os.Build;
import android.os.Bundle;
import androidx.annotation.NonNull;
import androidx.annotation.RequiresApi;
import org.signal.core.util.logging.Log;
import org.thoughtcrime.securesms.video.interfaces.MediaInput;
import org.thoughtcrime.securesms.video.videoconverter.utils.MediaCodecCompat;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
@RequiresApi(api = 23)
final class VideoThumbnailsExtractor {
private static final String TAG = Log.tag(VideoThumbnailsExtractor.class);
interface Callback {
void durationKnown(long duration);
boolean publishProgress(int index, Bitmap thumbnail);
void failed();
}
static void extractThumbnails(final @NonNull MediaInput input,
final int thumbnailCount,
final int thumbnailResolution,
final @NonNull Callback callback)
{
MediaExtractor extractor = null;
MediaCodec decoder = null;
OutputSurface outputSurface = null;
try {
extractor = input.createExtractor();
MediaFormat mediaFormat = null;
for (int index = 0; index < extractor.getTrackCount(); ++index) {
final String mimeType = extractor.getTrackFormat(index).getString(MediaFormat.KEY_MIME);
if (mimeType != null && mimeType.startsWith("video/")) {
extractor.selectTrack(index);
mediaFormat = extractor.getTrackFormat(index);
break;
}
}
if (mediaFormat != null) {
final String mime = mediaFormat.getString(MediaFormat.KEY_MIME);
if (mime == null) {
throw new IllegalArgumentException("Mime type for MediaFormat was null: \t" + mediaFormat);
}
final int rotation = mediaFormat.containsKey(MediaFormat.KEY_ROTATION) ? mediaFormat.getInteger(MediaFormat.KEY_ROTATION) : 0;
final int width = mediaFormat.getInteger(MediaFormat.KEY_WIDTH);
final int height = mediaFormat.getInteger(MediaFormat.KEY_HEIGHT);
final int outputWidth;
final int outputHeight;
if (width < height) {
outputWidth = thumbnailResolution;
outputHeight = height * outputWidth / width;
} else {
outputHeight = thumbnailResolution;
outputWidth = width * outputHeight / height;
}
final int outputWidthRotated;
final int outputHeightRotated;
if ((rotation % 180 == 90)) {
//noinspection SuspiciousNameCombination
outputWidthRotated = outputHeight;
//noinspection SuspiciousNameCombination
outputHeightRotated = outputWidth;
} else {
outputWidthRotated = outputWidth;
outputHeightRotated = outputHeight;
}
Log.i(TAG, "video :" + width + "x" + height + " " + rotation);
Log.i(TAG, "output: " + outputWidthRotated + "x" + outputHeightRotated);
outputSurface = new OutputSurface(outputWidthRotated, outputHeightRotated, true);
decoder = MediaCodec.createDecoderByType(mime);
final boolean isHdr = MediaCodecCompat.isHdrVideo(mediaFormat);
if (Build.VERSION.SDK_INT >= 31 && isHdr) {
mediaFormat.setInteger(MediaFormat.KEY_COLOR_TRANSFER_REQUEST, MediaFormat.COLOR_TRANSFER_SDR_VIDEO);
}
decoder.configure(mediaFormat, outputSurface.getSurface(), null, 0);
decoder.start();
if (Build.VERSION.SDK_INT >= 31 && isHdr) {
try {
final String VENDOR_DOLBY_CODEC_TRANSFER_PARAMKEY = "vendor.dolby.codec.transfer.value";
MediaCodec.ParameterDescriptor descriptor = decoder.getParameterDescriptor(VENDOR_DOLBY_CODEC_TRANSFER_PARAMKEY);
if (descriptor != null) {
Bundle transferBundle = new Bundle();
transferBundle.putString(VENDOR_DOLBY_CODEC_TRANSFER_PARAMKEY, "transfer.sdr.normal");
decoder.setParameters(transferBundle);
}
} catch (IllegalStateException e) {
Log.w(TAG, "Failed to set Dolby Vision transfer parameter", e);
}
}
long duration = 0;
if (mediaFormat.containsKey(MediaFormat.KEY_DURATION)) {
duration = mediaFormat.getLong(MediaFormat.KEY_DURATION);
} else {
Log.w(TAG, "Video is missing duration!");
}
callback.durationKnown(duration);
doExtract(extractor, decoder, outputSurface, outputWidthRotated, outputHeightRotated, duration, thumbnailCount, callback);
}
} catch (Throwable t) {
Log.w(TAG, t);
callback.failed();
} finally {
if (outputSurface != null) {
outputSurface.release();
}
if (decoder != null) {
try {
decoder.stop();
} catch (MediaCodec.CodecException codecException) {
Log.w(TAG, "Decoder stop failed: " + codecException.getDiagnosticInfo(), codecException);
} catch (IllegalStateException ise) {
Log.w(TAG, "Decoder stop failed", ise);
}
decoder.release();
}
if (extractor != null) {
extractor.release();
}
}
}
private static void doExtract(final @NonNull MediaExtractor extractor,
final @NonNull MediaCodec decoder,
final @NonNull OutputSurface outputSurface,
final int outputWidth, int outputHeight, long duration, int thumbnailCount,
final @NonNull Callback callback)
throws TranscodingException
{
final int TIMEOUT_USEC = 10000;
final ByteBuffer[] decoderInputBuffers = decoder.getInputBuffers();
final MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
int samplesExtracted = 0;
int thumbnailsCreated = 0;
Log.i(TAG, "doExtract started");
final ByteBuffer pixelBuf = ByteBuffer.allocateDirect(outputWidth * outputHeight * 4);
pixelBuf.order(ByteOrder.LITTLE_ENDIAN);
boolean outputDone = false;
boolean inputDone = false;
while (!outputDone) {
if (!inputDone) {
int inputBufIndex = decoder.dequeueInputBuffer(TIMEOUT_USEC);
if (inputBufIndex >= 0) {
final ByteBuffer inputBuf = decoderInputBuffers[inputBufIndex];
final int sampleSize = extractor.readSampleData(inputBuf, 0);
if (sampleSize < 0 || samplesExtracted >= thumbnailCount) {
decoder.queueInputBuffer(inputBufIndex, 0, 0, 0L, MediaCodec.BUFFER_FLAG_END_OF_STREAM);
inputDone = true;
Log.i(TAG, "input done");
} else {
final long presentationTimeUs = extractor.getSampleTime();
decoder.queueInputBuffer(inputBufIndex, 0, sampleSize, presentationTimeUs, 0 /*flags*/);
samplesExtracted++;
extractor.seekTo(duration * samplesExtracted / thumbnailCount, MediaExtractor.SEEK_TO_CLOSEST_SYNC);
Log.i(TAG, "seek to " + duration * samplesExtracted / thumbnailCount + ", actual " + extractor.getSampleTime());
}
}
}
final int outputBufIndex;
try {
outputBufIndex = decoder.dequeueOutputBuffer(info, TIMEOUT_USEC);
} catch (IllegalStateException e) {
Log.w(TAG, "Decoder not in the Executing state, or codec is configured in asynchronous mode.", e);
throw new TranscodingException("Decoder not in the Executing state, or codec is configured in asynchronous mode.", e);
}
if (outputBufIndex >= 0) {
if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
outputDone = true;
}
final boolean shouldRender = (info.size != 0) /*&& (info.presentationTimeUs >= duration * decodeCount / thumbnailCount)*/;
decoder.releaseOutputBuffer(outputBufIndex, shouldRender);
if (shouldRender) {
outputSurface.awaitNewImage();
outputSurface.drawImage();
if (thumbnailsCreated < thumbnailCount) {
pixelBuf.rewind();
GLES20.glReadPixels(0, 0, outputWidth, outputHeight, GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, pixelBuf);
final Bitmap bitmap = Bitmap.createBitmap(outputWidth, outputHeight, Bitmap.Config.ARGB_8888);
pixelBuf.rewind();
bitmap.copyPixelsFromBuffer(pixelBuf);
if (!callback.publishProgress(thumbnailsCreated, bitmap)) {
break;
}
Log.i(TAG, "publishProgress for frame " + thumbnailsCreated + " at " + info.presentationTimeUs + " (target " + duration * thumbnailsCreated / thumbnailCount + ")");
}
thumbnailsCreated++;
}
}
}
Log.i(TAG, "doExtract finished");
}
}

1406
lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/VideoTrackConverter.java
View File

File diff suppressed because it is too large Load Diff

246
lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/muxer/AacTrack.java
View File

@@ -1,123 +1,123 @@
package org.thoughtcrime.securesms.video.videoconverter.muxer;
import android.util.SparseIntArray;
import org.mp4parser.boxes.iso14496.part1.objectdescriptors.AudioSpecificConfig;
import org.mp4parser.boxes.iso14496.part1.objectdescriptors.DecoderConfigDescriptor;
import org.mp4parser.boxes.iso14496.part1.objectdescriptors.DecoderSpecificInfo;
import org.mp4parser.boxes.iso14496.part1.objectdescriptors.ESDescriptor;
import org.mp4parser.boxes.iso14496.part1.objectdescriptors.SLConfigDescriptor;
import org.mp4parser.boxes.iso14496.part12.SampleDescriptionBox;
import org.mp4parser.boxes.iso14496.part14.ESDescriptorBox;
import org.mp4parser.boxes.sampleentry.AudioSampleEntry;
import org.mp4parser.streaming.extensions.DefaultSampleFlagsTrackExtension;
import org.mp4parser.streaming.input.AbstractStreamingTrack;
import org.mp4parser.streaming.input.StreamingSampleImpl;
import java.io.IOException;
import java.nio.ByteBuffer;
import androidx.annotation.Nullable;
abstract class AacTrack extends AbstractStreamingTrack {
private static final SparseIntArray SAMPLING_FREQUENCY_INDEX_MAP = new SparseIntArray();
static {
SAMPLING_FREQUENCY_INDEX_MAP.put(96000, 0);
SAMPLING_FREQUENCY_INDEX_MAP.put(88200, 1);
SAMPLING_FREQUENCY_INDEX_MAP.put(64000, 2);
SAMPLING_FREQUENCY_INDEX_MAP.put(48000, 3);
SAMPLING_FREQUENCY_INDEX_MAP.put(44100, 4);
SAMPLING_FREQUENCY_INDEX_MAP.put(32000, 5);
SAMPLING_FREQUENCY_INDEX_MAP.put(24000, 6);
SAMPLING_FREQUENCY_INDEX_MAP.put(22050, 7);
SAMPLING_FREQUENCY_INDEX_MAP.put(16000, 8);
SAMPLING_FREQUENCY_INDEX_MAP.put(12000, 9);
SAMPLING_FREQUENCY_INDEX_MAP.put(11025, 10);
SAMPLING_FREQUENCY_INDEX_MAP.put(8000, 11);
}
private final SampleDescriptionBox stsd;
private int sampleRate;
AacTrack(long avgBitrate, long maxBitrate, int sampleRate, int channelCount, int aacProfile, @Nullable DecoderSpecificInfo decoderSpecificInfo) {
this.sampleRate = sampleRate;
final DefaultSampleFlagsTrackExtension defaultSampleFlagsTrackExtension = new DefaultSampleFlagsTrackExtension();
defaultSampleFlagsTrackExtension.setIsLeading(2);
defaultSampleFlagsTrackExtension.setSampleDependsOn(2);
defaultSampleFlagsTrackExtension.setSampleIsDependedOn(2);
defaultSampleFlagsTrackExtension.setSampleHasRedundancy(2);
defaultSampleFlagsTrackExtension.setSampleIsNonSyncSample(false);
this.addTrackExtension(defaultSampleFlagsTrackExtension);
stsd = new SampleDescriptionBox();
final AudioSampleEntry audioSampleEntry = new AudioSampleEntry("mp4a");
if (channelCount == 7) {
audioSampleEntry.setChannelCount(8);
} else {
audioSampleEntry.setChannelCount(channelCount);
}
audioSampleEntry.setSampleRate(sampleRate);
audioSampleEntry.setDataReferenceIndex(1);
audioSampleEntry.setSampleSize(16);
final ESDescriptorBox esds = new ESDescriptorBox();
ESDescriptor descriptor = new ESDescriptor();
descriptor.setEsId(0);
final SLConfigDescriptor slConfigDescriptor = new SLConfigDescriptor();
slConfigDescriptor.setPredefined(2);
descriptor.setSlConfigDescriptor(slConfigDescriptor);
final DecoderConfigDescriptor decoderConfigDescriptor = new DecoderConfigDescriptor();
decoderConfigDescriptor.setObjectTypeIndication(0x40 /*Audio ISO/IEC 14496-3*/);
decoderConfigDescriptor.setStreamType(5 /*audio stream*/);
decoderConfigDescriptor.setBufferSizeDB(1536);
decoderConfigDescriptor.setMaxBitRate(maxBitrate);
decoderConfigDescriptor.setAvgBitRate(avgBitrate);
final AudioSpecificConfig audioSpecificConfig = new AudioSpecificConfig();
audioSpecificConfig.setOriginalAudioObjectType(aacProfile);
audioSpecificConfig.setSamplingFrequencyIndex(SAMPLING_FREQUENCY_INDEX_MAP.get(sampleRate));
audioSpecificConfig.setChannelConfiguration(channelCount);
decoderConfigDescriptor.setAudioSpecificInfo(audioSpecificConfig);
if (decoderSpecificInfo != null) {
decoderConfigDescriptor.setDecoderSpecificInfo(decoderSpecificInfo);
}
descriptor.setDecoderConfigDescriptor(decoderConfigDescriptor);
esds.setEsDescriptor(descriptor);
audioSampleEntry.addBox(esds);
stsd.addBox(audioSampleEntry);
}
public long getTimescale() {
return sampleRate;
}
public String getHandler() {
return "soun";
}
public String getLanguage() {
return "\u0060\u0060\u0060"; // 0 in Iso639
}
public synchronized SampleDescriptionBox getSampleDescriptionBox() {
return stsd;
}
public void close() {
}
void processSample(ByteBuffer frame) throws IOException {
sampleSink.acceptSample(new StreamingSampleImpl(frame, 1024), this);
}
}
package org.thoughtcrime.securesms.video.videoconverter.muxer;
import android.util.SparseIntArray;
import org.mp4parser.boxes.iso14496.part1.objectdescriptors.AudioSpecificConfig;
import org.mp4parser.boxes.iso14496.part1.objectdescriptors.DecoderConfigDescriptor;
import org.mp4parser.boxes.iso14496.part1.objectdescriptors.DecoderSpecificInfo;
import org.mp4parser.boxes.iso14496.part1.objectdescriptors.ESDescriptor;
import org.mp4parser.boxes.iso14496.part1.objectdescriptors.SLConfigDescriptor;
import org.mp4parser.boxes.iso14496.part12.SampleDescriptionBox;
import org.mp4parser.boxes.iso14496.part14.ESDescriptorBox;
import org.mp4parser.boxes.sampleentry.AudioSampleEntry;
import org.mp4parser.streaming.extensions.DefaultSampleFlagsTrackExtension;
import org.mp4parser.streaming.input.AbstractStreamingTrack;
import org.mp4parser.streaming.input.StreamingSampleImpl;
import java.io.IOException;
import java.nio.ByteBuffer;
import androidx.annotation.Nullable;
abstract class AacTrack extends AbstractStreamingTrack {
private static final SparseIntArray SAMPLING_FREQUENCY_INDEX_MAP = new SparseIntArray();
static {
SAMPLING_FREQUENCY_INDEX_MAP.put(96000, 0);
SAMPLING_FREQUENCY_INDEX_MAP.put(88200, 1);
SAMPLING_FREQUENCY_INDEX_MAP.put(64000, 2);
SAMPLING_FREQUENCY_INDEX_MAP.put(48000, 3);
SAMPLING_FREQUENCY_INDEX_MAP.put(44100, 4);
SAMPLING_FREQUENCY_INDEX_MAP.put(32000, 5);
SAMPLING_FREQUENCY_INDEX_MAP.put(24000, 6);
SAMPLING_FREQUENCY_INDEX_MAP.put(22050, 7);
SAMPLING_FREQUENCY_INDEX_MAP.put(16000, 8);
SAMPLING_FREQUENCY_INDEX_MAP.put(12000, 9);
SAMPLING_FREQUENCY_INDEX_MAP.put(11025, 10);
SAMPLING_FREQUENCY_INDEX_MAP.put(8000, 11);
}
private final SampleDescriptionBox stsd;
private int sampleRate;
AacTrack(long avgBitrate, long maxBitrate, int sampleRate, int channelCount, int aacProfile, @Nullable DecoderSpecificInfo decoderSpecificInfo) {
this.sampleRate = sampleRate;
final DefaultSampleFlagsTrackExtension defaultSampleFlagsTrackExtension = new DefaultSampleFlagsTrackExtension();
defaultSampleFlagsTrackExtension.setIsLeading(2);
defaultSampleFlagsTrackExtension.setSampleDependsOn(2);
defaultSampleFlagsTrackExtension.setSampleIsDependedOn(2);
defaultSampleFlagsTrackExtension.setSampleHasRedundancy(2);
defaultSampleFlagsTrackExtension.setSampleIsNonSyncSample(false);
this.addTrackExtension(defaultSampleFlagsTrackExtension);
stsd = new SampleDescriptionBox();
final AudioSampleEntry audioSampleEntry = new AudioSampleEntry("mp4a");
if (channelCount == 7) {
audioSampleEntry.setChannelCount(8);
} else {
audioSampleEntry.setChannelCount(channelCount);
}
audioSampleEntry.setSampleRate(sampleRate);
audioSampleEntry.setDataReferenceIndex(1);
audioSampleEntry.setSampleSize(16);
final ESDescriptorBox esds = new ESDescriptorBox();
ESDescriptor descriptor = new ESDescriptor();
descriptor.setEsId(0);
final SLConfigDescriptor slConfigDescriptor = new SLConfigDescriptor();
slConfigDescriptor.setPredefined(2);
descriptor.setSlConfigDescriptor(slConfigDescriptor);
final DecoderConfigDescriptor decoderConfigDescriptor = new DecoderConfigDescriptor();
decoderConfigDescriptor.setObjectTypeIndication(0x40 /*Audio ISO/IEC 14496-3*/);
decoderConfigDescriptor.setStreamType(5 /*audio stream*/);
decoderConfigDescriptor.setBufferSizeDB(1536);
decoderConfigDescriptor.setMaxBitRate(maxBitrate);
decoderConfigDescriptor.setAvgBitRate(avgBitrate);
final AudioSpecificConfig audioSpecificConfig = new AudioSpecificConfig();
audioSpecificConfig.setOriginalAudioObjectType(aacProfile);
audioSpecificConfig.setSamplingFrequencyIndex(SAMPLING_FREQUENCY_INDEX_MAP.get(sampleRate));
audioSpecificConfig.setChannelConfiguration(channelCount);
decoderConfigDescriptor.setAudioSpecificInfo(audioSpecificConfig);
if (decoderSpecificInfo != null) {
decoderConfigDescriptor.setDecoderSpecificInfo(decoderSpecificInfo);
}
descriptor.setDecoderConfigDescriptor(decoderConfigDescriptor);
esds.setEsDescriptor(descriptor);
audioSampleEntry.addBox(esds);
stsd.addBox(audioSampleEntry);
}
public long getTimescale() {
return sampleRate;
}
public String getHandler() {
return "soun";
}
public String getLanguage() {
return "\u0060\u0060\u0060"; // 0 in Iso639
}
public synchronized SampleDescriptionBox getSampleDescriptionBox() {
return stsd;
}
public void close() {
}
void processSample(ByteBuffer frame) throws IOException {
sampleSink.acceptSample(new StreamingSampleImpl(frame, 1024), this);
}
}

956
lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/muxer/AvcTrack.java
View File

@@ -1,478 +1,478 @@
package org.thoughtcrime.securesms.video.videoconverter.muxer;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import org.mp4parser.boxes.iso14496.part12.SampleDescriptionBox;
import org.mp4parser.boxes.iso14496.part15.AvcConfigurationBox;
import org.mp4parser.boxes.sampleentry.VisualSampleEntry;
import org.mp4parser.streaming.SampleExtension;
import org.mp4parser.streaming.StreamingSample;
import org.mp4parser.streaming.extensions.CompositionTimeSampleExtension;
import org.mp4parser.streaming.extensions.CompositionTimeTrackExtension;
import org.mp4parser.streaming.extensions.DimensionTrackExtension;
import org.mp4parser.streaming.extensions.SampleFlagsSampleExtension;
import org.mp4parser.streaming.input.AbstractStreamingTrack;
import org.mp4parser.streaming.input.StreamingSampleImpl;
import org.mp4parser.streaming.input.h264.H264NalUnitHeader;
import org.mp4parser.streaming.input.h264.H264NalUnitTypes;
import org.mp4parser.streaming.input.h264.spspps.PictureParameterSet;
import org.mp4parser.streaming.input.h264.spspps.SeqParameterSet;
import org.mp4parser.streaming.input.h264.spspps.SliceHeader;
import org.signal.core.util.logging.Log;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.List;
abstract class AvcTrack extends AbstractStreamingTrack {
private static final String TAG = "AvcTrack";
private int maxDecFrameBuffering = 16;
private final List<StreamingSample> decFrameBuffer = new ArrayList<>();
private final List<StreamingSample> decFrameBuffer2 = new ArrayList<>();
private final LinkedHashMap<Integer, ByteBuffer> spsIdToSpsBytes = new LinkedHashMap<>();
private final LinkedHashMap<Integer, SeqParameterSet> spsIdToSps = new LinkedHashMap<>();
private final LinkedHashMap<Integer, ByteBuffer> ppsIdToPpsBytes = new LinkedHashMap<>();
private final LinkedHashMap<Integer, PictureParameterSet> ppsIdToPps = new LinkedHashMap<>();
private int timescale = 90000;
private int frametick = 3000;
private final SampleDescriptionBox stsd;
private final List<ByteBuffer> bufferedNals = new ArrayList<>();
private FirstVclNalDetector fvnd;
private H264NalUnitHeader sliceNalUnitHeader;
private long currentPresentationTimeUs;
AvcTrack(final @NonNull ByteBuffer spsBuffer, final @NonNull ByteBuffer ppsBuffer) {
handlePPS(ppsBuffer);
final SeqParameterSet sps = handleSPS(spsBuffer);
int width = (sps.pic_width_in_mbs_minus1 + 1) * 16;
int mult = 2;
if (sps.frame_mbs_only_flag) {
mult = 1;
}
int height = 16 * (sps.pic_height_in_map_units_minus1 + 1) * mult;
if (sps.frame_cropping_flag) {
int chromaArrayType = 0;
if (!sps.residual_color_transform_flag) {
chromaArrayType = sps.chroma_format_idc.getId();
}
int cropUnitX = 1;
int cropUnitY = mult;
if (chromaArrayType != 0) {
cropUnitX = sps.chroma_format_idc.getSubWidth();
cropUnitY = sps.chroma_format_idc.getSubHeight() * mult;
}
width -= cropUnitX * (sps.frame_crop_left_offset + sps.frame_crop_right_offset);
height -= cropUnitY * (sps.frame_crop_top_offset + sps.frame_crop_bottom_offset);
}
final VisualSampleEntry visualSampleEntry = new VisualSampleEntry("avc1");
visualSampleEntry.setDataReferenceIndex(1);
visualSampleEntry.setDepth(24);
visualSampleEntry.setFrameCount(1);
visualSampleEntry.setHorizresolution(72);
visualSampleEntry.setVertresolution(72);
final DimensionTrackExtension dte = this.getTrackExtension(DimensionTrackExtension.class);
if (dte == null) {
this.addTrackExtension(new DimensionTrackExtension(width, height));
}
visualSampleEntry.setWidth(width);
visualSampleEntry.setHeight(height);
visualSampleEntry.setCompressorname("AVC Coding");
final AvcConfigurationBox avcConfigurationBox = new AvcConfigurationBox();
avcConfigurationBox.setSequenceParameterSets(Collections.singletonList(spsBuffer));
avcConfigurationBox.setPictureParameterSets(Collections.singletonList(ppsBuffer));
avcConfigurationBox.setAvcLevelIndication(sps.level_idc);
avcConfigurationBox.setAvcProfileIndication(sps.profile_idc);
avcConfigurationBox.setBitDepthLumaMinus8(sps.bit_depth_luma_minus8);
avcConfigurationBox.setBitDepthChromaMinus8(sps.bit_depth_chroma_minus8);
avcConfigurationBox.setChromaFormat(sps.chroma_format_idc.getId());
avcConfigurationBox.setConfigurationVersion(1);
avcConfigurationBox.setLengthSizeMinusOne(3);
avcConfigurationBox.setProfileCompatibility(
(sps.constraint_set_0_flag ? 128 : 0) +
(sps.constraint_set_1_flag ? 64 : 0) +
(sps.constraint_set_2_flag ? 32 : 0) +
(sps.constraint_set_3_flag ? 16 : 0) +
(sps.constraint_set_4_flag ? 8 : 0) +
(int) (sps.reserved_zero_2bits & 0x3)
);
visualSampleEntry.addBox(avcConfigurationBox);
stsd = new SampleDescriptionBox();
stsd.addBox(visualSampleEntry);
int _timescale;
int _frametick;
if (sps.vuiParams != null) {
_timescale = sps.vuiParams.time_scale >> 1; // Not sure why, but I found this in several places, and it works...
_frametick = sps.vuiParams.num_units_in_tick;
if (_timescale == 0 || _frametick == 0) {
Log.w(TAG, "vuiParams contain invalid values: time_scale: " + _timescale + " and frame_tick: " + _frametick + ". Setting frame rate to 30fps");
_timescale = 0;
_frametick = 0;
}
if (_frametick > 0) {
if (_timescale / _frametick > 100) {
Log.w(TAG, "Framerate is " + (_timescale / _frametick) + ". That is suspicious.");
}
} else {
Log.w(TAG, "Frametick is " + _frametick + ". That is suspicious.");
}
if (sps.vuiParams.bitstreamRestriction != null) {
maxDecFrameBuffering = sps.vuiParams.bitstreamRestriction.max_dec_frame_buffering;
}
} else {
Log.w(TAG, "Can't determine frame rate as SPS does not contain vuiParama");
_timescale = 0;
_frametick = 0;
}
if (_timescale != 0 && _frametick != 0) {
timescale = _timescale;
frametick = _frametick;
}
if (sps.pic_order_cnt_type == 0) {
addTrackExtension(new CompositionTimeTrackExtension());
} else if (sps.pic_order_cnt_type == 1) {
throw new MuxingException("Have not yet imlemented pic_order_cnt_type 1");
}
}
public long getTimescale() {
return timescale;
}
public String getHandler() {
return "vide";
}
public String getLanguage() {
return "\u0060\u0060\u0060"; // 0 in Iso639
}
public SampleDescriptionBox getSampleDescriptionBox() {
return stsd;
}
public void close() {
}
private static H264NalUnitHeader getNalUnitHeader(@NonNull final ByteBuffer nal) {
final H264NalUnitHeader nalUnitHeader = new H264NalUnitHeader();
final int type = nal.get(0);
nalUnitHeader.nal_ref_idc = (type >> 5) & 3;
nalUnitHeader.nal_unit_type = type & 0x1f;
return nalUnitHeader;
}
void consumeNal(@NonNull final ByteBuffer nal, final long presentationTimeUs) throws IOException {
final H264NalUnitHeader nalUnitHeader = getNalUnitHeader(nal);
switch (nalUnitHeader.nal_unit_type) {
case H264NalUnitTypes.CODED_SLICE_NON_IDR:
case H264NalUnitTypes.CODED_SLICE_DATA_PART_A:
case H264NalUnitTypes.CODED_SLICE_DATA_PART_B:
case H264NalUnitTypes.CODED_SLICE_DATA_PART_C:
case H264NalUnitTypes.CODED_SLICE_IDR:
final FirstVclNalDetector current = new FirstVclNalDetector(nal, nalUnitHeader.nal_ref_idc, nalUnitHeader.nal_unit_type);
if (fvnd != null && fvnd.isFirstInNew(current)) {
pushSample(createSample(bufferedNals, fvnd.sliceHeader, sliceNalUnitHeader, presentationTimeUs - currentPresentationTimeUs), false, false);
bufferedNals.clear();
}
currentPresentationTimeUs = Math.max(currentPresentationTimeUs, presentationTimeUs);
sliceNalUnitHeader = nalUnitHeader;
fvnd = current;
bufferedNals.add(nal);
break;
case H264NalUnitTypes.SEI:
case H264NalUnitTypes.AU_UNIT_DELIMITER:
if (fvnd != null) {
pushSample(createSample(bufferedNals, fvnd.sliceHeader, sliceNalUnitHeader, presentationTimeUs - currentPresentationTimeUs), false, false);
bufferedNals.clear();
fvnd = null;
}
bufferedNals.add(nal);
break;
case H264NalUnitTypes.SEQ_PARAMETER_SET:
if (fvnd != null) {
pushSample(createSample(bufferedNals, fvnd.sliceHeader, sliceNalUnitHeader, presentationTimeUs - currentPresentationTimeUs), false, false);
bufferedNals.clear();
fvnd = null;
}
handleSPS(nal);
break;
case H264NalUnitTypes.PIC_PARAMETER_SET:
if (fvnd != null) {
pushSample(createSample(bufferedNals, fvnd.sliceHeader, sliceNalUnitHeader, presentationTimeUs - currentPresentationTimeUs), false, false);
bufferedNals.clear();
fvnd = null;
}
handlePPS(nal);
break;
case H264NalUnitTypes.END_OF_SEQUENCE:
case H264NalUnitTypes.END_OF_STREAM:
return;
case H264NalUnitTypes.SEQ_PARAMETER_SET_EXT:
throw new IOException("Sequence parameter set extension is not yet handled. Needs TLC.");
default:
Log.w(TAG, "Unknown NAL unit type: " + nalUnitHeader.nal_unit_type);
}
}
void consumeLastNal() throws IOException {
pushSample(createSample(bufferedNals, fvnd.sliceHeader, sliceNalUnitHeader, 0), true, true);
}
private void pushSample(final StreamingSample ss, final boolean all, final boolean force) throws IOException {
if (ss != null) {
decFrameBuffer.add(ss);
}
if (all) {
while (decFrameBuffer.size() > 0) {
pushSample(null, false, true);
}
} else {
if ((decFrameBuffer.size() - 1 > maxDecFrameBuffering) || force) {
final StreamingSample first = decFrameBuffer.remove(0);
final PictureOrderCountType0SampleExtension poct0se = first.getSampleExtension(PictureOrderCountType0SampleExtension.class);
if (poct0se == null) {
sampleSink.acceptSample(first, this);
} else {
int delay = 0;
for (StreamingSample streamingSample : decFrameBuffer) {
if (poct0se.getPoc() > streamingSample.getSampleExtension(PictureOrderCountType0SampleExtension.class).getPoc()) {
delay++;
}
}
for (StreamingSample streamingSample : decFrameBuffer2) {
if (poct0se.getPoc() < streamingSample.getSampleExtension(PictureOrderCountType0SampleExtension.class).getPoc()) {
delay--;
}
}
decFrameBuffer2.add(first);
if (decFrameBuffer2.size() > maxDecFrameBuffering) {
decFrameBuffer2.remove(0).removeSampleExtension(PictureOrderCountType0SampleExtension.class);
}
first.addSampleExtension(CompositionTimeSampleExtension.create(delay * frametick));
sampleSink.acceptSample(first, this);
}
}
}
}
private SampleFlagsSampleExtension createSampleFlagsSampleExtension(H264NalUnitHeader nu, SliceHeader sliceHeader) {
final SampleFlagsSampleExtension sampleFlagsSampleExtension = new SampleFlagsSampleExtension();
if (nu.nal_ref_idc == 0) {
sampleFlagsSampleExtension.setSampleIsDependedOn(2);
} else {
sampleFlagsSampleExtension.setSampleIsDependedOn(1);
}
if ((sliceHeader.slice_type == SliceHeader.SliceType.I) || (sliceHeader.slice_type == SliceHeader.SliceType.SI)) {
sampleFlagsSampleExtension.setSampleDependsOn(2);
} else {
sampleFlagsSampleExtension.setSampleDependsOn(1);
}
sampleFlagsSampleExtension.setSampleIsNonSyncSample(H264NalUnitTypes.CODED_SLICE_IDR != nu.nal_unit_type);
return sampleFlagsSampleExtension;
}
private PictureOrderCountType0SampleExtension createPictureOrderCountType0SampleExtension(SliceHeader sliceHeader) {
if (sliceHeader.sps.pic_order_cnt_type == 0) {
return new PictureOrderCountType0SampleExtension(
sliceHeader, decFrameBuffer.size() > 0 ?
decFrameBuffer.get(decFrameBuffer.size() - 1).getSampleExtension(PictureOrderCountType0SampleExtension.class) :
null);
/* decFrameBuffer.add(ssi);
if (decFrameBuffer.size() - 1 > maxDecFrameBuffering) { // just added one
drainDecPictureBuffer(false);
}*/
} else if (sliceHeader.sps.pic_order_cnt_type == 1) {
throw new MuxingException("pic_order_cnt_type == 1 needs to be implemented");
} else if (sliceHeader.sps.pic_order_cnt_type == 2) {
return null; // no ctts
}
throw new MuxingException("I don't know sliceHeader.sps.pic_order_cnt_type of " + sliceHeader.sps.pic_order_cnt_type);
}
private StreamingSample createSample(List<ByteBuffer> nals, SliceHeader sliceHeader, H264NalUnitHeader nu, long sampleDurationNs) {
final long sampleDuration = getTimescale() * Math.max(0, sampleDurationNs) / 1000000L;
final StreamingSample ss = new StreamingSampleImpl(nals, sampleDuration);
ss.addSampleExtension(createSampleFlagsSampleExtension(nu, sliceHeader));
final SampleExtension pictureOrderCountType0SampleExtension = createPictureOrderCountType0SampleExtension(sliceHeader);
if (pictureOrderCountType0SampleExtension != null) {
ss.addSampleExtension(pictureOrderCountType0SampleExtension);
}
return ss;
}
private void handlePPS(final @NonNull ByteBuffer nal) {
nal.position(1);
try {
final PictureParameterSet _pictureParameterSet = PictureParameterSet.read(nal);
final ByteBuffer oldPpsSameId = ppsIdToPpsBytes.get(_pictureParameterSet.pic_parameter_set_id);
if (oldPpsSameId != null && !oldPpsSameId.equals(nal)) {
throw new MuxingException("OMG - I got two SPS with same ID but different settings! (AVC3 is the solution)");
} else {
ppsIdToPpsBytes.put(_pictureParameterSet.pic_parameter_set_id, nal);
ppsIdToPps.put(_pictureParameterSet.pic_parameter_set_id, _pictureParameterSet);
}
} catch (IOException e) {
throw new MuxingException("That's surprising to get IOException when working on ByteArrayInputStream", e);
}
}
private @NonNull SeqParameterSet handleSPS(final @NonNull ByteBuffer nal) {
nal.position(1);
try {
final SeqParameterSet seqParameterSet = SeqParameterSet.read(nal);
final ByteBuffer oldSpsSameId = spsIdToSpsBytes.get(seqParameterSet.seq_parameter_set_id);
if (oldSpsSameId != null && !oldSpsSameId.equals(nal)) {
throw new MuxingException("OMG - I got two SPS with same ID but different settings!");
} else {
spsIdToSpsBytes.put(seqParameterSet.seq_parameter_set_id, nal);
spsIdToSps.put(seqParameterSet.seq_parameter_set_id, seqParameterSet);
}
return seqParameterSet;
} catch (IOException e) {
throw new MuxingException("That's surprising to get IOException when working on ByteArrayInputStream", e);
}
}
class FirstVclNalDetector {
final SliceHeader sliceHeader;
final int frame_num;
final int pic_parameter_set_id;
final boolean field_pic_flag;
final boolean bottom_field_flag;
final int nal_ref_idc;
final int pic_order_cnt_type;
final int delta_pic_order_cnt_bottom;
final int pic_order_cnt_lsb;
final int delta_pic_order_cnt_0;
final int delta_pic_order_cnt_1;
final int idr_pic_id;
FirstVclNalDetector(ByteBuffer nal, int nal_ref_idc, int nal_unit_type) {
SliceHeader sh = new SliceHeader(nal, spsIdToSps, ppsIdToPps, nal_unit_type == 5);
this.sliceHeader = sh;
this.frame_num = sh.frame_num;
this.pic_parameter_set_id = sh.pic_parameter_set_id;
this.field_pic_flag = sh.field_pic_flag;
this.bottom_field_flag = sh.bottom_field_flag;
this.nal_ref_idc = nal_ref_idc;
this.pic_order_cnt_type = spsIdToSps.get(ppsIdToPps.get(sh.pic_parameter_set_id).seq_parameter_set_id).pic_order_cnt_type;
this.delta_pic_order_cnt_bottom = sh.delta_pic_order_cnt_bottom;
this.pic_order_cnt_lsb = sh.pic_order_cnt_lsb;
this.delta_pic_order_cnt_0 = sh.delta_pic_order_cnt_0;
this.delta_pic_order_cnt_1 = sh.delta_pic_order_cnt_1;
this.idr_pic_id = sh.idr_pic_id;
}
boolean isFirstInNew(FirstVclNalDetector nu) {
if (nu.frame_num != frame_num) {
return true;
}
if (nu.pic_parameter_set_id != pic_parameter_set_id) {
return true;
}
if (nu.field_pic_flag != field_pic_flag) {
return true;
}
if (nu.field_pic_flag) {
if (nu.bottom_field_flag != bottom_field_flag) {
return true;
}
}
if (nu.nal_ref_idc != nal_ref_idc) {
return true;
}
if (nu.pic_order_cnt_type == 0 && pic_order_cnt_type == 0) {
if (nu.pic_order_cnt_lsb != pic_order_cnt_lsb) {
return true;
}
if (nu.delta_pic_order_cnt_bottom != delta_pic_order_cnt_bottom) {
return true;
}
}
if (nu.pic_order_cnt_type == 1 && pic_order_cnt_type == 1) {
if (nu.delta_pic_order_cnt_0 != delta_pic_order_cnt_0) {
return true;
}
if (nu.delta_pic_order_cnt_1 != delta_pic_order_cnt_1) {
return true;
}
}
return false;
}
}
static class PictureOrderCountType0SampleExtension implements SampleExtension {
int picOrderCntMsb;
int picOrderCountLsb;
PictureOrderCountType0SampleExtension(final @NonNull SliceHeader currentSlice, final @Nullable PictureOrderCountType0SampleExtension previous) {
int prevPicOrderCntLsb = 0;
int prevPicOrderCntMsb = 0;
if (previous != null) {
prevPicOrderCntLsb = previous.picOrderCountLsb;
prevPicOrderCntMsb = previous.picOrderCntMsb;
}
final int maxPicOrderCountLsb = (1 << (currentSlice.sps.log2_max_pic_order_cnt_lsb_minus4 + 4));
// System.out.print(" pic_order_cnt_lsb " + pic_order_cnt_lsb + " " + max_pic_order_count);
picOrderCountLsb = currentSlice.pic_order_cnt_lsb;
picOrderCntMsb = 0;
if ((picOrderCountLsb < prevPicOrderCntLsb) && ((prevPicOrderCntLsb - picOrderCountLsb) >= (maxPicOrderCountLsb / 2))) {
picOrderCntMsb = prevPicOrderCntMsb + maxPicOrderCountLsb;
} else if ((picOrderCountLsb > prevPicOrderCntLsb) && ((picOrderCountLsb - prevPicOrderCntLsb) > (maxPicOrderCountLsb / 2))) {
picOrderCntMsb = prevPicOrderCntMsb - maxPicOrderCountLsb;
} else {
picOrderCntMsb = prevPicOrderCntMsb;
}
}
int getPoc() {
return picOrderCntMsb + picOrderCountLsb;
}
@NonNull
@Override
public String toString() {
return "picOrderCntMsb=" + picOrderCntMsb + ", picOrderCountLsb=" + picOrderCountLsb;
}
}
}
package org.thoughtcrime.securesms.video.videoconverter.muxer;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import org.mp4parser.boxes.iso14496.part12.SampleDescriptionBox;
import org.mp4parser.boxes.iso14496.part15.AvcConfigurationBox;
import org.mp4parser.boxes.sampleentry.VisualSampleEntry;
import org.mp4parser.streaming.SampleExtension;
import org.mp4parser.streaming.StreamingSample;
import org.mp4parser.streaming.extensions.CompositionTimeSampleExtension;
import org.mp4parser.streaming.extensions.CompositionTimeTrackExtension;
import org.mp4parser.streaming.extensions.DimensionTrackExtension;
import org.mp4parser.streaming.extensions.SampleFlagsSampleExtension;
import org.mp4parser.streaming.input.AbstractStreamingTrack;
import org.mp4parser.streaming.input.StreamingSampleImpl;
import org.mp4parser.streaming.input.h264.H264NalUnitHeader;
import org.mp4parser.streaming.input.h264.H264NalUnitTypes;
import org.mp4parser.streaming.input.h264.spspps.PictureParameterSet;
import org.mp4parser.streaming.input.h264.spspps.SeqParameterSet;
import org.mp4parser.streaming.input.h264.spspps.SliceHeader;
import org.signal.core.util.logging.Log;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.List;
abstract class AvcTrack extends AbstractStreamingTrack {
private static final String TAG = "AvcTrack";
private int maxDecFrameBuffering = 16;
private final List<StreamingSample> decFrameBuffer = new ArrayList<>();
private final List<StreamingSample> decFrameBuffer2 = new ArrayList<>();
private final LinkedHashMap<Integer, ByteBuffer> spsIdToSpsBytes = new LinkedHashMap<>();
private final LinkedHashMap<Integer, SeqParameterSet> spsIdToSps = new LinkedHashMap<>();
private final LinkedHashMap<Integer, ByteBuffer> ppsIdToPpsBytes = new LinkedHashMap<>();
private final LinkedHashMap<Integer, PictureParameterSet> ppsIdToPps = new LinkedHashMap<>();
private int timescale = 90000;
private int frametick = 3000;
private final SampleDescriptionBox stsd;
private final List<ByteBuffer> bufferedNals = new ArrayList<>();
private FirstVclNalDetector fvnd;
private H264NalUnitHeader sliceNalUnitHeader;
private long currentPresentationTimeUs;
AvcTrack(final @NonNull ByteBuffer spsBuffer, final @NonNull ByteBuffer ppsBuffer) {
handlePPS(ppsBuffer);
final SeqParameterSet sps = handleSPS(spsBuffer);
int width = (sps.pic_width_in_mbs_minus1 + 1) * 16;
int mult = 2;
if (sps.frame_mbs_only_flag) {
mult = 1;
}
int height = 16 * (sps.pic_height_in_map_units_minus1 + 1) * mult;
if (sps.frame_cropping_flag) {
int chromaArrayType = 0;
if (!sps.residual_color_transform_flag) {
chromaArrayType = sps.chroma_format_idc.getId();
}
int cropUnitX = 1;
int cropUnitY = mult;
if (chromaArrayType != 0) {
cropUnitX = sps.chroma_format_idc.getSubWidth();
cropUnitY = sps.chroma_format_idc.getSubHeight() * mult;
}
width -= cropUnitX * (sps.frame_crop_left_offset + sps.frame_crop_right_offset);
height -= cropUnitY * (sps.frame_crop_top_offset + sps.frame_crop_bottom_offset);
}
final VisualSampleEntry visualSampleEntry = new VisualSampleEntry("avc1");
visualSampleEntry.setDataReferenceIndex(1);
visualSampleEntry.setDepth(24);
visualSampleEntry.setFrameCount(1);
visualSampleEntry.setHorizresolution(72);
visualSampleEntry.setVertresolution(72);
final DimensionTrackExtension dte = this.getTrackExtension(DimensionTrackExtension.class);
if (dte == null) {
this.addTrackExtension(new DimensionTrackExtension(width, height));
}
visualSampleEntry.setWidth(width);
visualSampleEntry.setHeight(height);
visualSampleEntry.setCompressorname("AVC Coding");
final AvcConfigurationBox avcConfigurationBox = new AvcConfigurationBox();
avcConfigurationBox.setSequenceParameterSets(Collections.singletonList(spsBuffer));
avcConfigurationBox.setPictureParameterSets(Collections.singletonList(ppsBuffer));
avcConfigurationBox.setAvcLevelIndication(sps.level_idc);
avcConfigurationBox.setAvcProfileIndication(sps.profile_idc);
avcConfigurationBox.setBitDepthLumaMinus8(sps.bit_depth_luma_minus8);
avcConfigurationBox.setBitDepthChromaMinus8(sps.bit_depth_chroma_minus8);
avcConfigurationBox.setChromaFormat(sps.chroma_format_idc.getId());
avcConfigurationBox.setConfigurationVersion(1);
avcConfigurationBox.setLengthSizeMinusOne(3);
avcConfigurationBox.setProfileCompatibility(
(sps.constraint_set_0_flag ? 128 : 0) +
(sps.constraint_set_1_flag ? 64 : 0) +
(sps.constraint_set_2_flag ? 32 : 0) +
(sps.constraint_set_3_flag ? 16 : 0) +
(sps.constraint_set_4_flag ? 8 : 0) +
(int) (sps.reserved_zero_2bits & 0x3)
);
visualSampleEntry.addBox(avcConfigurationBox);
stsd = new SampleDescriptionBox();
stsd.addBox(visualSampleEntry);
int _timescale;
int _frametick;
if (sps.vuiParams != null) {
_timescale = sps.vuiParams.time_scale >> 1; // Not sure why, but I found this in several places, and it works...
_frametick = sps.vuiParams.num_units_in_tick;
if (_timescale == 0 || _frametick == 0) {
Log.w(TAG, "vuiParams contain invalid values: time_scale: " + _timescale + " and frame_tick: " + _frametick + ". Setting frame rate to 30fps");
_timescale = 0;
_frametick = 0;
}
if (_frametick > 0) {
if (_timescale / _frametick > 100) {
Log.w(TAG, "Framerate is " + (_timescale / _frametick) + ". That is suspicious.");
}
} else {
Log.w(TAG, "Frametick is " + _frametick + ". That is suspicious.");
}
if (sps.vuiParams.bitstreamRestriction != null) {
maxDecFrameBuffering = sps.vuiParams.bitstreamRestriction.max_dec_frame_buffering;
}
} else {
Log.w(TAG, "Can't determine frame rate as SPS does not contain vuiParama");
_timescale = 0;
_frametick = 0;
}
if (_timescale != 0 && _frametick != 0) {
timescale = _timescale;
frametick = _frametick;
}
if (sps.pic_order_cnt_type == 0) {
addTrackExtension(new CompositionTimeTrackExtension());
} else if (sps.pic_order_cnt_type == 1) {
throw new MuxingException("Have not yet imlemented pic_order_cnt_type 1");
}
}
public long getTimescale() {
return timescale;
}
public String getHandler() {
return "vide";
}
public String getLanguage() {
return "\u0060\u0060\u0060"; // 0 in Iso639
}
public SampleDescriptionBox getSampleDescriptionBox() {
return stsd;
}
public void close() {
}
private static H264NalUnitHeader getNalUnitHeader(@NonNull final ByteBuffer nal) {
final H264NalUnitHeader nalUnitHeader = new H264NalUnitHeader();
final int type = nal.get(0);
nalUnitHeader.nal_ref_idc = (type >> 5) & 3;
nalUnitHeader.nal_unit_type = type & 0x1f;
return nalUnitHeader;
}
void consumeNal(@NonNull final ByteBuffer nal, final long presentationTimeUs) throws IOException {
final H264NalUnitHeader nalUnitHeader = getNalUnitHeader(nal);
switch (nalUnitHeader.nal_unit_type) {
case H264NalUnitTypes.CODED_SLICE_NON_IDR:
case H264NalUnitTypes.CODED_SLICE_DATA_PART_A:
case H264NalUnitTypes.CODED_SLICE_DATA_PART_B:
case H264NalUnitTypes.CODED_SLICE_DATA_PART_C:
case H264NalUnitTypes.CODED_SLICE_IDR:
final FirstVclNalDetector current = new FirstVclNalDetector(nal, nalUnitHeader.nal_ref_idc, nalUnitHeader.nal_unit_type);
if (fvnd != null && fvnd.isFirstInNew(current)) {
pushSample(createSample(bufferedNals, fvnd.sliceHeader, sliceNalUnitHeader, presentationTimeUs - currentPresentationTimeUs), false, false);
bufferedNals.clear();
}
currentPresentationTimeUs = Math.max(currentPresentationTimeUs, presentationTimeUs);
sliceNalUnitHeader = nalUnitHeader;
fvnd = current;
bufferedNals.add(nal);
break;
case H264NalUnitTypes.SEI:
case H264NalUnitTypes.AU_UNIT_DELIMITER:
if (fvnd != null) {
pushSample(createSample(bufferedNals, fvnd.sliceHeader, sliceNalUnitHeader, presentationTimeUs - currentPresentationTimeUs), false, false);
bufferedNals.clear();
fvnd = null;
}
bufferedNals.add(nal);
break;
case H264NalUnitTypes.SEQ_PARAMETER_SET:
if (fvnd != null) {
pushSample(createSample(bufferedNals, fvnd.sliceHeader, sliceNalUnitHeader, presentationTimeUs - currentPresentationTimeUs), false, false);
bufferedNals.clear();
fvnd = null;
}
handleSPS(nal);
break;
case H264NalUnitTypes.PIC_PARAMETER_SET:
if (fvnd != null) {
pushSample(createSample(bufferedNals, fvnd.sliceHeader, sliceNalUnitHeader, presentationTimeUs - currentPresentationTimeUs), false, false);
bufferedNals.clear();
fvnd = null;
}
handlePPS(nal);
break;
case H264NalUnitTypes.END_OF_SEQUENCE:
case H264NalUnitTypes.END_OF_STREAM:
return;
case H264NalUnitTypes.SEQ_PARAMETER_SET_EXT:
throw new IOException("Sequence parameter set extension is not yet handled. Needs TLC.");
default:
Log.w(TAG, "Unknown NAL unit type: " + nalUnitHeader.nal_unit_type);
}
}
void consumeLastNal() throws IOException {
pushSample(createSample(bufferedNals, fvnd.sliceHeader, sliceNalUnitHeader, 0), true, true);
}
private void pushSample(final StreamingSample ss, final boolean all, final boolean force) throws IOException {
if (ss != null) {
decFrameBuffer.add(ss);
}
if (all) {
while (decFrameBuffer.size() > 0) {
pushSample(null, false, true);
}
} else {
if ((decFrameBuffer.size() - 1 > maxDecFrameBuffering) || force) {
final StreamingSample first = decFrameBuffer.remove(0);
final PictureOrderCountType0SampleExtension poct0se = first.getSampleExtension(PictureOrderCountType0SampleExtension.class);
if (poct0se == null) {
sampleSink.acceptSample(first, this);
} else {
int delay = 0;
for (StreamingSample streamingSample : decFrameBuffer) {
if (poct0se.getPoc() > streamingSample.getSampleExtension(PictureOrderCountType0SampleExtension.class).getPoc()) {
delay++;
}
}
for (StreamingSample streamingSample : decFrameBuffer2) {
if (poct0se.getPoc() < streamingSample.getSampleExtension(PictureOrderCountType0SampleExtension.class).getPoc()) {
delay--;
}
}
decFrameBuffer2.add(first);
if (decFrameBuffer2.size() > maxDecFrameBuffering) {
decFrameBuffer2.remove(0).removeSampleExtension(PictureOrderCountType0SampleExtension.class);
}
first.addSampleExtension(CompositionTimeSampleExtension.create(delay * frametick));
sampleSink.acceptSample(first, this);
}
}
}
}
private SampleFlagsSampleExtension createSampleFlagsSampleExtension(H264NalUnitHeader nu, SliceHeader sliceHeader) {
final SampleFlagsSampleExtension sampleFlagsSampleExtension = new SampleFlagsSampleExtension();
if (nu.nal_ref_idc == 0) {
sampleFlagsSampleExtension.setSampleIsDependedOn(2);
} else {
sampleFlagsSampleExtension.setSampleIsDependedOn(1);
}
if ((sliceHeader.slice_type == SliceHeader.SliceType.I) || (sliceHeader.slice_type == SliceHeader.SliceType.SI)) {
sampleFlagsSampleExtension.setSampleDependsOn(2);
} else {
sampleFlagsSampleExtension.setSampleDependsOn(1);
}
sampleFlagsSampleExtension.setSampleIsNonSyncSample(H264NalUnitTypes.CODED_SLICE_IDR != nu.nal_unit_type);
return sampleFlagsSampleExtension;
}
private PictureOrderCountType0SampleExtension createPictureOrderCountType0SampleExtension(SliceHeader sliceHeader) {
if (sliceHeader.sps.pic_order_cnt_type == 0) {
return new PictureOrderCountType0SampleExtension(
sliceHeader, decFrameBuffer.size() > 0 ?
decFrameBuffer.get(decFrameBuffer.size() - 1).getSampleExtension(PictureOrderCountType0SampleExtension.class) :
null);
/* decFrameBuffer.add(ssi);
if (decFrameBuffer.size() - 1 > maxDecFrameBuffering) { // just added one
drainDecPictureBuffer(false);
}*/
} else if (sliceHeader.sps.pic_order_cnt_type == 1) {
throw new MuxingException("pic_order_cnt_type == 1 needs to be implemented");
} else if (sliceHeader.sps.pic_order_cnt_type == 2) {
return null; // no ctts
}
throw new MuxingException("I don't know sliceHeader.sps.pic_order_cnt_type of " + sliceHeader.sps.pic_order_cnt_type);
}
private StreamingSample createSample(List<ByteBuffer> nals, SliceHeader sliceHeader, H264NalUnitHeader nu, long sampleDurationNs) {
final long sampleDuration = getTimescale() * Math.max(0, sampleDurationNs) / 1000000L;
final StreamingSample ss = new StreamingSampleImpl(nals, sampleDuration);
ss.addSampleExtension(createSampleFlagsSampleExtension(nu, sliceHeader));
final SampleExtension pictureOrderCountType0SampleExtension = createPictureOrderCountType0SampleExtension(sliceHeader);
if (pictureOrderCountType0SampleExtension != null) {
ss.addSampleExtension(pictureOrderCountType0SampleExtension);
}
return ss;
}
private void handlePPS(final @NonNull ByteBuffer nal) {
nal.position(1);
try {
final PictureParameterSet _pictureParameterSet = PictureParameterSet.read(nal);
final ByteBuffer oldPpsSameId = ppsIdToPpsBytes.get(_pictureParameterSet.pic_parameter_set_id);
if (oldPpsSameId != null && !oldPpsSameId.equals(nal)) {
throw new MuxingException("OMG - I got two SPS with same ID but different settings! (AVC3 is the solution)");
} else {
ppsIdToPpsBytes.put(_pictureParameterSet.pic_parameter_set_id, nal);
ppsIdToPps.put(_pictureParameterSet.pic_parameter_set_id, _pictureParameterSet);
}
} catch (IOException e) {
throw new MuxingException("That's surprising to get IOException when working on ByteArrayInputStream", e);
}
}
private @NonNull SeqParameterSet handleSPS(final @NonNull ByteBuffer nal) {
nal.position(1);
try {
final SeqParameterSet seqParameterSet = SeqParameterSet.read(nal);
final ByteBuffer oldSpsSameId = spsIdToSpsBytes.get(seqParameterSet.seq_parameter_set_id);
if (oldSpsSameId != null && !oldSpsSameId.equals(nal)) {
throw new MuxingException("OMG - I got two SPS with same ID but different settings!");
} else {
spsIdToSpsBytes.put(seqParameterSet.seq_parameter_set_id, nal);
spsIdToSps.put(seqParameterSet.seq_parameter_set_id, seqParameterSet);
}
return seqParameterSet;
} catch (IOException e) {
throw new MuxingException("That's surprising to get IOException when working on ByteArrayInputStream", e);
}
}
class FirstVclNalDetector {
final SliceHeader sliceHeader;
final int frame_num;
final int pic_parameter_set_id;
final boolean field_pic_flag;
final boolean bottom_field_flag;
final int nal_ref_idc;
final int pic_order_cnt_type;
final int delta_pic_order_cnt_bottom;
final int pic_order_cnt_lsb;
final int delta_pic_order_cnt_0;
final int delta_pic_order_cnt_1;
final int idr_pic_id;
FirstVclNalDetector(ByteBuffer nal, int nal_ref_idc, int nal_unit_type) {
SliceHeader sh = new SliceHeader(nal, spsIdToSps, ppsIdToPps, nal_unit_type == 5);
this.sliceHeader = sh;
this.frame_num = sh.frame_num;
this.pic_parameter_set_id = sh.pic_parameter_set_id;
this.field_pic_flag = sh.field_pic_flag;
this.bottom_field_flag = sh.bottom_field_flag;
this.nal_ref_idc = nal_ref_idc;
this.pic_order_cnt_type = spsIdToSps.get(ppsIdToPps.get(sh.pic_parameter_set_id).seq_parameter_set_id).pic_order_cnt_type;
this.delta_pic_order_cnt_bottom = sh.delta_pic_order_cnt_bottom;
this.pic_order_cnt_lsb = sh.pic_order_cnt_lsb;
this.delta_pic_order_cnt_0 = sh.delta_pic_order_cnt_0;
this.delta_pic_order_cnt_1 = sh.delta_pic_order_cnt_1;
this.idr_pic_id = sh.idr_pic_id;
}
boolean isFirstInNew(FirstVclNalDetector nu) {
if (nu.frame_num != frame_num) {
return true;
}
if (nu.pic_parameter_set_id != pic_parameter_set_id) {
return true;
}
if (nu.field_pic_flag != field_pic_flag) {
return true;
}
if (nu.field_pic_flag) {
if (nu.bottom_field_flag != bottom_field_flag) {
return true;
}
}
if (nu.nal_ref_idc != nal_ref_idc) {
return true;
}
if (nu.pic_order_cnt_type == 0 && pic_order_cnt_type == 0) {
if (nu.pic_order_cnt_lsb != pic_order_cnt_lsb) {
return true;
}
if (nu.delta_pic_order_cnt_bottom != delta_pic_order_cnt_bottom) {
return true;
}
}
if (nu.pic_order_cnt_type == 1 && pic_order_cnt_type == 1) {
if (nu.delta_pic_order_cnt_0 != delta_pic_order_cnt_0) {
return true;
}
if (nu.delta_pic_order_cnt_1 != delta_pic_order_cnt_1) {
return true;
}
}
return false;
}
}
static class PictureOrderCountType0SampleExtension implements SampleExtension {
int picOrderCntMsb;
int picOrderCountLsb;
PictureOrderCountType0SampleExtension(final @NonNull SliceHeader currentSlice, final @Nullable PictureOrderCountType0SampleExtension previous) {
int prevPicOrderCntLsb = 0;
int prevPicOrderCntMsb = 0;
if (previous != null) {
prevPicOrderCntLsb = previous.picOrderCountLsb;
prevPicOrderCntMsb = previous.picOrderCntMsb;
}
final int maxPicOrderCountLsb = (1 << (currentSlice.sps.log2_max_pic_order_cnt_lsb_minus4 + 4));
// System.out.print(" pic_order_cnt_lsb " + pic_order_cnt_lsb + " " + max_pic_order_count);
picOrderCountLsb = currentSlice.pic_order_cnt_lsb;
picOrderCntMsb = 0;
if ((picOrderCountLsb < prevPicOrderCntLsb) && ((prevPicOrderCntLsb - picOrderCountLsb) >= (maxPicOrderCountLsb / 2))) {
picOrderCntMsb = prevPicOrderCntMsb + maxPicOrderCountLsb;
} else if ((picOrderCountLsb > prevPicOrderCntLsb) && ((picOrderCountLsb - prevPicOrderCntLsb) > (maxPicOrderCountLsb / 2))) {
picOrderCntMsb = prevPicOrderCntMsb - maxPicOrderCountLsb;
} else {
picOrderCntMsb = prevPicOrderCntMsb;
}
}
int getPoc() {
return picOrderCntMsb + picOrderCountLsb;
}
@NonNull
@Override
public String toString() {
return "picOrderCntMsb=" + picOrderCntMsb + ", picOrderCountLsb=" + picOrderCountLsb;
}
}
}

198
lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/muxer/H264Utils.java
View File

@@ -1,99 +1,99 @@
/*
* Copyright 2008-2019 JCodecProject
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer. Redistributions in binary form
* must reproduce the above copyright notice, this list of conditions and the
* following disclaimer in the documentation and/or other materials provided with
* the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
* ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
* ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* https://github.com/jcodec/jcodec/blob/master/src/main/java/org/jcodec/codecs/h264/H264Utils.java
*
* This file has been modified by Signal.
*/
package org.thoughtcrime.securesms.video.videoconverter.muxer;
import androidx.annotation.NonNull;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.util.ArrayList;
import java.util.List;
final class H264Utils {
private H264Utils() {}
static @NonNull List<ByteBuffer> getNals(ByteBuffer buffer) {
final List<ByteBuffer> nals = new ArrayList<>();
ByteBuffer nal;
while ((nal = nextNALUnit(buffer)) != null) {
nals.add(nal);
}
return nals;
}
static ByteBuffer nextNALUnit(ByteBuffer buf) {
skipToNALUnit(buf);
return gotoNALUnit(buf);
}
static void skipToNALUnit(ByteBuffer buf) {
if (!buf.hasRemaining())
return;
int val = 0xffffffff;
while (buf.hasRemaining()) {
val <<= 8;
val |= (buf.get() & 0xff);
if ((val & 0xffffff) == 1) {
buf.position(buf.position());
break;
}
}
}
/**
* Finds next Nth H.264 bitstream NAL unit (0x00000001) and returns the data
* that preceeds it as a ByteBuffer slice
* <p>
* Segment byte order is always little endian
* <p>
* TODO: emulation prevention
*/
static ByteBuffer gotoNALUnit(ByteBuffer buf) {
if (!buf.hasRemaining())
return null;
int from = buf.position();
ByteBuffer result = buf.slice();
result.order(ByteOrder.BIG_ENDIAN);
int val = 0xffffffff;
while (buf.hasRemaining()) {
val <<= 8;
val |= (buf.get() & 0xff);
if ((val & 0xffffff) == 1) {
buf.position(buf.position() - (val == 1 ? 4 : 3));
result.limit(buf.position() - from);
break;
}
}
return result;
}
}
/*
* Copyright 2008-2019 JCodecProject
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer. Redistributions in binary form
* must reproduce the above copyright notice, this list of conditions and the
* following disclaimer in the documentation and/or other materials provided with
* the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
* ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
* ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* https://github.com/jcodec/jcodec/blob/master/src/main/java/org/jcodec/codecs/h264/H264Utils.java
*
* This file has been modified by Signal.
*/
package org.thoughtcrime.securesms.video.videoconverter.muxer;
import androidx.annotation.NonNull;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.util.ArrayList;
import java.util.List;
final class H264Utils {
private H264Utils() {}
static @NonNull List<ByteBuffer> getNals(ByteBuffer buffer) {
final List<ByteBuffer> nals = new ArrayList<>();
ByteBuffer nal;
while ((nal = nextNALUnit(buffer)) != null) {
nals.add(nal);
}
return nals;
}
static ByteBuffer nextNALUnit(ByteBuffer buf) {
skipToNALUnit(buf);
return gotoNALUnit(buf);
}
static void skipToNALUnit(ByteBuffer buf) {
if (!buf.hasRemaining())
return;
int val = 0xffffffff;
while (buf.hasRemaining()) {
val <<= 8;
val |= (buf.get() & 0xff);
if ((val & 0xffffff) == 1) {
buf.position(buf.position());
break;
}
}
}
/**
* Finds next Nth H.264 bitstream NAL unit (0x00000001) and returns the data
* that preceeds it as a ByteBuffer slice
* <p>
* Segment byte order is always little endian
* <p>
* TODO: emulation prevention
*/
static ByteBuffer gotoNALUnit(ByteBuffer buf) {
if (!buf.hasRemaining())
return null;
int from = buf.position();
ByteBuffer result = buf.slice();
result.order(ByteOrder.BIG_ENDIAN);
int val = 0xffffffff;
while (buf.hasRemaining()) {
val <<= 8;
val |= (buf.get() & 0xff);
if ((val & 0xffffff) == 1) {
buf.position(buf.position() - (val == 1 ? 4 : 3));
result.limit(buf.position() - from);
break;
}
}
return result;
}
}

522
lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/muxer/HevcTrack.java
View File

@@ -1,261 +1,261 @@
package org.thoughtcrime.securesms.video.videoconverter.muxer;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import org.mp4parser.boxes.iso14496.part12.SampleDescriptionBox;
import org.mp4parser.boxes.iso14496.part15.HevcConfigurationBox;
import org.mp4parser.boxes.iso14496.part15.HevcDecoderConfigurationRecord;
import org.mp4parser.boxes.sampleentry.VisualSampleEntry;
import org.mp4parser.muxer.tracks.CleanInputStream;
import org.mp4parser.muxer.tracks.h265.H265NalUnitHeader;
import org.mp4parser.muxer.tracks.h265.H265NalUnitTypes;
import org.mp4parser.muxer.tracks.h265.SequenceParameterSetRbsp;
import org.mp4parser.streaming.StreamingSample;
import org.mp4parser.streaming.extensions.DimensionTrackExtension;
import org.mp4parser.streaming.extensions.SampleFlagsSampleExtension;
import org.mp4parser.streaming.input.AbstractStreamingTrack;
import org.mp4parser.streaming.input.StreamingSampleImpl;
import org.mp4parser.tools.ByteBufferByteChannel;
import org.mp4parser.tools.IsoTypeReader;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.Channels;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
abstract class HevcTrack extends AbstractStreamingTrack implements H265NalUnitTypes {
private final ArrayList<ByteBuffer> bufferedNals = new ArrayList<>();
private boolean vclNalUnitSeenInAU;
private boolean isIdr = true;
private long currentPresentationTimeUs;
private final SampleDescriptionBox stsd;
HevcTrack(final @NonNull List<ByteBuffer> csd) throws IOException {
final ArrayList<ByteBuffer> sps = new ArrayList<>();
final ArrayList<ByteBuffer> pps = new ArrayList<>();
final ArrayList<ByteBuffer> vps = new ArrayList<>();
SequenceParameterSetRbsp spsStruct = null;
for (ByteBuffer nal : csd) {
final H265NalUnitHeader unitHeader = getNalUnitHeader(nal);
nal.position(0);
// collect sps/vps/pps
switch (unitHeader.nalUnitType) {
case NAL_TYPE_PPS_NUT:
pps.add(nal.duplicate());
break;
case NAL_TYPE_VPS_NUT:
vps.add(nal.duplicate());
break;
case NAL_TYPE_SPS_NUT:
sps.add(nal.duplicate());
nal.position(2);
spsStruct = new SequenceParameterSetRbsp(new CleanInputStream(Channels.newInputStream(new ByteBufferByteChannel(nal.slice()))));
break;
case NAL_TYPE_PREFIX_SEI_NUT:
//new SEIMessage(new BitReaderBuffer(nal.slice()));
break;
}
}
stsd = new SampleDescriptionBox();
stsd.addBox(createSampleEntry(sps, pps, vps, spsStruct));
}
@Override
public long getTimescale() {
return 90000;
}
@Override
public String getHandler() {
return "vide";
}
@Override
public String getLanguage() {
return "\u0060\u0060\u0060"; // 0 in Iso639
}
@Override
public SampleDescriptionBox getSampleDescriptionBox() {
return stsd;
}
@Override
public void close() {
}
void consumeLastNal() throws IOException {
wrapUp(bufferedNals, currentPresentationTimeUs);
}
void consumeNal(final @NonNull ByteBuffer nal, final long presentationTimeUs) throws IOException {
final H265NalUnitHeader unitHeader = getNalUnitHeader(nal);
final boolean isVcl = isVcl(unitHeader);
//
if (vclNalUnitSeenInAU) { // we need at least 1 VCL per AU
// This branch checks if we encountered the start of a samples/AU
if (isVcl) {
if ((nal.get(2) & -128) != 0) { // this is: first_slice_segment_in_pic_flag u(1)
wrapUp(bufferedNals, presentationTimeUs);
}
} else {
switch (unitHeader.nalUnitType) {
case NAL_TYPE_PREFIX_SEI_NUT:
case NAL_TYPE_AUD_NUT:
case NAL_TYPE_PPS_NUT:
case NAL_TYPE_VPS_NUT:
case NAL_TYPE_SPS_NUT:
case NAL_TYPE_RSV_NVCL41:
case NAL_TYPE_RSV_NVCL42:
case NAL_TYPE_RSV_NVCL43:
case NAL_TYPE_RSV_NVCL44:
case NAL_TYPE_UNSPEC48:
case NAL_TYPE_UNSPEC49:
case NAL_TYPE_UNSPEC50:
case NAL_TYPE_UNSPEC51:
case NAL_TYPE_UNSPEC52:
case NAL_TYPE_UNSPEC53:
case NAL_TYPE_UNSPEC54:
case NAL_TYPE_UNSPEC55:
case NAL_TYPE_EOB_NUT: // a bit special but also causes a sample to be formed
case NAL_TYPE_EOS_NUT:
wrapUp(bufferedNals, presentationTimeUs);
break;
}
}
}
switch (unitHeader.nalUnitType) {
case NAL_TYPE_SPS_NUT:
case NAL_TYPE_VPS_NUT:
case NAL_TYPE_PPS_NUT:
case NAL_TYPE_EOB_NUT:
case NAL_TYPE_EOS_NUT:
case NAL_TYPE_AUD_NUT:
case NAL_TYPE_FD_NUT:
// ignore these
break;
default:
bufferedNals.add(nal);
break;
}
if (isVcl) {
isIdr = unitHeader.nalUnitType == NAL_TYPE_IDR_W_RADL || unitHeader.nalUnitType == NAL_TYPE_IDR_N_LP;
vclNalUnitSeenInAU = true;
}
}
private void wrapUp(final @NonNull List<ByteBuffer> nals, final long presentationTimeUs) throws IOException {
final long duration = presentationTimeUs - currentPresentationTimeUs;
currentPresentationTimeUs = presentationTimeUs;
final StreamingSample sample = new StreamingSampleImpl(
nals, getTimescale() * Math.max(0, duration) / 1000000L);
final SampleFlagsSampleExtension sampleFlagsSampleExtension = new SampleFlagsSampleExtension();
sampleFlagsSampleExtension.setSampleIsNonSyncSample(!isIdr);
sample.addSampleExtension(sampleFlagsSampleExtension);
sampleSink.acceptSample(sample, this);
vclNalUnitSeenInAU = false;
isIdr = true;
nals.clear();
}
private static @NonNull H265NalUnitHeader getNalUnitHeader(final @NonNull ByteBuffer nal) {
nal.position(0);
final int nalUnitHeaderValue = IsoTypeReader.readUInt16(nal);
final H265NalUnitHeader nalUnitHeader = new H265NalUnitHeader();
nalUnitHeader.forbiddenZeroFlag = (nalUnitHeaderValue & 0x8000) >> 15;
nalUnitHeader.nalUnitType = (nalUnitHeaderValue & 0x7E00) >> 9;
nalUnitHeader.nuhLayerId = (nalUnitHeaderValue & 0x1F8) >> 3;
nalUnitHeader.nuhTemporalIdPlusOne = (nalUnitHeaderValue & 0x7);
return nalUnitHeader;
}
private @NonNull VisualSampleEntry createSampleEntry(
final @NonNull ArrayList<ByteBuffer> sps,
final @NonNull ArrayList<ByteBuffer> pps,
final @NonNull ArrayList<ByteBuffer> vps,
final @Nullable SequenceParameterSetRbsp spsStruct)
{
final VisualSampleEntry visualSampleEntry = new VisualSampleEntry("hvc1");
visualSampleEntry.setDataReferenceIndex(1);
visualSampleEntry.setDepth(24);
visualSampleEntry.setFrameCount(1);
visualSampleEntry.setHorizresolution(72);
visualSampleEntry.setVertresolution(72);
visualSampleEntry.setCompressorname("HEVC Coding");
final HevcConfigurationBox hevcConfigurationBox = new HevcConfigurationBox();
hevcConfigurationBox.getHevcDecoderConfigurationRecord().setConfigurationVersion(1);
if (spsStruct != null) {
visualSampleEntry.setWidth(spsStruct.pic_width_in_luma_samples);
visualSampleEntry.setHeight(spsStruct.pic_height_in_luma_samples);
final DimensionTrackExtension dte = this.getTrackExtension(DimensionTrackExtension.class);
if (dte == null) {
this.addTrackExtension(new DimensionTrackExtension(spsStruct.pic_width_in_luma_samples, spsStruct.pic_height_in_luma_samples));
}
final HevcDecoderConfigurationRecord hevcDecoderConfigurationRecord = hevcConfigurationBox.getHevcDecoderConfigurationRecord();
hevcDecoderConfigurationRecord.setChromaFormat(spsStruct.chroma_format_idc);
hevcDecoderConfigurationRecord.setGeneral_profile_idc(spsStruct.general_profile_idc);
hevcDecoderConfigurationRecord.setGeneral_profile_compatibility_flags(spsStruct.general_profile_compatibility_flags);
hevcDecoderConfigurationRecord.setGeneral_constraint_indicator_flags(spsStruct.general_constraint_indicator_flags);
hevcDecoderConfigurationRecord.setGeneral_level_idc(spsStruct.general_level_idc);
hevcDecoderConfigurationRecord.setGeneral_tier_flag(spsStruct.general_tier_flag);
hevcDecoderConfigurationRecord.setGeneral_profile_space(spsStruct.general_profile_space);
hevcDecoderConfigurationRecord.setBitDepthChromaMinus8(spsStruct.bit_depth_chroma_minus8);
hevcDecoderConfigurationRecord.setBitDepthLumaMinus8(spsStruct.bit_depth_luma_minus8);
hevcDecoderConfigurationRecord.setTemporalIdNested(spsStruct.sps_temporal_id_nesting_flag);
}
hevcConfigurationBox.getHevcDecoderConfigurationRecord().setLengthSizeMinusOne(3);
final HevcDecoderConfigurationRecord.Array vpsArray = new HevcDecoderConfigurationRecord.Array();
vpsArray.array_completeness = false;
vpsArray.nal_unit_type = NAL_TYPE_VPS_NUT;
vpsArray.nalUnits = new ArrayList<>();
for (ByteBuffer vp : vps) {
vpsArray.nalUnits.add(Utils.toArray(vp));
}
final HevcDecoderConfigurationRecord.Array spsArray = new HevcDecoderConfigurationRecord.Array();
spsArray.array_completeness = false;
spsArray.nal_unit_type = NAL_TYPE_SPS_NUT;
spsArray.nalUnits = new ArrayList<>();
for (ByteBuffer sp : sps) {
spsArray.nalUnits.add(Utils.toArray(sp));
}
final HevcDecoderConfigurationRecord.Array ppsArray = new HevcDecoderConfigurationRecord.Array();
ppsArray.array_completeness = false;
ppsArray.nal_unit_type = NAL_TYPE_PPS_NUT;
ppsArray.nalUnits = new ArrayList<>();
for (ByteBuffer pp : pps) {
ppsArray.nalUnits.add(Utils.toArray(pp));
}
hevcConfigurationBox.getArrays().addAll(Arrays.asList(spsArray, vpsArray, ppsArray));
visualSampleEntry.addBox(hevcConfigurationBox);
return visualSampleEntry;
}
private boolean isVcl(final @NonNull H265NalUnitHeader nalUnitHeader) {
return nalUnitHeader.nalUnitType >= 0 && nalUnitHeader.nalUnitType <= 31;
}
}
package org.thoughtcrime.securesms.video.videoconverter.muxer;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import org.mp4parser.boxes.iso14496.part12.SampleDescriptionBox;
import org.mp4parser.boxes.iso14496.part15.HevcConfigurationBox;
import org.mp4parser.boxes.iso14496.part15.HevcDecoderConfigurationRecord;
import org.mp4parser.boxes.sampleentry.VisualSampleEntry;
import org.mp4parser.muxer.tracks.CleanInputStream;
import org.mp4parser.muxer.tracks.h265.H265NalUnitHeader;
import org.mp4parser.muxer.tracks.h265.H265NalUnitTypes;
import org.mp4parser.muxer.tracks.h265.SequenceParameterSetRbsp;
import org.mp4parser.streaming.StreamingSample;
import org.mp4parser.streaming.extensions.DimensionTrackExtension;
import org.mp4parser.streaming.extensions.SampleFlagsSampleExtension;
import org.mp4parser.streaming.input.AbstractStreamingTrack;
import org.mp4parser.streaming.input.StreamingSampleImpl;
import org.mp4parser.tools.ByteBufferByteChannel;
import org.mp4parser.tools.IsoTypeReader;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.Channels;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
abstract class HevcTrack extends AbstractStreamingTrack implements H265NalUnitTypes {
private final ArrayList<ByteBuffer> bufferedNals = new ArrayList<>();
private boolean vclNalUnitSeenInAU;
private boolean isIdr = true;
private long currentPresentationTimeUs;
private final SampleDescriptionBox stsd;
HevcTrack(final @NonNull List<ByteBuffer> csd) throws IOException {
final ArrayList<ByteBuffer> sps = new ArrayList<>();
final ArrayList<ByteBuffer> pps = new ArrayList<>();
final ArrayList<ByteBuffer> vps = new ArrayList<>();
SequenceParameterSetRbsp spsStruct = null;
for (ByteBuffer nal : csd) {
final H265NalUnitHeader unitHeader = getNalUnitHeader(nal);
nal.position(0);
// collect sps/vps/pps
switch (unitHeader.nalUnitType) {
case NAL_TYPE_PPS_NUT:
pps.add(nal.duplicate());
break;
case NAL_TYPE_VPS_NUT:
vps.add(nal.duplicate());
break;
case NAL_TYPE_SPS_NUT:
sps.add(nal.duplicate());
nal.position(2);
spsStruct = new SequenceParameterSetRbsp(new CleanInputStream(Channels.newInputStream(new ByteBufferByteChannel(nal.slice()))));
break;
case NAL_TYPE_PREFIX_SEI_NUT:
//new SEIMessage(new BitReaderBuffer(nal.slice()));
break;
}
}
stsd = new SampleDescriptionBox();
stsd.addBox(createSampleEntry(sps, pps, vps, spsStruct));
}
@Override
public long getTimescale() {
return 90000;
}
@Override
public String getHandler() {
return "vide";
}
@Override
public String getLanguage() {
return "\u0060\u0060\u0060"; // 0 in Iso639
}
@Override
public SampleDescriptionBox getSampleDescriptionBox() {
return stsd;
}
@Override
public void close() {
}
void consumeLastNal() throws IOException {
wrapUp(bufferedNals, currentPresentationTimeUs);
}
void consumeNal(final @NonNull ByteBuffer nal, final long presentationTimeUs) throws IOException {
final H265NalUnitHeader unitHeader = getNalUnitHeader(nal);
final boolean isVcl = isVcl(unitHeader);
//
if (vclNalUnitSeenInAU) { // we need at least 1 VCL per AU
// This branch checks if we encountered the start of a samples/AU
if (isVcl) {
if ((nal.get(2) & -128) != 0) { // this is: first_slice_segment_in_pic_flag u(1)
wrapUp(bufferedNals, presentationTimeUs);
}
} else {
switch (unitHeader.nalUnitType) {
case NAL_TYPE_PREFIX_SEI_NUT:
case NAL_TYPE_AUD_NUT:
case NAL_TYPE_PPS_NUT:
case NAL_TYPE_VPS_NUT:
case NAL_TYPE_SPS_NUT:
case NAL_TYPE_RSV_NVCL41:
case NAL_TYPE_RSV_NVCL42:
case NAL_TYPE_RSV_NVCL43:
case NAL_TYPE_RSV_NVCL44:
case NAL_TYPE_UNSPEC48:
case NAL_TYPE_UNSPEC49:
case NAL_TYPE_UNSPEC50:
case NAL_TYPE_UNSPEC51:
case NAL_TYPE_UNSPEC52:
case NAL_TYPE_UNSPEC53:
case NAL_TYPE_UNSPEC54:
case NAL_TYPE_UNSPEC55:
case NAL_TYPE_EOB_NUT: // a bit special but also causes a sample to be formed
case NAL_TYPE_EOS_NUT:
wrapUp(bufferedNals, presentationTimeUs);
break;
}
}
}
switch (unitHeader.nalUnitType) {
case NAL_TYPE_SPS_NUT:
case NAL_TYPE_VPS_NUT:
case NAL_TYPE_PPS_NUT:
case NAL_TYPE_EOB_NUT:
case NAL_TYPE_EOS_NUT:
case NAL_TYPE_AUD_NUT:
case NAL_TYPE_FD_NUT:
// ignore these
break;
default:
bufferedNals.add(nal);
break;
}
if (isVcl) {
isIdr = unitHeader.nalUnitType == NAL_TYPE_IDR_W_RADL || unitHeader.nalUnitType == NAL_TYPE_IDR_N_LP;
vclNalUnitSeenInAU = true;
}
}
private void wrapUp(final @NonNull List<ByteBuffer> nals, final long presentationTimeUs) throws IOException {
final long duration = presentationTimeUs - currentPresentationTimeUs;
currentPresentationTimeUs = presentationTimeUs;
final StreamingSample sample = new StreamingSampleImpl(
nals, getTimescale() * Math.max(0, duration) / 1000000L);
final SampleFlagsSampleExtension sampleFlagsSampleExtension = new SampleFlagsSampleExtension();
sampleFlagsSampleExtension.setSampleIsNonSyncSample(!isIdr);
sample.addSampleExtension(sampleFlagsSampleExtension);
sampleSink.acceptSample(sample, this);
vclNalUnitSeenInAU = false;
isIdr = true;
nals.clear();
}
private static @NonNull H265NalUnitHeader getNalUnitHeader(final @NonNull ByteBuffer nal) {
nal.position(0);
final int nalUnitHeaderValue = IsoTypeReader.readUInt16(nal);
final H265NalUnitHeader nalUnitHeader = new H265NalUnitHeader();
nalUnitHeader.forbiddenZeroFlag = (nalUnitHeaderValue & 0x8000) >> 15;
nalUnitHeader.nalUnitType = (nalUnitHeaderValue & 0x7E00) >> 9;
nalUnitHeader.nuhLayerId = (nalUnitHeaderValue & 0x1F8) >> 3;
nalUnitHeader.nuhTemporalIdPlusOne = (nalUnitHeaderValue & 0x7);
return nalUnitHeader;
}
private @NonNull VisualSampleEntry createSampleEntry(
final @NonNull ArrayList<ByteBuffer> sps,
final @NonNull ArrayList<ByteBuffer> pps,
final @NonNull ArrayList<ByteBuffer> vps,
final @Nullable SequenceParameterSetRbsp spsStruct)
{
final VisualSampleEntry visualSampleEntry = new VisualSampleEntry("hvc1");
visualSampleEntry.setDataReferenceIndex(1);
visualSampleEntry.setDepth(24);
visualSampleEntry.setFrameCount(1);
visualSampleEntry.setHorizresolution(72);
visualSampleEntry.setVertresolution(72);
visualSampleEntry.setCompressorname("HEVC Coding");
final HevcConfigurationBox hevcConfigurationBox = new HevcConfigurationBox();
hevcConfigurationBox.getHevcDecoderConfigurationRecord().setConfigurationVersion(1);
if (spsStruct != null) {
visualSampleEntry.setWidth(spsStruct.pic_width_in_luma_samples);
visualSampleEntry.setHeight(spsStruct.pic_height_in_luma_samples);
final DimensionTrackExtension dte = this.getTrackExtension(DimensionTrackExtension.class);
if (dte == null) {
this.addTrackExtension(new DimensionTrackExtension(spsStruct.pic_width_in_luma_samples, spsStruct.pic_height_in_luma_samples));
}
final HevcDecoderConfigurationRecord hevcDecoderConfigurationRecord = hevcConfigurationBox.getHevcDecoderConfigurationRecord();
hevcDecoderConfigurationRecord.setChromaFormat(spsStruct.chroma_format_idc);
hevcDecoderConfigurationRecord.setGeneral_profile_idc(spsStruct.general_profile_idc);
hevcDecoderConfigurationRecord.setGeneral_profile_compatibility_flags(spsStruct.general_profile_compatibility_flags);
hevcDecoderConfigurationRecord.setGeneral_constraint_indicator_flags(spsStruct.general_constraint_indicator_flags);
hevcDecoderConfigurationRecord.setGeneral_level_idc(spsStruct.general_level_idc);
hevcDecoderConfigurationRecord.setGeneral_tier_flag(spsStruct.general_tier_flag);
hevcDecoderConfigurationRecord.setGeneral_profile_space(spsStruct.general_profile_space);
hevcDecoderConfigurationRecord.setBitDepthChromaMinus8(spsStruct.bit_depth_chroma_minus8);
hevcDecoderConfigurationRecord.setBitDepthLumaMinus8(spsStruct.bit_depth_luma_minus8);
hevcDecoderConfigurationRecord.setTemporalIdNested(spsStruct.sps_temporal_id_nesting_flag);
}
hevcConfigurationBox.getHevcDecoderConfigurationRecord().setLengthSizeMinusOne(3);
final HevcDecoderConfigurationRecord.Array vpsArray = new HevcDecoderConfigurationRecord.Array();
vpsArray.array_completeness = false;
vpsArray.nal_unit_type = NAL_TYPE_VPS_NUT;
vpsArray.nalUnits = new ArrayList<>();
for (ByteBuffer vp : vps) {
vpsArray.nalUnits.add(Utils.toArray(vp));
}
final HevcDecoderConfigurationRecord.Array spsArray = new HevcDecoderConfigurationRecord.Array();
spsArray.array_completeness = false;
spsArray.nal_unit_type = NAL_TYPE_SPS_NUT;
spsArray.nalUnits = new ArrayList<>();
for (ByteBuffer sp : sps) {
spsArray.nalUnits.add(Utils.toArray(sp));
}
final HevcDecoderConfigurationRecord.Array ppsArray = new HevcDecoderConfigurationRecord.Array();
ppsArray.array_completeness = false;
ppsArray.nal_unit_type = NAL_TYPE_PPS_NUT;
ppsArray.nalUnits = new ArrayList<>();
for (ByteBuffer pp : pps) {
ppsArray.nalUnits.add(Utils.toArray(pp));
}
hevcConfigurationBox.getArrays().addAll(Arrays.asList(spsArray, vpsArray, ppsArray));
visualSampleEntry.addBox(hevcConfigurationBox);
return visualSampleEntry;
}
private boolean isVcl(final @NonNull H265NalUnitHeader nalUnitHeader) {
return nalUnitHeader.nalUnitType >= 0 && nalUnitHeader.nalUnitType <= 31;
}
}

1046
lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/muxer/Mp4Writer.java
View File

File diff suppressed because it is too large Load Diff

24
lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/muxer/MuxingException.java
View File

@@ -1,12 +1,12 @@
package org.thoughtcrime.securesms.video.videoconverter.muxer;
final class MuxingException extends RuntimeException {
public MuxingException(String message) {
super(message);
}
public MuxingException(String message, Throwable cause) {
super(message, cause);
}
}
package org.thoughtcrime.securesms.video.videoconverter.muxer;
final class MuxingException extends RuntimeException {
public MuxingException(String message) {
super(message);
}
public MuxingException(String message, Throwable cause) {
super(message, cause);
}
}

380
lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/muxer/StreamingMuxer.java
View File

@@ -1,190 +1,190 @@
package org.thoughtcrime.securesms.video.videoconverter.muxer;
import android.media.MediaCodec;
import android.media.MediaFormat;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import org.mp4parser.boxes.iso14496.part1.objectdescriptors.DecoderSpecificInfo;
import org.mp4parser.streaming.StreamingTrack;
import org.signal.core.util.logging.Log;
import org.thoughtcrime.securesms.video.interfaces.Muxer;
import org.thoughtcrime.securesms.video.videoconverter.utils.MediaCodecCompat;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.nio.channels.Channels;
import java.util.ArrayList;
import java.util.List;
public final class StreamingMuxer implements Muxer {
private static final String TAG = Log.tag(StreamingMuxer.class);
private final OutputStream outputStream;
private final List<MediaCodecTrack> tracks = new ArrayList<>();
private Mp4Writer mp4Writer;
public StreamingMuxer(OutputStream outputStream) {
this.outputStream = outputStream;
}
@Override
public void start() throws IOException {
final List<StreamingTrack> source = new ArrayList<>();
for (MediaCodecTrack track : tracks) {
source.add((StreamingTrack) track);
}
mp4Writer = new Mp4Writer(source, Channels.newChannel(outputStream));
}
@Override
public long stop() throws IOException {
if (mp4Writer == null) {
throw new IllegalStateException("calling stop prior to start");
}
for (MediaCodecTrack track : tracks) {
track.finish();
}
mp4Writer.close();
long mdatLength = mp4Writer.getTotalMdatContentLength();
mp4Writer = null;
return mdatLength;
}
@Override
public int addTrack(@NonNull MediaFormat format) throws IOException {
final String mime = format.getString(MediaFormat.KEY_MIME);
switch (mime) {
case "video/avc":
tracks.add(new MediaCodecAvcTrack(format));
break;
case "audio/mp4a-latm":
tracks.add(MediaCodecAacTrack.create(format));
break;
case "video/hevc":
tracks.add(new MediaCodecHevcTrack(format));
break;
default:
throw new IllegalArgumentException("unknown track format");
}
return tracks.size() - 1;
}
@Override
public void writeSampleData(int trackIndex, @NonNull ByteBuffer byteBuf, @NonNull MediaCodec.BufferInfo bufferInfo) throws IOException {
tracks.get(trackIndex).writeSampleData(byteBuf, bufferInfo);
}
@Override
public void release() {
}
@Override
public boolean supportsAudioRemux() {
return true;
}
interface MediaCodecTrack {
void writeSampleData(@NonNull ByteBuffer byteBuf, @NonNull MediaCodec.BufferInfo bufferInfo) throws IOException;
void finish() throws IOException;
}
static class MediaCodecAvcTrack extends AvcTrack implements MediaCodecTrack {
MediaCodecAvcTrack(@NonNull MediaFormat format) {
super(Utils.subBuffer(format.getByteBuffer("csd-0"), 4), Utils.subBuffer(format.getByteBuffer("csd-1"), 4));
}
@Override
public void writeSampleData(@NonNull ByteBuffer byteBuf, @NonNull MediaCodec.BufferInfo bufferInfo) throws IOException {
final List<ByteBuffer> nals = H264Utils.getNals(byteBuf);
for (ByteBuffer nal : nals) {
consumeNal(Utils.clone(nal), bufferInfo.presentationTimeUs);
}
}
@Override
public void finish() throws IOException {
consumeLastNal();
}
}
static class MediaCodecHevcTrack extends HevcTrack implements MediaCodecTrack {
MediaCodecHevcTrack(@NonNull MediaFormat format) throws IOException {
super(H264Utils.getNals(format.getByteBuffer("csd-0")));
}
@Override
public void writeSampleData(@NonNull ByteBuffer byteBuf, @NonNull MediaCodec.BufferInfo bufferInfo) throws IOException {
final List<ByteBuffer> nals = H264Utils.getNals(byteBuf);
for (ByteBuffer nal : nals) {
consumeNal(Utils.clone(nal), bufferInfo.presentationTimeUs);
}
}
@Override
public void finish() throws IOException {
consumeLastNal();
}
}
static class MediaCodecAacTrack extends AacTrack implements MediaCodecTrack {
private MediaCodecAacTrack(long avgBitrate, long maxBitrate, int sampleRate, int channelCount, int aacProfile, @Nullable DecoderSpecificInfo decoderSpecificInfo) {
super(avgBitrate, maxBitrate, sampleRate, channelCount, aacProfile, decoderSpecificInfo);
}
public static MediaCodecAacTrack create(@NonNull MediaFormat format) {
final int bitrate = format.getInteger(MediaFormat.KEY_BIT_RATE);
final int maxBitrate;
if (format.containsKey(MediaCodecCompat.MEDIA_FORMAT_KEY_MAX_BIT_RATE)) {
maxBitrate = format.getInteger(MediaCodecCompat.MEDIA_FORMAT_KEY_MAX_BIT_RATE);
} else {
maxBitrate = bitrate;
}
final DecoderSpecificInfo filledDecoderSpecificInfo;
if (format.containsKey(MediaCodecCompat.MEDIA_FORMAT_KEY_MAX_BIT_RATE)) {
final ByteBuffer csd = format.getByteBuffer(MediaCodecCompat.MEDIA_FORMAT_KEY_CODEC_SPECIFIC_DATA_0);
DecoderSpecificInfo decoderSpecificInfo = new DecoderSpecificInfo();
boolean parseSuccess = false;
try {
decoderSpecificInfo.parseDetail(csd);
parseSuccess = true;
} catch (IOException e) {
Log.w(TAG, "Could not parse AAC codec-specific data!", e);
}
if (parseSuccess) {
filledDecoderSpecificInfo = decoderSpecificInfo;
} else {
filledDecoderSpecificInfo = null;
}
} else {
filledDecoderSpecificInfo = null;
}
return new MediaCodecAacTrack(bitrate, maxBitrate,
format.getInteger(MediaFormat.KEY_SAMPLE_RATE), format.getInteger(MediaFormat.KEY_CHANNEL_COUNT),
format.getInteger(MediaFormat.KEY_AAC_PROFILE), filledDecoderSpecificInfo);
}
@Override
public void writeSampleData(@NonNull ByteBuffer byteBuf, @NonNull MediaCodec.BufferInfo bufferInfo) throws IOException {
final byte[] buffer = new byte[bufferInfo.size];
byteBuf.position(bufferInfo.offset);
byteBuf.get(buffer, 0, bufferInfo.size);
processSample(ByteBuffer.wrap(buffer));
}
@Override
public void finish() {
}
}
}
package org.thoughtcrime.securesms.video.videoconverter.muxer;
import android.media.MediaCodec;
import android.media.MediaFormat;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import org.mp4parser.boxes.iso14496.part1.objectdescriptors.DecoderSpecificInfo;
import org.mp4parser.streaming.StreamingTrack;
import org.signal.core.util.logging.Log;
import org.thoughtcrime.securesms.video.interfaces.Muxer;
import org.thoughtcrime.securesms.video.videoconverter.utils.MediaCodecCompat;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.nio.channels.Channels;
import java.util.ArrayList;
import java.util.List;
public final class StreamingMuxer implements Muxer {
private static final String TAG = Log.tag(StreamingMuxer.class);
private final OutputStream outputStream;
private final List<MediaCodecTrack> tracks = new ArrayList<>();
private Mp4Writer mp4Writer;
public StreamingMuxer(OutputStream outputStream) {
this.outputStream = outputStream;
}
@Override
public void start() throws IOException {
final List<StreamingTrack> source = new ArrayList<>();
for (MediaCodecTrack track : tracks) {
source.add((StreamingTrack) track);
}
mp4Writer = new Mp4Writer(source, Channels.newChannel(outputStream));
}
@Override
public long stop() throws IOException {
if (mp4Writer == null) {
throw new IllegalStateException("calling stop prior to start");
}
for (MediaCodecTrack track : tracks) {
track.finish();
}
mp4Writer.close();
long mdatLength = mp4Writer.getTotalMdatContentLength();
mp4Writer = null;
return mdatLength;
}
@Override
public int addTrack(@NonNull MediaFormat format) throws IOException {
final String mime = format.getString(MediaFormat.KEY_MIME);
switch (mime) {
case "video/avc":
tracks.add(new MediaCodecAvcTrack(format));
break;
case "audio/mp4a-latm":
tracks.add(MediaCodecAacTrack.create(format));
break;
case "video/hevc":
tracks.add(new MediaCodecHevcTrack(format));
break;
default:
throw new IllegalArgumentException("unknown track format");
}
return tracks.size() - 1;
}
@Override
public void writeSampleData(int trackIndex, @NonNull ByteBuffer byteBuf, @NonNull MediaCodec.BufferInfo bufferInfo) throws IOException {
tracks.get(trackIndex).writeSampleData(byteBuf, bufferInfo);
}
@Override
public void release() {
}
@Override
public boolean supportsAudioRemux() {
return true;
}
interface MediaCodecTrack {
void writeSampleData(@NonNull ByteBuffer byteBuf, @NonNull MediaCodec.BufferInfo bufferInfo) throws IOException;
void finish() throws IOException;
}
static class MediaCodecAvcTrack extends AvcTrack implements MediaCodecTrack {
MediaCodecAvcTrack(@NonNull MediaFormat format) {
super(Utils.subBuffer(format.getByteBuffer("csd-0"), 4), Utils.subBuffer(format.getByteBuffer("csd-1"), 4));
}
@Override
public void writeSampleData(@NonNull ByteBuffer byteBuf, @NonNull MediaCodec.BufferInfo bufferInfo) throws IOException {
final List<ByteBuffer> nals = H264Utils.getNals(byteBuf);
for (ByteBuffer nal : nals) {
consumeNal(Utils.clone(nal), bufferInfo.presentationTimeUs);
}
}
@Override
public void finish() throws IOException {
consumeLastNal();
}
}
static class MediaCodecHevcTrack extends HevcTrack implements MediaCodecTrack {
MediaCodecHevcTrack(@NonNull MediaFormat format) throws IOException {
super(H264Utils.getNals(format.getByteBuffer("csd-0")));
}
@Override
public void writeSampleData(@NonNull ByteBuffer byteBuf, @NonNull MediaCodec.BufferInfo bufferInfo) throws IOException {
final List<ByteBuffer> nals = H264Utils.getNals(byteBuf);
for (ByteBuffer nal : nals) {
consumeNal(Utils.clone(nal), bufferInfo.presentationTimeUs);
}
}
@Override
public void finish() throws IOException {
consumeLastNal();
}
}
static class MediaCodecAacTrack extends AacTrack implements MediaCodecTrack {
private MediaCodecAacTrack(long avgBitrate, long maxBitrate, int sampleRate, int channelCount, int aacProfile, @Nullable DecoderSpecificInfo decoderSpecificInfo) {
super(avgBitrate, maxBitrate, sampleRate, channelCount, aacProfile, decoderSpecificInfo);
}
public static MediaCodecAacTrack create(@NonNull MediaFormat format) {
final int bitrate = format.getInteger(MediaFormat.KEY_BIT_RATE);
final int maxBitrate;
if (format.containsKey(MediaCodecCompat.MEDIA_FORMAT_KEY_MAX_BIT_RATE)) {
maxBitrate = format.getInteger(MediaCodecCompat.MEDIA_FORMAT_KEY_MAX_BIT_RATE);
} else {
maxBitrate = bitrate;
}
final DecoderSpecificInfo filledDecoderSpecificInfo;
if (format.containsKey(MediaCodecCompat.MEDIA_FORMAT_KEY_MAX_BIT_RATE)) {
final ByteBuffer csd = format.getByteBuffer(MediaCodecCompat.MEDIA_FORMAT_KEY_CODEC_SPECIFIC_DATA_0);
DecoderSpecificInfo decoderSpecificInfo = new DecoderSpecificInfo();
boolean parseSuccess = false;
try {
decoderSpecificInfo.parseDetail(csd);
parseSuccess = true;
} catch (IOException e) {
Log.w(TAG, "Could not parse AAC codec-specific data!", e);
}
if (parseSuccess) {
filledDecoderSpecificInfo = decoderSpecificInfo;
} else {
filledDecoderSpecificInfo = null;
}
} else {
filledDecoderSpecificInfo = null;
}
return new MediaCodecAacTrack(bitrate, maxBitrate,
format.getInteger(MediaFormat.KEY_SAMPLE_RATE), format.getInteger(MediaFormat.KEY_CHANNEL_COUNT),
format.getInteger(MediaFormat.KEY_AAC_PROFILE), filledDecoderSpecificInfo);
}
@Override
public void writeSampleData(@NonNull ByteBuffer byteBuf, @NonNull MediaCodec.BufferInfo bufferInfo) throws IOException {
final byte[] buffer = new byte[bufferInfo.size];
byteBuf.position(bufferInfo.offset);
byteBuf.get(buffer, 0, bufferInfo.size);
processSample(ByteBuffer.wrap(buffer));
}
@Override
public void finish() {
}
}
}

88
lib/video/src/main/java/org/thoughtcrime/securesms/video/videoconverter/muxer/Utils.java
View File

@@ -1,44 +1,44 @@
package org.thoughtcrime.securesms.video.videoconverter.muxer;
import androidx.annotation.NonNull;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.util.ArrayList;
import java.util.List;
/**
* Based on https://github.com/jcodec/jcodec/blob/master/src/main/java/org/jcodec/codecs/h264/H264Utils.java
*/
final class Utils {
private Utils() {}
static byte[] toArray(final @NonNull ByteBuffer buf) {
final ByteBuffer newBuf = buf.duplicate();
byte[] bytes = new byte[newBuf.remaining()];
newBuf.get(bytes, 0, bytes.length);
return bytes;
}
public static ByteBuffer clone(final @NonNull ByteBuffer original) {
final ByteBuffer clone = ByteBuffer.allocate(original.capacity());
original.rewind();
clone.put(original);
original.rewind();
clone.flip();
return clone;
}
static @NonNull ByteBuffer subBuffer(final @NonNull ByteBuffer buf, final int start) {
return subBuffer(buf, start, buf.limit() - start);
}
static @NonNull ByteBuffer subBuffer(final @NonNull ByteBuffer buf, final int start, final int count) {
final ByteBuffer newBuf = buf.duplicate();
byte[] bytes = new byte[count];
newBuf.position(start);
newBuf.get(bytes, 0, bytes.length);
return ByteBuffer.wrap(bytes);
}
}
package org.thoughtcrime.securesms.video.videoconverter.muxer;
import androidx.annotation.NonNull;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.util.ArrayList;
import java.util.List;
/**
* Based on https://github.com/jcodec/jcodec/blob/master/src/main/java/org/jcodec/codecs/h264/H264Utils.java
*/
final class Utils {
private Utils() {}
static byte[] toArray(final @NonNull ByteBuffer buf) {
final ByteBuffer newBuf = buf.duplicate();
byte[] bytes = new byte[newBuf.remaining()];
newBuf.get(bytes, 0, bytes.length);
return bytes;
}
public static ByteBuffer clone(final @NonNull ByteBuffer original) {
final ByteBuffer clone = ByteBuffer.allocate(original.capacity());
original.rewind();
clone.put(original);
original.rewind();
clone.flip();
return clone;
}
static @NonNull ByteBuffer subBuffer(final @NonNull ByteBuffer buf, final int start) {
return subBuffer(buf, start, buf.limit() - start);
}
static @NonNull ByteBuffer subBuffer(final @NonNull ByteBuffer buf, final int start, final int count) {
final ByteBuffer newBuf = buf.duplicate();
byte[] bytes = new byte[count];
newBuf.position(start);
newBuf.get(bytes, 0, bytes.length);
return ByteBuffer.wrap(bytes);
}
}