From 7c7dc679e9f85520cfc06218f5876d0758fe796e Mon Sep 17 00:00:00 2001 From: Greyson Parrelli Date: Wed, 15 Jan 2025 15:11:09 -0500 Subject: [PATCH] Only write out one MDAT box for a video transcode. Co-authored-by: Milan Stevanovic --- .../jobs/AttachmentCompressionJob.java | 5 ++- .../securesms/video/StreamingTranscoder.java | 9 +++- .../securesms/video/interfaces/Muxer.java | 2 +- .../Mp4FaststartPostProcessor.kt | 17 ++++++++ .../video/videoconverter/AndroidMuxer.java | 3 +- .../video/videoconverter/MediaConverter.java | 17 +++++++- .../video/videoconverter/muxer/Mp4Writer.java | 42 +++++++++++++++---- .../videoconverter/muxer/StreamingMuxer.java | 6 ++- 8 files changed, 83 insertions(+), 18 deletions(-) diff --git a/app/src/main/java/org/thoughtcrime/securesms/jobs/AttachmentCompressionJob.java b/app/src/main/java/org/thoughtcrime/securesms/jobs/AttachmentCompressionJob.java index 3ecc6a137c..2f8500f4fc 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/jobs/AttachmentCompressionJob.java +++ b/app/src/main/java/org/thoughtcrime/securesms/jobs/AttachmentCompressionJob.java @@ -267,8 +267,9 @@ public final class AttachmentCompressionJob extends BaseJob { boolean faststart = false; try { + int mdatLength; try (OutputStream outputStream = ModernEncryptingPartOutputStream.createFor(attachmentSecret, file, true).second) { - transcoder.transcode(percent -> { + mdatLength = (int) transcoder.transcode(percent -> { if (notification != null) { notification.setProgress(percent / 100f); } @@ -299,7 +300,7 @@ public final class AttachmentCompressionJob extends BaseJob { }); final long plaintextLength = ModernEncryptingPartOutputStream.getPlaintextLength(file.length()); - try (MediaStream mediaStream = new MediaStream(postProcessor.process(plaintextLength), MimeTypes.VIDEO_MP4, 0, 0, true)) { + try (MediaStream mediaStream = new MediaStream(postProcessor.processWithMdatLength(plaintextLength, mdatLength), MimeTypes.VIDEO_MP4, 0, 0, true)) { attachmentDatabase.updateAttachmentData(attachment, mediaStream); faststart = true; } catch (VideoPostProcessingException e) { diff --git a/video/lib/src/main/java/org/thoughtcrime/securesms/video/StreamingTranscoder.java b/video/lib/src/main/java/org/thoughtcrime/securesms/video/StreamingTranscoder.java index 044944eab4..2601b8c53c 100644 --- a/video/lib/src/main/java/org/thoughtcrime/securesms/video/StreamingTranscoder.java +++ b/video/lib/src/main/java/org/thoughtcrime/securesms/video/StreamingTranscoder.java @@ -127,7 +127,10 @@ public final class StreamingTranscoder { return new StreamingTranscoder(dataSource, options, codec, videoBitrate, audioBitrate, shortEdge, allowAudioRemux); } - public void transcode(@NonNull Progress progress, + /** + * @return The total content size of the MP4 mdat box. + */ + public long transcode(@NonNull Progress progress, @NonNull OutputStream stream, @Nullable TranscoderCancelationSignal cancelationSignal) throws IOException, EncodingException @@ -193,7 +196,7 @@ public final class StreamingTranscoder { return cancelationSignal != null && cancelationSignal.isCanceled(); }); - converter.convert(); + long mdatSize = converter.convert(); long outSize = outStream.getCount(); float encodeDurationSec = (System.currentTimeMillis() - startTime) / 1000f; @@ -217,6 +220,8 @@ public final class StreamingTranscoder { } stream.flush(); + + return mdatSize; } public boolean isTranscodeRequired() { diff --git a/video/lib/src/main/java/org/thoughtcrime/securesms/video/interfaces/Muxer.java b/video/lib/src/main/java/org/thoughtcrime/securesms/video/interfaces/Muxer.java index 918dd3de69..4e511fb67d 100644 --- a/video/lib/src/main/java/org/thoughtcrime/securesms/video/interfaces/Muxer.java +++ b/video/lib/src/main/java/org/thoughtcrime/securesms/video/interfaces/Muxer.java @@ -17,7 +17,7 @@ public interface Muxer { void start() throws IOException; - void stop() throws IOException; + long stop() throws IOException; int addTrack(@NonNull MediaFormat format) throws IOException; diff --git a/video/lib/src/main/java/org/thoughtcrime/securesms/video/postprocessing/Mp4FaststartPostProcessor.kt b/video/lib/src/main/java/org/thoughtcrime/securesms/video/postprocessing/Mp4FaststartPostProcessor.kt index 678f90e4d4..b829f7436a 100644 --- a/video/lib/src/main/java/org/thoughtcrime/securesms/video/postprocessing/Mp4FaststartPostProcessor.kt +++ b/video/lib/src/main/java/org/thoughtcrime/securesms/video/postprocessing/Mp4FaststartPostProcessor.kt @@ -43,6 +43,23 @@ class Mp4FaststartPostProcessor(private val inputStreamFactory: InputStreamFacto } } + /** + * It is the responsibility of the caller to close the resulting [InputStream]. + */ + fun processWithMdatLength(inputLength: Long, mdatLength: Int): SequenceInputStream { + val metadata = inputStreamFactory.create().use { inputStream -> + inputStream.use { + Mp4Sanitizer.sanitizeFileWithCompoundedMdatBoxes(it, inputLength, mdatLength) + } + } + if (metadata.sanitizedMetadata == null) { + throw VideoPostProcessingException("Sanitized metadata was null!") + } + val inputStream = inputStreamFactory.create() + inputStream.skip(metadata.dataOffset) + return SequenceInputStream(ByteArrayInputStream(metadata.sanitizedMetadata), LimitedInputStream(inputStream, metadata.dataLength)) + } + fun interface InputStreamFactory { fun create(): InputStream } diff --git a/video/lib/src/main/java/org/thoughtcrime/securesms/video/videoconverter/AndroidMuxer.java b/video/lib/src/main/java/org/thoughtcrime/securesms/video/videoconverter/AndroidMuxer.java index bfc7333784..9ed0536d43 100644 --- a/video/lib/src/main/java/org/thoughtcrime/securesms/video/videoconverter/AndroidMuxer.java +++ b/video/lib/src/main/java/org/thoughtcrime/securesms/video/videoconverter/AndroidMuxer.java @@ -33,8 +33,9 @@ final class AndroidMuxer implements Muxer { } @Override - public void stop() { + public long stop() { muxer.stop(); + return 0; } @Override diff --git a/video/lib/src/main/java/org/thoughtcrime/securesms/video/videoconverter/MediaConverter.java b/video/lib/src/main/java/org/thoughtcrime/securesms/video/videoconverter/MediaConverter.java index c7ad42ee2a..e19cc3ac45 100644 --- a/video/lib/src/main/java/org/thoughtcrime/securesms/video/videoconverter/MediaConverter.java +++ b/video/lib/src/main/java/org/thoughtcrime/securesms/video/videoconverter/MediaConverter.java @@ -138,15 +138,21 @@ public final class MediaConverter { mAllowAudioRemux = allow; } + /** + * @return The total content size of the MP4 mdat box. + */ @WorkerThread @RequiresApi(23) - public void convert() throws EncodingException, IOException { + public long convert() throws EncodingException, IOException { // Exception that may be thrown during release. Exception exception = null; Muxer muxer = null; VideoTrackConverter videoTrackConverter = null; AudioTrackConverter audioTrackConverter = null; + long mdatContentLength = 0; + boolean muxerStopped = false; + try { muxer = mOutput.createMuxer(); @@ -162,6 +168,9 @@ public final class MediaConverter { audioTrackConverter, muxer); + mdatContentLength = muxer.stop(); + muxerStopped = true; + } catch (EncodingException | IOException e) { Log.e(TAG, "error converting", e); exception = e; @@ -196,7 +205,9 @@ public final class MediaConverter { } try { if (muxer != null) { - muxer.stop(); + if (!muxerStopped) { + muxer.stop(); + } muxer.release(); } } catch (Exception e) { @@ -209,6 +220,8 @@ public final class MediaConverter { if (exception != null) { throw new EncodingException("Transcode failed", exception); } + + return mdatContentLength; } /** diff --git a/video/lib/src/main/java/org/thoughtcrime/securesms/video/videoconverter/muxer/Mp4Writer.java b/video/lib/src/main/java/org/thoughtcrime/securesms/video/videoconverter/muxer/Mp4Writer.java index 7b13c42c4b..980cafdd3d 100644 --- a/video/lib/src/main/java/org/thoughtcrime/securesms/video/videoconverter/muxer/Mp4Writer.java +++ b/video/lib/src/main/java/org/thoughtcrime/securesms/video/videoconverter/muxer/Mp4Writer.java @@ -52,6 +52,7 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.nio.channels.WritableByteChannel; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.Date; import java.util.HashMap; @@ -80,6 +81,7 @@ final class Mp4Writer extends DefaultBoxes implements SampleSink { private final List source; private final Date creationTime = new Date(); + private boolean hasWrittenMdat = false; /** * Contains the start time of the next segment in line that will be created. @@ -106,6 +108,8 @@ final class Mp4Writer extends DefaultBoxes implements SampleSink { private final Map sampleNumbers = new HashMap<>(); private long bytesWritten = 0; + private long mMDatTotalContentLength = 0; + Mp4Writer(final @NonNull List source, final @NonNull WritableByteChannel sink) throws IOException { this.source = new ArrayList<>(source); this.sink = sink; @@ -152,6 +156,11 @@ final class Mp4Writer extends DefaultBoxes implements SampleSink { streamingTrack.close(); } write(sink, createMoov()); + hasWrittenMdat = false; + } + + public long getTotalMdatContentLength() { + return mMDatTotalContentLength; } private Box createMoov() { @@ -252,8 +261,16 @@ final class Mp4Writer extends DefaultBoxes implements SampleSink { private void writeChunkContainer(ChunkContainer chunkContainer) throws IOException { final TrackBox tb = trackBoxes.get(chunkContainer.streamingTrack); final ChunkOffsetBox stco = Objects.requireNonNull(Path.getPath(tb, "mdia[0]/minf[0]/stbl[0]/stco[0]")); - stco.setChunkOffsets(Mp4Arrays.copyOfAndAppend(stco.getChunkOffsets(), bytesWritten + 8)); + final int extraChunkOffset = hasWrittenMdat ? 0 : 8; + stco.setChunkOffsets(Mp4Arrays.copyOfAndAppend(stco.getChunkOffsets(), bytesWritten + extraChunkOffset)); + chunkContainer.mdat.includeHeader = !hasWrittenMdat; write(sink, chunkContainer.mdat); + + mMDatTotalContentLength += chunkContainer.mdat.getSize(); + + if (!hasWrittenMdat) { + hasWrittenMdat = true; + } } public void acceptSample( @@ -401,9 +418,12 @@ final class Mp4Writer extends DefaultBoxes implements SampleSink { final ArrayList samples; long size; + boolean includeHeader; + Mdat(final @NonNull List samples) { this.samples = new ArrayList<>(samples); size = 8; + for (StreamingSample sample : samples) { size += sample.getContent().limit(); } @@ -416,19 +436,23 @@ final class Mp4Writer extends DefaultBoxes implements SampleSink { @Override public long getSize() { - return size; + if (includeHeader) { + return size; + } else { + return size - 8; + } } @Override public void getBox(WritableByteChannel writableByteChannel) throws IOException { - writableByteChannel.write(ByteBuffer.wrap(new byte[]{ - (byte) ((size & 0xff000000) >> 24), - (byte) ((size & 0xff0000) >> 16), - (byte) ((size & 0xff00) >> 8), - (byte) ((size & 0xff)), - 109, 100, 97, 116, // mdat + if (includeHeader) { + // When we include the header, we specify the declared size as 1, indicating the size is from here until the end of the file + writableByteChannel.write(ByteBuffer.wrap(new byte[] { + 0, 0, 0, 0, // size (4 bytes) + 109, 100, 97, 116, // 'm' 'd' 'a' 't' + })); + } - })); for (StreamingSample sample : samples) { writableByteChannel.write((ByteBuffer) sample.getContent().rewind()); } diff --git a/video/lib/src/main/java/org/thoughtcrime/securesms/video/videoconverter/muxer/StreamingMuxer.java b/video/lib/src/main/java/org/thoughtcrime/securesms/video/videoconverter/muxer/StreamingMuxer.java index bb6c9d7f10..b3f339dddb 100644 --- a/video/lib/src/main/java/org/thoughtcrime/securesms/video/videoconverter/muxer/StreamingMuxer.java +++ b/video/lib/src/main/java/org/thoughtcrime/securesms/video/videoconverter/muxer/StreamingMuxer.java @@ -39,7 +39,7 @@ public final class StreamingMuxer implements Muxer { } @Override - public void stop() throws IOException { + public long stop() throws IOException { if (mp4Writer == null) { throw new IllegalStateException("calling stop prior to start"); } @@ -47,7 +47,11 @@ public final class StreamingMuxer implements Muxer { track.finish(); } mp4Writer.close(); + long mdatLength = mp4Writer.getTotalMdatContentLength(); + mp4Writer = null; + + return mdatLength; } @Override