From caa5e233dfd070778a9591dfef67a96fb7d97c98 Mon Sep 17 00:00:00 2001 From: Nicholas Tinsley Date: Fri, 12 Jan 2024 19:11:54 -0500 Subject: [PATCH] Expose StreamingTranscoder configuration options in sample app. --- .../jobs/AttachmentCompressionJob.java | 42 ++-- .../securesms/scribbles/VideoEditorHud.java | 3 +- .../securesms/video/InMemoryTranscoder.java | 63 ++--- video/app/src/main/AndroidManifest.xml | 9 +- .../video/app/transcode/Constants.kt | 25 ++ .../video/app/transcode/TestTranscoder.kt | 8 - .../app/transcode/TranscodeJobSnapshot.kt | 11 - .../app/transcode/TranscodeTestActivity.kt | 88 ++++--- .../app/transcode/TranscodeTestRepository.kt | 47 ++-- .../app/transcode/TranscodeTestViewModel.kt | 20 +- .../video/app/transcode/TranscodeWorker.kt | 131 +++++++---- .../composables/ConfigurationSelection.kt | 215 ++++++++++++++++++ .../transcode/composables/InputSelection.kt | 33 +++ .../transcode/composables/OutputSelection.kt | 33 +++ .../app/transcode/composables/Progress.kt | 63 +++++ video/lib/build.gradle.kts | 2 + .../securesms/video/StreamingTranscoder.java | 47 +++- .../securesms/video/TranscodingQuality.kt | 29 +++ .../video/VideoBitRateCalculator.java | 70 +----- .../VideoPostProcessingException.kt | 11 + .../video/exceptions/VideoSizeException.kt | 2 +- .../Mp4FaststartPostProcessor.kt | 43 ++++ 22 files changed, 738 insertions(+), 257 deletions(-) create mode 100644 video/app/src/main/java/org/thoughtcrime/video/app/transcode/Constants.kt delete mode 100644 video/app/src/main/java/org/thoughtcrime/video/app/transcode/TestTranscoder.kt delete mode 100644 video/app/src/main/java/org/thoughtcrime/video/app/transcode/TranscodeJobSnapshot.kt create mode 100644 video/app/src/main/java/org/thoughtcrime/video/app/transcode/composables/ConfigurationSelection.kt create mode 100644 video/app/src/main/java/org/thoughtcrime/video/app/transcode/composables/InputSelection.kt create mode 100644 video/app/src/main/java/org/thoughtcrime/video/app/transcode/composables/OutputSelection.kt create mode 100644 video/app/src/main/java/org/thoughtcrime/video/app/transcode/composables/Progress.kt create mode 100644 video/lib/src/main/java/org/thoughtcrime/securesms/video/TranscodingQuality.kt create mode 100644 video/lib/src/main/java/org/thoughtcrime/securesms/video/exceptions/VideoPostProcessingException.kt create mode 100644 video/lib/src/main/java/org/thoughtcrime/securesms/video/postprocessing/Mp4FaststartPostProcessor.kt diff --git a/app/src/main/java/org/thoughtcrime/securesms/jobs/AttachmentCompressionJob.java b/app/src/main/java/org/thoughtcrime/securesms/jobs/AttachmentCompressionJob.java index 0273d77bc8..f19e727460 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/jobs/AttachmentCompressionJob.java +++ b/app/src/main/java/org/thoughtcrime/securesms/jobs/AttachmentCompressionJob.java @@ -9,13 +9,8 @@ import androidx.annotation.Nullable; import androidx.annotation.WorkerThread; import androidx.media3.common.MimeTypes; -import com.google.common.io.ByteStreams; - import org.greenrobot.eventbus.EventBus; import org.signal.core.util.logging.Log; -import org.signal.libsignal.media.Mp4Sanitizer; -import org.signal.libsignal.media.ParseException; -import org.signal.libsignal.media.SanitizedMetadata; import org.thoughtcrime.securesms.R; import org.thoughtcrime.securesms.attachments.Attachment; import org.thoughtcrime.securesms.attachments.AttachmentId; @@ -47,15 +42,16 @@ import org.thoughtcrime.securesms.video.InMemoryTranscoder; import org.thoughtcrime.securesms.video.StreamingTranscoder; import org.thoughtcrime.securesms.video.TranscoderCancelationSignal; import org.thoughtcrime.securesms.video.TranscoderOptions; +import org.thoughtcrime.securesms.video.exceptions.VideoPostProcessingException; +import org.thoughtcrime.securesms.video.exceptions.VideoSourceException; +import org.thoughtcrime.securesms.video.postprocessing.Mp4FaststartPostProcessor; import org.thoughtcrime.securesms.video.exceptions.VideoSourceException; import org.thoughtcrime.securesms.video.videoconverter.EncodingException; import java.io.ByteArrayInputStream; import java.io.File; import java.io.IOException; -import java.io.InputStream; import java.io.OutputStream; -import java.io.SequenceInputStream; import java.util.Objects; import java.util.concurrent.TimeUnit; @@ -289,23 +285,17 @@ public final class AttachmentCompressionJob extends BaseJob { 100, 100)); - InputStream transcodedFileStream = ModernDecryptingPartInputStream.createFor(attachmentSecret, file, 0); - SanitizedMetadata metadata = null; - try { - metadata = Mp4Sanitizer.sanitize(transcodedFileStream, file.length()); - } catch (ParseException e) { - Log.e(TAG, "Could not parse MP4 file.", e); - } + final Mp4FaststartPostProcessor postProcessor = new Mp4FaststartPostProcessor(() -> { + try { + return ModernDecryptingPartInputStream.createFor(attachmentSecret, file, 0); + } catch (IOException e) { + throw new RuntimeException(e); + } + }, file.length()); - if (metadata != null && metadata.getSanitizedMetadata() != null) { - try (MediaStream mediaStream = new MediaStream(new SequenceInputStream(new ByteArrayInputStream(metadata.getSanitizedMetadata()), ByteStreams.limit(ModernDecryptingPartInputStream.createFor(attachmentSecret, file, metadata.getDataOffset()), metadata.getDataLength())), MimeTypes.VIDEO_MP4, 0, 0, true)) { - attachmentDatabase.updateAttachmentData(attachment, mediaStream, true); - faststart = true; - } - } else { - try (MediaStream mediaStream = new MediaStream(ModernDecryptingPartInputStream.createFor(attachmentSecret, file, 0), MimeTypes.VIDEO_MP4, 0, 0)) { - attachmentDatabase.updateAttachmentData(attachment, mediaStream, true); - } + try (MediaStream mediaStream = new MediaStream(postProcessor.process(), MimeTypes.VIDEO_MP4, 0, 0, true)) { + attachmentDatabase.updateAttachmentData(attachment, mediaStream, true); + faststart = true; } } finally { if (!file.delete()) { @@ -360,6 +350,12 @@ public final class AttachmentCompressionJob extends BaseJob { } } catch (IOException | MmsException e) { throw new UndeliverableMessageException("Failed to transcode", e); + } catch (RuntimeException e) { + if (e.getCause() instanceof IOException) { + throw new UndeliverableMessageException("Failed to transcode", e); + } else { + throw e; + } } return attachment; } diff --git a/app/src/main/java/org/thoughtcrime/securesms/scribbles/VideoEditorHud.java b/app/src/main/java/org/thoughtcrime/securesms/scribbles/VideoEditorHud.java index 66e5b544e6..d84faa53f8 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/scribbles/VideoEditorHud.java +++ b/app/src/main/java/org/thoughtcrime/securesms/scribbles/VideoEditorHud.java @@ -21,6 +21,7 @@ import org.thoughtcrime.securesms.R; import org.thoughtcrime.securesms.media.DecryptableUriMediaInput; import org.thoughtcrime.securesms.mms.VideoSlide; import org.thoughtcrime.securesms.util.MediaUtil; +import org.thoughtcrime.securesms.video.TranscodingQuality; import org.thoughtcrime.securesms.video.VideoBitRateCalculator; import org.thoughtcrime.securesms.video.VideoUtil; import org.thoughtcrime.securesms.video.videoconverter.VideoThumbnailsRangeSelectorView; @@ -134,7 +135,7 @@ public final class VideoEditorHud extends LinearLayout { public VideoThumbnailsRangeSelectorView.Quality getQuality(long clipDurationUs, long totalDurationUs) { int inputBitRate = VideoBitRateCalculator.bitRate(size, TimeUnit.MICROSECONDS.toMillis(totalDurationUs)); - VideoBitRateCalculator.Quality targetQuality = videoBitRateCalculator.getTargetQuality(TimeUnit.MICROSECONDS.toMillis(clipDurationUs), inputBitRate); + TranscodingQuality targetQuality = videoBitRateCalculator.getTargetQuality(TimeUnit.MICROSECONDS.toMillis(clipDurationUs), inputBitRate); return new VideoThumbnailsRangeSelectorView.Quality(targetQuality.getFileSizeEstimate(), (int) (100 * targetQuality.getQuality())); } }); diff --git a/app/src/main/java/org/thoughtcrime/securesms/video/InMemoryTranscoder.java b/app/src/main/java/org/thoughtcrime/securesms/video/InMemoryTranscoder.java index c7b2aed1fa..9cd415b03b 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/video/InMemoryTranscoder.java +++ b/app/src/main/java/org/thoughtcrime/securesms/video/InMemoryTranscoder.java @@ -7,9 +7,7 @@ import android.media.MediaMetadataRetriever; import androidx.annotation.NonNull; import androidx.annotation.Nullable; import androidx.annotation.RequiresApi; - import androidx.media3.common.MimeTypes; -import com.google.common.io.ByteStreams; import org.signal.core.util.logging.Log; import org.signal.libsignal.media.Mp4Sanitizer; @@ -17,18 +15,18 @@ import org.signal.libsignal.media.ParseException; import org.signal.libsignal.media.SanitizedMetadata; import org.thoughtcrime.securesms.mms.MediaStream; import org.thoughtcrime.securesms.util.MemoryFileDescriptor; +import org.thoughtcrime.securesms.video.exceptions.VideoPostProcessingException; import org.thoughtcrime.securesms.video.exceptions.VideoSizeException; import org.thoughtcrime.securesms.video.exceptions.VideoSourceException; +import org.thoughtcrime.securesms.video.postprocessing.Mp4FaststartPostProcessor; import org.thoughtcrime.securesms.video.videoconverter.EncodingException; import org.thoughtcrime.securesms.video.videoconverter.MediaConverter; import org.thoughtcrime.securesms.video.videoconverter.mediadatasource.MediaDataSourceMediaInput; -import java.io.ByteArrayInputStream; import java.io.Closeable; import java.io.FileDescriptor; import java.io.FileInputStream; import java.io.IOException; -import java.io.SequenceInputStream; import java.text.NumberFormat; import java.util.Locale; @@ -37,17 +35,17 @@ public final class InMemoryTranscoder implements Closeable { private static final String TAG = Log.tag(InMemoryTranscoder.class); - private final Context context; - private final MediaDataSource dataSource; - private final long upperSizeLimit; - private final long inSize; - private final long duration; - private final int inputBitRate; - private final VideoBitRateCalculator.Quality targetQuality; - private final long memoryFileEstimate; - private final boolean transcodeRequired; - private final long fileSizeEstimate; - private final @Nullable TranscoderOptions options; + private final Context context; + private final MediaDataSource dataSource; + private final long upperSizeLimit; + private final long inSize; + private final long duration; + private final int inputBitRate; + private final TranscodingQuality targetQuality; + private final long memoryFileEstimate; + private final boolean transcodeRequired; + private final long fileSizeEstimate; + private final @Nullable TranscoderOptions options; private @Nullable MemoryFileDescriptor memoryFile; @@ -148,13 +146,6 @@ public final class InMemoryTranscoder implements Closeable { memoryFile.seek(0); - SanitizedMetadata metadata = null; - try { - metadata = Mp4Sanitizer.sanitize(new FileInputStream(memoryFileFileDescriptor), memoryFile.size()); - } catch (ParseException e) { - Log.e(TAG, "Could not parse MP4 file.", e); - } - // output details of the transcoding long outSize = memoryFile.size(); float encodeDurationSec = (System.currentTimeMillis() - startTime) / 1000f; @@ -179,14 +170,30 @@ public final class InMemoryTranscoder implements Closeable { throw new VideoSizeException("Size constraints could not be met!"); } + try { + final Mp4FaststartPostProcessor postProcessor = new Mp4FaststartPostProcessor(() -> { + try { + memoryFile.seek(0); + return new FileInputStream(memoryFileFileDescriptor); + } catch (IOException e) { + Log.w(TAG, "IOException thrown while creating FileInputStream.", e); + throw new VideoPostProcessingException("Exception while opening InputStream!", e); + } + }, memoryFile.size()); - if (metadata != null && metadata.getSanitizedMetadata() != null) { - memoryFile.seek(metadata.getDataOffset()); - return new MediaStream(new SequenceInputStream(new ByteArrayInputStream(metadata.getSanitizedMetadata()), ByteStreams.limit(new FileInputStream(memoryFileFileDescriptor), metadata.getDataLength())), MimeTypes.VIDEO_MP4, 0, 0, true); - } else { - memoryFile.seek(0); - return new MediaStream(new FileInputStream(memoryFileFileDescriptor), MimeTypes.VIDEO_MP4, 0, 0); + return new MediaStream(postProcessor.process(), MimeTypes.VIDEO_MP4, 0, 0, true); + } catch (VideoPostProcessingException e) { + Log.w(TAG, "Exception thrown during post processing.", e); + final Throwable cause = e.getCause(); + if (cause instanceof IOException) { + throw (IOException) cause; + } else if ( cause instanceof EncodingException) { + throw (EncodingException) cause; + } } + + memoryFile.seek(0); + return new MediaStream(new FileInputStream(memoryFileFileDescriptor), MimeTypes.VIDEO_MP4, 0, 0); } public boolean isTranscodeRequired() { diff --git a/video/app/src/main/AndroidManifest.xml b/video/app/src/main/AndroidManifest.xml index b2a2c0d8de..d1fcac3829 100644 --- a/video/app/src/main/AndroidManifest.xml +++ b/video/app/src/main/AndroidManifest.xml @@ -5,7 +5,10 @@ - + + + + + = Build.VERSION_CODES.O) { + val name = applicationContext.getString(R.string.channel_name) + val descriptionText = applicationContext.getString(R.string.channel_description) + val importance = NotificationManager.IMPORTANCE_DEFAULT + val mChannel = NotificationChannel(getString(R.string.notification_channel_id), name, importance) + mChannel.description = descriptionText + val notificationManager = applicationContext.getSystemService(NOTIFICATION_SERVICE) as NotificationManager + notificationManager.createNotificationChannel(mChannel) + } + setContent { SignalTheme { Surface(modifier = Modifier.fillMaxSize(), color = MaterialTheme.colorScheme.background) { - Column( - verticalArrangement = Arrangement.Center, - horizontalAlignment = Alignment.CenterHorizontally - ) { - val videoUris = viewModel.selectedVideos - val outputDir = viewModel.outputDirectory - val transcodingJobs = viewModel.getTranscodingJobsAsState().collectAsState(emptyList()) - if (transcodingJobs.value.isNotEmpty()) { - transcodingJobs.value.forEach { workInfo -> - val currentProgress = workInfo.progress.getInt(TranscodeWorker.KEY_PROGRESS, -1) - Row( - verticalAlignment = Alignment.CenterVertically, - modifier = Modifier.padding(horizontal = 16.dp) - ) { - Text(text = "...${workInfo.id.toString().takeLast(4)}", modifier = Modifier.padding(end = 16.dp).weight(1f)) - if (workInfo.state.isFinished) { - LinearProgressIndicator(progress = 1f, trackColor = MaterialTheme.colorScheme.secondary, modifier = Modifier.weight(3f)) - } else if (currentProgress >= 0) { - LinearProgressIndicator(progress = currentProgress / 100f, modifier = Modifier.weight(3f)) - } else { - LinearProgressIndicator(modifier = Modifier.weight(3f)) - } - } - } - LabeledButton("Reset/Cancel") { viewModel.reset() } - } else if (videoUris.isEmpty()) { - LabeledButton("Select Videos") { pickMedia.launch(PickVisualMediaRequest(ActivityResultContracts.PickVisualMedia.VideoOnly)) } - } else if (outputDir == null) { - LabeledButton("Select Output Directory") { outputDirRequest.launch(null) } - } else { - Text(text = "Selected videos:", modifier = Modifier.align(Alignment.Start).padding(16.dp)) - videoUris.forEach { - Text(text = it.toString(), fontSize = 8.sp, fontFamily = FontFamily.Monospace, modifier = Modifier.align(Alignment.Start).padding(horizontal = 16.dp)) - } - LabeledButton(buttonLabel = "Transcode") { + val videoUris = viewModel.selectedVideos + val outputDir = viewModel.outputDirectory + val transcodingJobs = viewModel.getTranscodingJobsAsState().collectAsState(emptyList()) + if (transcodingJobs.value.isNotEmpty()) { + TranscodingJobProgress(transcodingJobs = transcodingJobs.value, resetButtonOnClick = { viewModel.reset() }) + } else if (videoUris.isEmpty()) { + SelectInput { pickMedia.launch(PickVisualMediaRequest(ActivityResultContracts.PickVisualMedia.VideoOnly)) } + } else if (outputDir == null) { + SelectOutput { outputDirRequest.launch(null) } + } else { + ConfigureEncodingParameters( + videoUris = videoUris, + onAutoSettingsCheckChanged = { viewModel.useAutoTranscodingSettings = it }, + onRadioButtonSelected = { viewModel.videoResolution = it }, + onSliderValueChanged = { viewModel.videoMegaBitrate = it }, + onFastStartSettingCheckChanged = { viewModel.enableFastStart = it }, + onSequentialSettingCheckChanged = { viewModel.forceSequentialQueueProcessing = it }, + buttonClickListener = { viewModel.transcode() viewModel.selectedVideos = emptyList() viewModel.resetOutputDirectory() } - } + ) } } } diff --git a/video/app/src/main/java/org/thoughtcrime/video/app/transcode/TranscodeTestRepository.kt b/video/app/src/main/java/org/thoughtcrime/video/app/transcode/TranscodeTestRepository.kt index fe3b710281..de3a72b10e 100644 --- a/video/app/src/main/java/org/thoughtcrime/video/app/transcode/TranscodeTestRepository.kt +++ b/video/app/src/main/java/org/thoughtcrime/video/app/transcode/TranscodeTestRepository.kt @@ -20,11 +20,14 @@ import java.util.UUID import kotlin.math.absoluteValue import kotlin.random.Random +/** + * Repository to perform various transcoding functions. + */ class TranscodeTestRepository(context: Context) { private val workManager = WorkManager.getInstance(context) private val usedNotificationIds = emptySet() - fun transcode(selectedVideos: List, outputDirectory: Uri): Map { + fun transcode(selectedVideos: List, outputDirectory: Uri, forceSequentialProcessing: Boolean, customTranscodingOptions: CustomTranscodingOptions?): Map { if (selectedVideos.isEmpty()) { return emptyMap() } @@ -34,20 +37,35 @@ class TranscodeTestRepository(context: Context) { while (usedNotificationIds.contains(notificationId)) { notificationId = Random.nextInt().absoluteValue } + val inputData = Data.Builder() + .putString(TranscodeWorker.KEY_INPUT_URI, it.toString()) + .putString(TranscodeWorker.KEY_OUTPUT_URI, outputDirectory.toString()) + .putInt(TranscodeWorker.KEY_NOTIFICATION_ID, notificationId) + + if (customTranscodingOptions != null) { + inputData.putInt(TranscodeWorker.KEY_LONG_EDGE, customTranscodingOptions.videoResolution.longEdge) + inputData.putInt(TranscodeWorker.KEY_SHORT_EDGE, customTranscodingOptions.videoResolution.shortEdge) + inputData.putInt(TranscodeWorker.KEY_BIT_RATE, customTranscodingOptions.bitrate) + inputData.putBoolean(TranscodeWorker.KEY_ENABLE_FASTSTART, customTranscodingOptions.enableFastStart) + } + val transcodeRequest = OneTimeWorkRequestBuilder() - .setInputData(createInputDataForWorkRequest(it, outputDirectory, notificationId)) + .setInputData(inputData.build()) .addTag(TRANSCODING_WORK_TAG) .build() it to transcodeRequest } - val idsToUris = urisAndRequests.associateBy({ it.second.id }, { it.first }) val requests = urisAndRequests.map { it.second } - var continuation = workManager.beginWith(requests.first()) - for (request in requests.drop(1)) { - continuation = continuation.then(request) + if (forceSequentialProcessing) { + var continuation = workManager.beginWith(requests.first()) + for (request in requests.drop(1)) { + continuation = continuation.then(request) + } + continuation.enqueue() + } else { + workManager.enqueue(requests) } - continuation.enqueue() return idsToUris } @@ -58,19 +76,6 @@ class TranscodeTestRepository(context: Context) { return workManager.getWorkInfosFlow(WorkQuery.fromIds(jobIds)) } - /** - * Creates the input data bundle which includes the blur level to - * update the amount of blur to be applied and the Uri to operate on - * @return Data which contains the Image Uri as a String and blur level as an Integer - */ - private fun createInputDataForWorkRequest(selectedVideo: Uri, outputUri: Uri, notificationId: Int): Data { - return Data.Builder() - .putString(TranscodeWorker.KEY_INPUT_URI, selectedVideo.toString()) - .putString(TranscodeWorker.KEY_OUTPUT_URI, outputUri.toString()) - .putInt(TranscodeWorker.KEY_NOTIFICATION_ID, notificationId) - .build() - } - fun cancelAllTranscodes() { workManager.cancelAllWorkByTag(TRANSCODING_WORK_TAG) workManager.pruneWork() @@ -112,6 +117,8 @@ class TranscodeTestRepository(context: Context) { private data class FileMetadata(val documentId: String, val label: String, val size: Long) + data class CustomTranscodingOptions(val videoResolution: VideoResolution, val bitrate: Int, val enableFastStart: Boolean) + companion object { private const val TAG = "TranscodingTestRepository" const val TRANSCODING_WORK_TAG = "transcoding" diff --git a/video/app/src/main/java/org/thoughtcrime/video/app/transcode/TranscodeTestViewModel.kt b/video/app/src/main/java/org/thoughtcrime/video/app/transcode/TranscodeTestViewModel.kt index 2bae6d444a..f8127166c6 100644 --- a/video/app/src/main/java/org/thoughtcrime/video/app/transcode/TranscodeTestViewModel.kt +++ b/video/app/src/main/java/org/thoughtcrime/video/app/transcode/TranscodeTestViewModel.kt @@ -15,14 +15,24 @@ import androidx.lifecycle.ViewModel import androidx.work.WorkInfo import kotlinx.coroutines.flow.Flow import java.util.UUID +import kotlin.math.roundToInt +/** + * ViewModel for the transcoding screen of the video sample app. See [TranscodeTestActivity]. + */ class TranscodeTestViewModel : ViewModel() { private lateinit var repository: TranscodeTestRepository private var backPressedRunnable = {} private var transcodingJobs: Map = emptyMap() + var outputDirectory: Uri? by mutableStateOf(null) private set var selectedVideos: List by mutableStateOf(emptyList()) + var videoMegaBitrate = DEFAULT_VIDEO_MEGABITRATE + var videoResolution = VideoResolution.HD + var useAutoTranscodingSettings = true + var enableFastStart = true + var forceSequentialQueueProcessing = false fun initialize(context: Context) { repository = TranscodeTestRepository(context) @@ -31,7 +41,11 @@ class TranscodeTestViewModel : ViewModel() { fun transcode() { val output = outputDirectory ?: throw IllegalStateException("No output directory selected!") - transcodingJobs = repository.transcode(selectedVideos, output) + if (useAutoTranscodingSettings) { + transcodingJobs = repository.transcode(selectedVideos, output, forceSequentialQueueProcessing, null) + } else { + transcodingJobs = repository.transcode(selectedVideos, output, forceSequentialQueueProcessing, TranscodeTestRepository.CustomTranscodingOptions(videoResolution, (videoMegaBitrate * MEGABIT).roundToInt(), enableFastStart)) + } } fun getTranscodingJobsAsState(): Flow> { @@ -61,4 +75,8 @@ class TranscodeTestViewModel : ViewModel() { fun resetOutputDirectory() { outputDirectory = null } + + companion object { + private const val MEGABIT = 1000000 + } } diff --git a/video/app/src/main/java/org/thoughtcrime/video/app/transcode/TranscodeWorker.kt b/video/app/src/main/java/org/thoughtcrime/video/app/transcode/TranscodeWorker.kt index f96ede4039..d00dc6dd72 100644 --- a/video/app/src/main/java/org/thoughtcrime/video/app/transcode/TranscodeWorker.kt +++ b/video/app/src/main/java/org/thoughtcrime/video/app/transcode/TranscodeWorker.kt @@ -5,16 +5,15 @@ package org.thoughtcrime.video.app.transcode -import android.app.NotificationChannel -import android.app.NotificationManager +import android.app.PendingIntent import android.content.Context -import android.content.Context.NOTIFICATION_SERVICE +import android.content.Intent import android.content.pm.ServiceInfo.FOREGROUND_SERVICE_TYPE_DATA_SYNC import android.net.Uri import android.os.Build -import android.os.ParcelFileDescriptor import android.util.Log import androidx.core.app.NotificationCompat +import androidx.core.app.TaskStackBuilder import androidx.documentfile.provider.DocumentFile import androidx.media3.common.util.UnstableApi import androidx.work.CoroutineWorker @@ -24,70 +23,118 @@ import androidx.work.WorkManager import androidx.work.WorkerParameters import org.signal.core.util.getLength import org.thoughtcrime.securesms.video.StreamingTranscoder +import org.thoughtcrime.securesms.video.postprocessing.Mp4FaststartPostProcessor import org.thoughtcrime.securesms.video.videoconverter.VideoConstants import org.thoughtcrime.securesms.video.videoconverter.mediadatasource.InputStreamMediaDataSource import org.thoughtcrime.video.app.R +import java.io.FileInputStream import java.io.FileOutputStream import java.io.InputStream import java.time.Instant -class TranscodeWorker(private val ctx: Context, params: WorkerParameters) : CoroutineWorker(ctx, params) { +/** + * A WorkManager worker to transcode videos in the background. This utilizes [StreamingTranscoder]. + */ +class TranscodeWorker(ctx: Context, params: WorkerParameters) : CoroutineWorker(ctx, params) { @UnstableApi override suspend fun doWork(): Result { + val logPrefix = "[Job ${id.toString().takeLast(4)}]" val notificationId = inputData.getInt(KEY_NOTIFICATION_ID, -1) if (notificationId < 0) { - Log.w(TAG, "Notification ID was null!") + Log.w(TAG, "$logPrefix Notification ID was null!") return Result.failure() } val inputUri = inputData.getString(KEY_INPUT_URI) if (inputUri == null) { - Log.w(TAG, "Input URI was null!") + Log.w(TAG, "$logPrefix Input URI was null!") return Result.failure() } val outputDirUri = inputData.getString(KEY_OUTPUT_URI) if (outputDirUri == null) { - Log.w(TAG, "Output URI was null!") + Log.w(TAG, "$logPrefix Output URI was null!") return Result.failure() } - val input = DocumentFile.fromSingleUri(ctx, Uri.parse(inputUri))?.name + val postProcessForFastStart = inputData.getBoolean(KEY_ENABLE_FASTSTART, true) + val resolution = inputData.getInt(KEY_SHORT_EDGE, -1) + val desiredBitrate = inputData.getInt(KEY_BIT_RATE, -1) + val input = DocumentFile.fromSingleUri(applicationContext, Uri.parse(inputUri))?.name if (input == null) { - Log.w(TAG, "Could not read input file name!") + Log.w(TAG, "$logPrefix Could not read input file name!") return Result.failure() } - val outputFileUri = createFile(Uri.parse(outputDirUri), "transcoded-${Instant.now()}-$input$OUTPUT_FILE_EXTENSION") + val filenameBase = "transcoded-${Instant.now()}-$input" + val tempFilename = "$filenameBase$TEMP_FILE_EXTENSION" + val finalFilename = "$filenameBase$OUTPUT_FILE_EXTENSION" - if (outputFileUri == null) { - Log.w(TAG, "Could not create output file!") + val tempFile = createFile(Uri.parse(outputDirUri), tempFilename) + if (tempFile == null) { + Log.w(TAG, "$logPrefix Could not create temp file!") return Result.failure() } - val datasource = WorkerMediaDataSource(ctx, Uri.parse(inputUri)) + val datasource = WorkerMediaDataSource(applicationContext, Uri.parse(inputUri)) if (Build.VERSION.SDK_INT < Build.VERSION_CODES.O) { - Log.w(TAG, "Transcoder is only supported on API 26+!") + Log.w(TAG, "$logPrefix Transcoder is only supported on API 26+!") return Result.failure() } - val transcoder = StreamingTranscoder(datasource, null, 50 * 1024 * 1024) // TODO: set options + + val transcoder = if (resolution > 0 && desiredBitrate > 0) { + StreamingTranscoder(datasource, null, desiredBitrate, resolution) + } else { + StreamingTranscoder(datasource, null, DEFAULT_FILE_SIZE_LIMIT) + } + setForeground(createForegroundInfo(-1, notificationId)) - ctx.contentResolver.openFileDescriptor(outputFileUri, "w").use { it: ParcelFileDescriptor? -> - if (it == null) { - Log.w(TAG, "Could not open output file for writing!") + + applicationContext.contentResolver.openFileDescriptor(tempFile.uri, "w").use { tempFd -> + if (tempFd == null) { + Log.w(TAG, "$logPrefix Could not open temp file for I/O!") return Result.failure() } - transcoder.transcode( - { percent: Int -> - setProgressAsync(Data.Builder().putInt(KEY_PROGRESS, percent).build()) - setForegroundAsync(createForegroundInfo(percent, notificationId)) - }, - FileOutputStream(it.fileDescriptor), - { isStopped } - ) - return Result.success() + transcoder.transcode({ percent: Int -> + setProgressAsync(Data.Builder().putInt(KEY_PROGRESS, percent).build()) + setForegroundAsync(createForegroundInfo(percent, notificationId)) + }, FileOutputStream(tempFd.fileDescriptor), { isStopped }) } + Log.v(TAG, "$logPrefix Initial transcode completed successfully!") + if (!postProcessForFastStart) { + tempFile.renameTo(finalFilename) + Log.v(TAG, "$logPrefix Rename successful.") + } else { + applicationContext.contentResolver.openFileDescriptor(tempFile.uri, "r").use { tempFd -> + if (tempFd == null) { + Log.w(TAG, "$logPrefix Could not open temp file for I/O!") + return Result.failure() + } + + val finalFile = createFile(Uri.parse(outputDirUri), finalFilename) + if (finalFile == null) { + Log.w(TAG, "$logPrefix Could not create final file for faststart processing!") + return Result.failure() + } + applicationContext.contentResolver.openFileDescriptor(finalFile.uri, "w").use { finalFd -> + if (finalFd == null) { + Log.w(TAG, "$logPrefix Could not open output file for I/O!") + return Result.failure() + } + + Mp4FaststartPostProcessor({ FileInputStream(tempFd.fileDescriptor) }, tempFd.statSize).processAndWriteTo(FileOutputStream(finalFd.fileDescriptor)) + + if (!tempFile.delete()) { + Log.w(TAG, "$logPrefix Failed to delete temp file after processing!") + return Result.failure() + } + } + } + Log.v(TAG, "$logPrefix Faststart postprocess successful.") + } + Log.v(TAG, "$logPrefix Overall transcode job successful.") + return Result.success() } private fun createForegroundInfo(progress: Int, notificationId: Int): ForegroundInfo { @@ -96,23 +143,19 @@ class TranscodeWorker(private val ctx: Context, params: WorkerParameters) : Coro val cancel = applicationContext.getString(R.string.cancel_transcode) val intent = WorkManager.getInstance(applicationContext) .createCancelPendingIntent(getId()) - - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { - val name = applicationContext.getString(R.string.channel_name) - val descriptionText = applicationContext.getString(R.string.channel_description) - val importance = NotificationManager.IMPORTANCE_LOW - val mChannel = NotificationChannel(id, name, importance) - mChannel.description = descriptionText - val notificationManager = applicationContext.getSystemService(NOTIFICATION_SERVICE) as NotificationManager - notificationManager.createNotificationChannel(mChannel) + val transcodeActivityIntent = Intent(applicationContext, TranscodeTestActivity::class.java) + val pendingIntent: PendingIntent? = TaskStackBuilder.create(applicationContext).run { + addNextIntentWithParentStack(transcodeActivityIntent) + getPendingIntent(0, + PendingIntent.FLAG_UPDATE_CURRENT or PendingIntent.FLAG_IMMUTABLE) } - val notification = NotificationCompat.Builder(applicationContext, id) .setContentTitle(title) .setTicker(title) - .setProgress(100, progress, progress >= 0) + .setProgress(100, progress, progress <= 0) .setSmallIcon(R.drawable.ic_work_notification) .setOngoing(true) + .setContentIntent(pendingIntent) .addAction(android.R.drawable.ic_delete, cancel, intent) .build() @@ -123,8 +166,8 @@ class TranscodeWorker(private val ctx: Context, params: WorkerParameters) : Coro } } - private fun createFile(treeUri: Uri, filename: String): Uri? { - return DocumentFile.fromTreeUri(ctx, treeUri)?.createFile(VideoConstants.VIDEO_MIME_TYPE, filename)?.uri + private fun createFile(treeUri: Uri, filename: String): DocumentFile? { + return DocumentFile.fromTreeUri(applicationContext, treeUri)?.createFile(VideoConstants.VIDEO_MIME_TYPE, filename) } private class WorkerMediaDataSource(context: Context, private val uri: Uri) : InputStreamMediaDataSource() { @@ -154,9 +197,15 @@ class TranscodeWorker(private val ctx: Context, params: WorkerParameters) : Coro companion object { private const val TAG = "TranscodeWorker" private const val OUTPUT_FILE_EXTENSION = ".mp4" + private const val TEMP_FILE_EXTENSION = ".tmp" + private const val DEFAULT_FILE_SIZE_LIMIT: Long = 50 * 1024 * 1024 const val KEY_INPUT_URI = "input_uri" const val KEY_OUTPUT_URI = "output_uri" const val KEY_PROGRESS = "progress" + const val KEY_LONG_EDGE = "resolution_long_edge" + const val KEY_SHORT_EDGE = "resolution_short_edge" + const val KEY_BIT_RATE = "video_bit_rate" + const val KEY_ENABLE_FASTSTART = "video_enable_faststart" const val KEY_NOTIFICATION_ID = "notification_id" } } diff --git a/video/app/src/main/java/org/thoughtcrime/video/app/transcode/composables/ConfigurationSelection.kt b/video/app/src/main/java/org/thoughtcrime/video/app/transcode/composables/ConfigurationSelection.kt new file mode 100644 index 0000000000..bbd5564bb1 --- /dev/null +++ b/video/app/src/main/java/org/thoughtcrime/video/app/transcode/composables/ConfigurationSelection.kt @@ -0,0 +1,215 @@ +/* + * Copyright 2024 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ + +package org.thoughtcrime.video.app.transcode.composables + +import android.net.Uri +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.selection.selectable +import androidx.compose.foundation.selection.selectableGroup +import androidx.compose.material3.Checkbox +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.RadioButton +import androidx.compose.material3.Slider +import androidx.compose.material3.SliderDefaults +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableFloatStateOf +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember +import androidx.compose.runtime.setValue +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.semantics.Role +import androidx.compose.ui.semantics.contentDescription +import androidx.compose.ui.semantics.semantics +import androidx.compose.ui.text.font.FontFamily +import androidx.compose.ui.text.style.TextAlign +import androidx.compose.ui.tooling.preview.Preview +import androidx.compose.ui.unit.dp +import androidx.compose.ui.unit.sp +import org.thoughtcrime.video.app.transcode.DEFAULT_VIDEO_MEGABITRATE +import org.thoughtcrime.video.app.transcode.MAX_VIDEO_MEGABITRATE +import org.thoughtcrime.video.app.transcode.MIN_VIDEO_MEGABITRATE +import org.thoughtcrime.video.app.transcode.VideoResolution +import org.thoughtcrime.video.app.ui.composables.LabeledButton +import kotlin.math.roundToInt + +/** + * A view that shows the queue of video URIs to encode, and allows you to change the encoding options. + */ +@Composable +fun ConfigureEncodingParameters( + videoUris: List, + onAutoSettingsCheckChanged: (Boolean) -> Unit, + onRadioButtonSelected: (VideoResolution) -> Unit, + onSliderValueChanged: (Float) -> Unit, + onFastStartSettingCheckChanged: (Boolean) -> Unit, + onSequentialSettingCheckChanged: (Boolean) -> Unit, + buttonClickListener: () -> Unit, + modifier: Modifier = Modifier, + initialSettingsAutoSelected: Boolean = true +) { + var sliderPosition by remember { mutableFloatStateOf(DEFAULT_VIDEO_MEGABITRATE) } + var selectedResolution by remember { mutableStateOf(VideoResolution.HD) } + val autoSettingsChecked = remember { mutableStateOf(initialSettingsAutoSelected) } + val fastStartChecked = remember { mutableStateOf(true) } + val sequentialProcessingChecked = remember { mutableStateOf(false) } + Column( + verticalArrangement = Arrangement.Center, + horizontalAlignment = Alignment.CenterHorizontally, + modifier = modifier.padding(16.dp) + ) { + Text( + text = "Selected videos:", + modifier = Modifier + .align(Alignment.Start) + .padding(16.dp) + ) + videoUris.forEach { + Text( + text = it.toString(), + fontSize = 8.sp, + fontFamily = FontFamily.Monospace, + modifier = Modifier + .align(Alignment.Start) + .padding(horizontal = 16.dp) + ) + } + Row( + verticalAlignment = Alignment.CenterVertically, + modifier = Modifier + .padding(vertical = 8.dp, horizontal = 8.dp) + .fillMaxWidth() + ) { + Checkbox( + checked = autoSettingsChecked.value, + onCheckedChange = { isChecked -> + autoSettingsChecked.value = isChecked + onAutoSettingsCheckChanged(isChecked) + } + ) + Text( + text = "Calculate Output Settings Automatically", + style = MaterialTheme.typography.bodySmall + ) + } + if (!autoSettingsChecked.value) { + Row( + modifier = Modifier + .padding(vertical = 16.dp) + .fillMaxWidth() + .selectableGroup() + ) { + VideoResolution.values().forEach { + Column( + horizontalAlignment = Alignment.CenterHorizontally, + modifier = Modifier + .selectable( + selected = selectedResolution == it, + onClick = { + selectedResolution = it + onRadioButtonSelected(it) + }, + role = Role.RadioButton + ) + ) { + RadioButton( + selected = selectedResolution == it, + onClick = null, + modifier = Modifier.semantics { contentDescription = it.getContentDescription() } + ) + Text( + text = "${it.shortEdge}p", + textAlign = TextAlign.Center, + modifier = Modifier.padding(start = 16.dp) + ) + } + } + } + Slider( + value = sliderPosition, + onValueChange = { + sliderPosition = it + onSliderValueChanged(it) + }, + colors = SliderDefaults.colors( + thumbColor = MaterialTheme.colorScheme.secondary, + activeTrackColor = MaterialTheme.colorScheme.secondary, + inactiveTrackColor = MaterialTheme.colorScheme.secondaryContainer + ), + steps = 5, + valueRange = MIN_VIDEO_MEGABITRATE..MAX_VIDEO_MEGABITRATE, + modifier = Modifier.padding(top = 16.dp, start = 16.dp, end = 16.dp) + ) + Text(text = "${sliderPosition.roundToInt()} Mbit/s") + Row( + verticalAlignment = Alignment.CenterVertically, + modifier = Modifier + .padding(vertical = 8.dp, horizontal = 8.dp) + .fillMaxWidth() + ) { + Checkbox( + checked = fastStartChecked.value, + onCheckedChange = { isChecked -> + fastStartChecked.value = isChecked + onFastStartSettingCheckChanged(isChecked) + } + ) + Text(text = "Enable postprocessing for faststart", style = MaterialTheme.typography.bodySmall) + } + Row( + verticalAlignment = Alignment.CenterVertically, + modifier = Modifier + .padding(vertical = 8.dp, horizontal = 8.dp) + .fillMaxWidth() + ) { + Checkbox( + checked = sequentialProcessingChecked.value, + onCheckedChange = { isChecked -> + sequentialProcessingChecked.value = isChecked + onSequentialSettingCheckChanged(isChecked) + } + ) + Text(text = "Force Sequential Queue Processing", style = MaterialTheme.typography.bodySmall) + } + } + LabeledButton(buttonLabel = "Transcode", onClick = buttonClickListener, modifier = Modifier.padding(vertical = 8.dp)) + } +} + +@Preview +@Composable +private fun ConfigurationScreenPreviewChecked() { + ConfigureEncodingParameters( + videoUris = listOf(Uri.parse("content://1"), Uri.parse("content://2")), + onAutoSettingsCheckChanged = {}, + onRadioButtonSelected = {}, + onSliderValueChanged = {}, + onFastStartSettingCheckChanged = {}, + onSequentialSettingCheckChanged = {}, + buttonClickListener = {} + ) +} + +@Preview +@Composable +private fun ConfigurationScreenPreviewUnchecked() { + ConfigureEncodingParameters( + videoUris = listOf(Uri.parse("content://1"), Uri.parse("content://2")), + onAutoSettingsCheckChanged = {}, + onRadioButtonSelected = {}, + onSliderValueChanged = {}, + onFastStartSettingCheckChanged = {}, + onSequentialSettingCheckChanged = {}, + buttonClickListener = {}, + initialSettingsAutoSelected = false + ) +} diff --git a/video/app/src/main/java/org/thoughtcrime/video/app/transcode/composables/InputSelection.kt b/video/app/src/main/java/org/thoughtcrime/video/app/transcode/composables/InputSelection.kt new file mode 100644 index 0000000000..31a9a6d65b --- /dev/null +++ b/video/app/src/main/java/org/thoughtcrime/video/app/transcode/composables/InputSelection.kt @@ -0,0 +1,33 @@ +/* + * Copyright 2024 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ + +package org.thoughtcrime.video.app.transcode.composables + +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Column +import androidx.compose.runtime.Composable +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.tooling.preview.Preview +import org.thoughtcrime.video.app.ui.composables.LabeledButton + +/** + * A view that prompts you to select input videos for transcoding. + */ +@Composable +fun SelectInput(modifier: Modifier = Modifier, onClick: () -> Unit) { + Column( + verticalArrangement = Arrangement.Center, + horizontalAlignment = Alignment.CenterHorizontally + ) { + LabeledButton("Select Videos", onClick = onClick, modifier = modifier) + } +} + +@Preview +@Composable +private fun InputSelectionPreview() { + SelectInput { } +} diff --git a/video/app/src/main/java/org/thoughtcrime/video/app/transcode/composables/OutputSelection.kt b/video/app/src/main/java/org/thoughtcrime/video/app/transcode/composables/OutputSelection.kt new file mode 100644 index 0000000000..b4c1aa863e --- /dev/null +++ b/video/app/src/main/java/org/thoughtcrime/video/app/transcode/composables/OutputSelection.kt @@ -0,0 +1,33 @@ +/* + * Copyright 2024 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ + +package org.thoughtcrime.video.app.transcode.composables + +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Column +import androidx.compose.runtime.Composable +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.tooling.preview.Preview +import org.thoughtcrime.video.app.ui.composables.LabeledButton + +/** + * A view that prompts you to select an output directory that transcoded videos will be saved to. + */ +@Composable +fun SelectOutput(modifier: Modifier = Modifier, onClick: () -> Unit) { + Column( + verticalArrangement = Arrangement.Center, + horizontalAlignment = Alignment.CenterHorizontally + ) { + LabeledButton("Select Output Directory", onClick = onClick, modifier = modifier) + } +} + +@Preview +@Composable +private fun OutputSelectionPreview() { + SelectOutput { } +} diff --git a/video/app/src/main/java/org/thoughtcrime/video/app/transcode/composables/Progress.kt b/video/app/src/main/java/org/thoughtcrime/video/app/transcode/composables/Progress.kt new file mode 100644 index 0000000000..31edbfbdb3 --- /dev/null +++ b/video/app/src/main/java/org/thoughtcrime/video/app/transcode/composables/Progress.kt @@ -0,0 +1,63 @@ +/* + * Copyright 2024 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ + +package org.thoughtcrime.video.app.transcode.composables + +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.padding +import androidx.compose.material3.LinearProgressIndicator +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.text.style.TextAlign +import androidx.compose.ui.tooling.preview.Preview +import androidx.compose.ui.unit.dp +import androidx.work.WorkInfo +import org.thoughtcrime.video.app.transcode.TranscodeWorker +import org.thoughtcrime.video.app.ui.composables.LabeledButton + +/** + * A view that shows the current encodes in progress. + */ +@Composable +fun TranscodingJobProgress(transcodingJobs: List, resetButtonOnClick: () -> Unit, modifier: Modifier = Modifier) { + Column( + verticalArrangement = Arrangement.Center, + horizontalAlignment = Alignment.CenterHorizontally + ) { + transcodingJobs.forEach { workInfo -> + val currentProgress = workInfo.progress.getInt(TranscodeWorker.KEY_PROGRESS, -1) + Row( + verticalAlignment = Alignment.CenterVertically, + modifier = modifier.padding(horizontal = 16.dp) + ) { + val progressIndicatorModifier = Modifier.weight(3f) + Text( + text = "Job ${workInfo.id.toString().takeLast(4)}", + modifier = Modifier + .padding(end = 16.dp) + .weight(1f) + ) + if (workInfo.state.isFinished) { + Text(text = workInfo.state.toString(), textAlign = TextAlign.Center, modifier = progressIndicatorModifier) + } else if (currentProgress >= 0) { + LinearProgressIndicator(progress = currentProgress / 100f, modifier = progressIndicatorModifier) + } else { + LinearProgressIndicator(modifier = progressIndicatorModifier) + } + } + } + LabeledButton("Reset/Cancel", onClick = resetButtonOnClick) + } +} + +@Preview +@Composable +private fun ProgressScreenPreview() { + TranscodingJobProgress(emptyList(), resetButtonOnClick = {}) +} diff --git a/video/lib/build.gradle.kts b/video/lib/build.gradle.kts index 176d39d5ba..02c75964af 100644 --- a/video/lib/build.gradle.kts +++ b/video/lib/build.gradle.kts @@ -8,6 +8,8 @@ android { dependencies { implementation(project(":core-util")) + implementation(libs.libsignal.android) + implementation(libs.google.guava.android) implementation(libs.bundles.mp4parser) { exclude(group = "junit", module = "junit") diff --git a/video/lib/src/main/java/org/thoughtcrime/securesms/video/StreamingTranscoder.java b/video/lib/src/main/java/org/thoughtcrime/securesms/video/StreamingTranscoder.java index 1b752d090f..f99ccaac91 100644 --- a/video/lib/src/main/java/org/thoughtcrime/securesms/video/StreamingTranscoder.java +++ b/video/lib/src/main/java/org/thoughtcrime/securesms/video/StreamingTranscoder.java @@ -12,6 +12,7 @@ import org.thoughtcrime.securesms.video.exceptions.VideoSizeException; import org.thoughtcrime.securesms.video.exceptions.VideoSourceException; import org.thoughtcrime.securesms.video.videoconverter.EncodingException; import org.thoughtcrime.securesms.video.videoconverter.MediaConverter; +import org.thoughtcrime.securesms.video.videoconverter.VideoConstants; import org.thoughtcrime.securesms.video.videoconverter.mediadatasource.MediaDataSourceMediaInput; import java.io.FilterOutputStream; @@ -25,15 +26,15 @@ public final class StreamingTranscoder { private static final String TAG = Log.tag(StreamingTranscoder.class); - private final MediaDataSource dataSource; - private final long upperSizeLimit; - private final long inSize; - private final long duration; - private final int inputBitRate; - private final VideoBitRateCalculator.Quality targetQuality; - private final boolean transcodeRequired; - private final long fileSizeEstimate; - private final @Nullable TranscoderOptions options; + private final MediaDataSource dataSource; + private final long upperSizeLimit; + private final long inSize; + private final long duration; + private final int inputBitRate; + private final TranscodingQuality targetQuality; + private final boolean transcodeRequired; + private final long fileSizeEstimate; + private final @Nullable TranscoderOptions options; /** * @param upperSizeLimit A upper size to transcode to. The actual output size can be up to 10% smaller. @@ -68,6 +69,34 @@ public final class StreamingTranscoder { this.fileSizeEstimate = targetQuality.getFileSizeEstimate(); } + public StreamingTranscoder(@NonNull MediaDataSource dataSource, + @Nullable TranscoderOptions options, + int videoBitrate, + int shortEdge) + throws IOException, VideoSourceException + { + this.dataSource = dataSource; + this.options = options; + + final MediaMetadataRetriever mediaMetadataRetriever = new MediaMetadataRetriever(); + try { + mediaMetadataRetriever.setDataSource(dataSource); + } catch (RuntimeException e) { + Log.w(TAG, "Unable to read datasource", e); + throw new VideoSourceException("Unable to read datasource", e); + } + + this.inSize = dataSource.getSize(); + this.duration = getDuration(mediaMetadataRetriever); + this.inputBitRate = VideoBitRateCalculator.bitRate(inSize, duration); + this.targetQuality = new TranscodingQuality(videoBitrate, VideoConstants.AUDIO_BIT_RATE, 1.0, duration, shortEdge); + this.upperSizeLimit = Long.MAX_VALUE; + + this.transcodeRequired = true; + + this.fileSizeEstimate = targetQuality.getFileSizeEstimate(); + } + public void transcode(@NonNull Progress progress, @NonNull OutputStream stream, @Nullable TranscoderCancelationSignal cancelationSignal) diff --git a/video/lib/src/main/java/org/thoughtcrime/securesms/video/TranscodingQuality.kt b/video/lib/src/main/java/org/thoughtcrime/securesms/video/TranscodingQuality.kt new file mode 100644 index 0000000000..b264078585 --- /dev/null +++ b/video/lib/src/main/java/org/thoughtcrime/securesms/video/TranscodingQuality.kt @@ -0,0 +1,29 @@ +/* + * Copyright 2024 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ +package org.thoughtcrime.securesms.video + +/** + * A data class to hold various video transcoding parameters, such as bitrate. + */ +data class TranscodingQuality(val targetVideoBitRate: Int, val targetAudioBitRate: Int, val quality: Double, private val duration: Long, val outputResolution: Int) { + init { + if (quality < 0.0 || quality > 1.0) { + throw IllegalArgumentException("Quality $quality is outside of accepted range [0.0, 1.0]!") + } + } + + val targetTotalBitRate = targetVideoBitRate + targetAudioBitRate + val fileSizeEstimate = targetTotalBitRate * duration / 8000 + + override fun toString(): String { + return "Quality{" + + "targetVideoBitRate=" + targetVideoBitRate + + ", targetAudioBitRate=" + targetAudioBitRate + + ", quality=" + quality + + ", duration=" + duration + + ", filesize=" + fileSizeEstimate + + '}' + } +} diff --git a/video/lib/src/main/java/org/thoughtcrime/securesms/video/VideoBitRateCalculator.java b/video/lib/src/main/java/org/thoughtcrime/securesms/video/VideoBitRateCalculator.java index a7304de8ff..4045c8af04 100644 --- a/video/lib/src/main/java/org/thoughtcrime/securesms/video/VideoBitRateCalculator.java +++ b/video/lib/src/main/java/org/thoughtcrime/securesms/video/VideoBitRateCalculator.java @@ -10,8 +10,6 @@ public final class VideoBitRateCalculator { private static final int MAXIMUM_TARGET_VIDEO_BITRATE = VideoConstants.VIDEO_BIT_RATE; private static final int LOW_RES_TARGET_VIDEO_BITRATE = 1_750_000; private static final int MINIMUM_TARGET_VIDEO_BITRATE = 500_000; - private static final int AUDIO_BITRATE = VideoConstants.AUDIO_BIT_RATE; - private static final int OUTPUT_FORMAT = VideoConstants.VIDEO_SHORT_EDGE; private static final int LOW_RES_OUTPUT_FORMAT = 480; private final long upperFileSizeLimitWithMargin; @@ -23,21 +21,22 @@ public final class VideoBitRateCalculator { /** * Gets the output quality of a video of the given {@param duration}. */ - public Quality getTargetQuality(long duration, int inputTotalBitRate) { - int maxVideoBitRate = Math.min(MAXIMUM_TARGET_VIDEO_BITRATE, inputTotalBitRate - AUDIO_BITRATE); + public TranscodingQuality getTargetQuality(long duration, int inputTotalBitRate) { + int maxVideoBitRate = Math.min(MAXIMUM_TARGET_VIDEO_BITRATE, inputTotalBitRate - VideoConstants.AUDIO_BIT_RATE); int minVideoBitRate = Math.min(MINIMUM_TARGET_VIDEO_BITRATE, maxVideoBitRate); int targetVideoBitRate = Math.max(minVideoBitRate, Math.min(getTargetVideoBitRate(upperFileSizeLimitWithMargin, duration), maxVideoBitRate)); int bitRateRange = maxVideoBitRate - minVideoBitRate; double quality = bitRateRange == 0 ? 1 : (targetVideoBitRate - minVideoBitRate) / (double) bitRateRange; + int outputResolution = targetVideoBitRate < LOW_RES_TARGET_VIDEO_BITRATE ? LOW_RES_OUTPUT_FORMAT : VideoConstants.VIDEO_SHORT_EDGE; - return new Quality(targetVideoBitRate, AUDIO_BITRATE, quality, duration); + return new TranscodingQuality(targetVideoBitRate, VideoConstants.AUDIO_BIT_RATE, Math.max(0, Math.min(quality, 1)), duration, outputResolution); } private int getTargetVideoBitRate(long sizeGuideBytes, long duration) { double durationSeconds = duration / 1000d; - sizeGuideBytes -= durationSeconds * AUDIO_BITRATE / 8; + sizeGuideBytes -= durationSeconds * VideoConstants.AUDIO_BIT_RATE / 8; double targetAttachmentSizeBits = sizeGuideBytes * 8L; @@ -48,63 +47,4 @@ public final class VideoBitRateCalculator { return (int) (bytes * 8 / (durationMs / 1000f)); } - public static class Quality { - private final int targetVideoBitRate; - private final int targetAudioBitRate; - private final double quality; - private final long duration; - - private Quality(int targetVideoBitRate, int targetAudioBitRate, double quality, long duration) { - this.targetVideoBitRate = targetVideoBitRate; - this.targetAudioBitRate = targetAudioBitRate; - this.quality = Math.max(0, Math.min(quality, 1)); - this.duration = duration; - } - - /** - * [0..1] - *

- * 0 = {@link #MINIMUM_TARGET_VIDEO_BITRATE} - * 1 = {@link #MAXIMUM_TARGET_VIDEO_BITRATE} - */ - public double getQuality() { - return quality; - } - - public int getTargetVideoBitRate() { - return targetVideoBitRate; - } - - public int getTargetAudioBitRate() { - return targetAudioBitRate; - } - - public int getTargetTotalBitRate() { - return targetVideoBitRate + targetAudioBitRate; - } - - public boolean useLowRes() { - return targetVideoBitRate < LOW_RES_TARGET_VIDEO_BITRATE; - } - - public int getOutputResolution() { - return useLowRes() ? LOW_RES_OUTPUT_FORMAT - : OUTPUT_FORMAT; - } - - public long getFileSizeEstimate() { - return getTargetTotalBitRate() * duration / 8000; - } - - @Override - public String toString() { - return "Quality{" + - "targetVideoBitRate=" + targetVideoBitRate + - ", targetAudioBitRate=" + targetAudioBitRate + - ", quality=" + quality + - ", duration=" + duration + - ", filesize=" + getFileSizeEstimate() + - '}'; - } - } } diff --git a/video/lib/src/main/java/org/thoughtcrime/securesms/video/exceptions/VideoPostProcessingException.kt b/video/lib/src/main/java/org/thoughtcrime/securesms/video/exceptions/VideoPostProcessingException.kt new file mode 100644 index 0000000000..bfaf97e031 --- /dev/null +++ b/video/lib/src/main/java/org/thoughtcrime/securesms/video/exceptions/VideoPostProcessingException.kt @@ -0,0 +1,11 @@ +/* + * Copyright 2024 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ + +package org.thoughtcrime.securesms.video.exceptions + +class VideoPostProcessingException : RuntimeException { + internal constructor(message: String?) : super(message) + internal constructor(message: String?, inner: Exception?) : super(message, inner) +} diff --git a/video/lib/src/main/java/org/thoughtcrime/securesms/video/exceptions/VideoSizeException.kt b/video/lib/src/main/java/org/thoughtcrime/securesms/video/exceptions/VideoSizeException.kt index 068ba7dbaf..71b711e7ad 100644 --- a/video/lib/src/main/java/org/thoughtcrime/securesms/video/exceptions/VideoSizeException.kt +++ b/video/lib/src/main/java/org/thoughtcrime/securesms/video/exceptions/VideoSizeException.kt @@ -9,4 +9,4 @@ import java.io.IOException /** * Exception to denote when video processing has been unable to meet its output file size requirements. */ -class VideoSizeException internal constructor(message: String?) : IOException(message) +class VideoSizeException(message: String?) : IOException(message) diff --git a/video/lib/src/main/java/org/thoughtcrime/securesms/video/postprocessing/Mp4FaststartPostProcessor.kt b/video/lib/src/main/java/org/thoughtcrime/securesms/video/postprocessing/Mp4FaststartPostProcessor.kt new file mode 100644 index 0000000000..cf8a8874f1 --- /dev/null +++ b/video/lib/src/main/java/org/thoughtcrime/securesms/video/postprocessing/Mp4FaststartPostProcessor.kt @@ -0,0 +1,43 @@ +/* + * Copyright 2024 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ + +package org.thoughtcrime.securesms.video.postprocessing + +import com.google.common.io.ByteStreams +import org.signal.libsignal.media.Mp4Sanitizer +import org.signal.libsignal.media.SanitizedMetadata +import org.thoughtcrime.securesms.video.exceptions.VideoPostProcessingException +import java.io.ByteArrayInputStream +import java.io.InputStream +import java.io.OutputStream +import java.io.SequenceInputStream + +/** + * A post processor that takes a stream of bytes, and using [Mp4Sanitizer], moves the metadata to the front of the file. + * + * @property inputStreamFactory factory for the [InputStream]. May be called multiple times. + * @property inputLength the exact stream of the [InputStream] + */ +class Mp4FaststartPostProcessor(private val inputStreamFactory: () -> InputStream, private val inputLength: Long) { + fun process(): InputStream { + val metadata: SanitizedMetadata? = Mp4Sanitizer.sanitize(inputStreamFactory(), inputLength) + + if (metadata?.sanitizedMetadata == null) { + throw VideoPostProcessingException("Mp4Sanitizer could not parse media metadata!") + } + + val inputStream = inputStreamFactory() + inputStream.skip(metadata.dataOffset) + return SequenceInputStream(ByteArrayInputStream(metadata.sanitizedMetadata), ByteStreams.limit(inputStream, metadata.dataLength)) + } + + fun processAndWriteTo(outputStream: OutputStream) { + process().copyTo(outputStream) + } + + companion object { + const val TAG = "Mp4Faststart" + } +}