mirror of
https://github.com/signalapp/Signal-Desktop.git
synced 2026-05-08 08:58:38 +01:00
Calling: Move VideoSupport.ts to Client
This commit is contained in:
Jim Gustafson
GitHub
@@ -0,0 +1,498 @@
|
||||
// Copyright 2025 Signal Messenger, LLC
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
/* eslint-disable max-classes-per-file */
|
||||
/* eslint-disable no-restricted-syntax */
|
||||
/* eslint-disable @typescript-eslint/no-explicit-any */
|
||||
/* eslint-disable no-await-in-loop */
|
||||
|
||||
import { videoPixelFormatToEnum } from '@signalapp/ringrtc';
|
||||
import type { VideoFrameSender, VideoFrameSource } from '@signalapp/ringrtc';
|
||||
import type { RefObject } from 'react';
|
||||
import * as log from '../logging/log';
|
||||
|
||||
export class GumVideoCaptureOptions {
|
||||
maxWidth = 640;
|
||||
maxHeight = 480;
|
||||
maxFramerate = 30;
|
||||
preferredDeviceId?: string;
|
||||
screenShareSourceId?: string;
|
||||
mediaStream?: MediaStream;
|
||||
onEnded?: () => void;
|
||||
}
|
||||
|
||||
interface GumConstraints extends MediaStreamConstraints {
|
||||
video?: boolean | GumTrackConstraints;
|
||||
}
|
||||
|
||||
interface GumTrackConstraints extends MediaTrackConstraints {
|
||||
mandatory?: GumTrackConstraintSet;
|
||||
}
|
||||
|
||||
type GumTrackConstraintSet = {
|
||||
chromeMediaSource: string;
|
||||
chromeMediaSourceId?: string;
|
||||
maxWidth: number;
|
||||
maxHeight: number;
|
||||
minFrameRate: number;
|
||||
maxFrameRate: number;
|
||||
};
|
||||
|
||||
export class GumVideoCapturer {
|
||||
private defaultCaptureOptions: GumVideoCaptureOptions;
|
||||
private localPreview?: RefObject<HTMLVideoElement>;
|
||||
private captureOptions?: GumVideoCaptureOptions;
|
||||
private sender?: VideoFrameSender;
|
||||
private mediaStream?: MediaStream;
|
||||
private spawnedSenderRunning = false;
|
||||
private preferredDeviceId?: string;
|
||||
private updateLocalPreviewIntervalId?: any;
|
||||
|
||||
constructor(defaultCaptureOptions: GumVideoCaptureOptions) {
|
||||
this.defaultCaptureOptions = defaultCaptureOptions;
|
||||
}
|
||||
|
||||
capturing(): boolean {
|
||||
return this.captureOptions !== undefined;
|
||||
}
|
||||
|
||||
setLocalPreview(localPreview: RefObject<HTMLVideoElement> | undefined): void {
|
||||
const oldLocalPreview = this.localPreview?.current;
|
||||
if (oldLocalPreview) {
|
||||
oldLocalPreview.srcObject = null;
|
||||
}
|
||||
|
||||
this.localPreview = localPreview;
|
||||
|
||||
this.updateLocalPreviewSourceObject();
|
||||
|
||||
// This is a dumb hack around the fact that sometimes the
|
||||
// this.localPreview.current is updated without a call
|
||||
// to setLocalPreview, in which case the local preview
|
||||
// won't be rendered.
|
||||
if (this.updateLocalPreviewIntervalId !== undefined) {
|
||||
clearInterval(this.updateLocalPreviewIntervalId);
|
||||
}
|
||||
this.updateLocalPreviewIntervalId = setInterval(
|
||||
this.updateLocalPreviewSourceObject.bind(this),
|
||||
1000
|
||||
);
|
||||
}
|
||||
|
||||
async enableCapture(options?: GumVideoCaptureOptions): Promise<void> {
|
||||
return this.startCapturing(options ?? this.defaultCaptureOptions);
|
||||
}
|
||||
|
||||
async enableCaptureAndSend(
|
||||
sender?: VideoFrameSender,
|
||||
options?: GumVideoCaptureOptions
|
||||
): Promise<void> {
|
||||
const startCapturingPromise = this.startCapturing(
|
||||
options ?? this.defaultCaptureOptions
|
||||
);
|
||||
if (sender) {
|
||||
this.startSending(sender);
|
||||
}
|
||||
// Bubble up the error.
|
||||
return startCapturingPromise;
|
||||
}
|
||||
|
||||
disable(): void {
|
||||
this.stopCapturing();
|
||||
this.stopSending();
|
||||
|
||||
if (this.updateLocalPreviewIntervalId !== undefined) {
|
||||
clearInterval(this.updateLocalPreviewIntervalId);
|
||||
}
|
||||
this.updateLocalPreviewIntervalId = undefined;
|
||||
}
|
||||
|
||||
async setPreferredDevice(deviceId: string): Promise<void> {
|
||||
this.preferredDeviceId = deviceId;
|
||||
|
||||
if (this.captureOptions) {
|
||||
const { captureOptions, sender } = this;
|
||||
|
||||
this.disable();
|
||||
// Bubble up the error if starting video failed.
|
||||
return this.enableCaptureAndSend(sender, captureOptions);
|
||||
}
|
||||
}
|
||||
|
||||
async enumerateDevices(): Promise<Array<MediaDeviceInfo>> {
|
||||
const devices = await window.navigator.mediaDevices.enumerateDevices();
|
||||
const cameras = devices.filter(d => d.kind === 'videoinput');
|
||||
return cameras;
|
||||
}
|
||||
|
||||
private async getUserMedia(
|
||||
options: GumVideoCaptureOptions
|
||||
): Promise<MediaStream> {
|
||||
// Return provided media stream
|
||||
if (options.mediaStream) {
|
||||
return options.mediaStream;
|
||||
}
|
||||
|
||||
const constraints: GumConstraints = {
|
||||
audio: false,
|
||||
video: {
|
||||
deviceId: options.preferredDeviceId ?? this.preferredDeviceId,
|
||||
width: {
|
||||
max: options.maxWidth,
|
||||
ideal: options.maxWidth,
|
||||
},
|
||||
height: {
|
||||
max: options.maxHeight,
|
||||
ideal: options.maxHeight,
|
||||
},
|
||||
frameRate: {
|
||||
max: options.maxFramerate,
|
||||
ideal: options.maxFramerate,
|
||||
},
|
||||
},
|
||||
};
|
||||
if (options.screenShareSourceId !== undefined) {
|
||||
constraints.video = {
|
||||
mandatory: {
|
||||
chromeMediaSource: 'desktop',
|
||||
chromeMediaSourceId: options.screenShareSourceId,
|
||||
maxWidth: options.maxWidth,
|
||||
maxHeight: options.maxHeight,
|
||||
minFrameRate: 1,
|
||||
maxFrameRate: options.maxFramerate,
|
||||
},
|
||||
};
|
||||
}
|
||||
return window.navigator.mediaDevices.getUserMedia(constraints);
|
||||
}
|
||||
|
||||
private async startCapturing(options: GumVideoCaptureOptions): Promise<void> {
|
||||
if (this.capturing()) {
|
||||
log.warn('startCapturing(): already capturing');
|
||||
return;
|
||||
}
|
||||
log.info(
|
||||
`startCapturing(): ${options.maxWidth}x${options.maxHeight}@${options.maxFramerate}`
|
||||
);
|
||||
this.captureOptions = options;
|
||||
try {
|
||||
// If we start/stop/start, we may have concurrent calls to getUserMedia,
|
||||
// which is what we want if we're switching from camera to screenshare.
|
||||
// But we need to make sure we deal with the fact that things might be
|
||||
// different after the await here.
|
||||
const mediaStream = await this.getUserMedia(options);
|
||||
// It's possible video was disabled, switched to screenshare, or
|
||||
// switched to a different camera while awaiting a response, in
|
||||
// which case we need to disable the camera we just accessed.
|
||||
if (this.captureOptions !== options) {
|
||||
log.warn('startCapturing(): different state after getUserMedia()');
|
||||
for (const track of mediaStream.getVideoTracks()) {
|
||||
// Make the light turn off faster
|
||||
track.stop();
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (
|
||||
this.mediaStream !== undefined &&
|
||||
this.mediaStream.getVideoTracks().length > 0
|
||||
) {
|
||||
// We have a stream and track for the requested camera already. Stop
|
||||
// the duplicate track that we just started.
|
||||
log.warn('startCapturing(): dropping duplicate call to startCapturing');
|
||||
for (const track of mediaStream.getVideoTracks()) {
|
||||
track.stop();
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
this.mediaStream = mediaStream;
|
||||
if (
|
||||
!this.spawnedSenderRunning &&
|
||||
this.mediaStream !== undefined &&
|
||||
this.sender !== undefined
|
||||
) {
|
||||
this.spawnSender(this.mediaStream, this.sender);
|
||||
}
|
||||
|
||||
this.updateLocalPreviewSourceObject();
|
||||
} catch (e) {
|
||||
log.error(`startCapturing(): ${e}`);
|
||||
|
||||
// It's possible video was disabled, switched to screenshare, or
|
||||
// switched to a different camera while awaiting a response, in
|
||||
// which case we should reset the captureOptions if we set them.
|
||||
if (this.captureOptions === options) {
|
||||
// We couldn't open the camera. Oh well.
|
||||
this.captureOptions = undefined;
|
||||
}
|
||||
// Re-raise so that callers can surface this condition to the user.
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
private stopCapturing(): void {
|
||||
if (!this.capturing()) {
|
||||
log.warn('stopCapturing(): not capturing');
|
||||
return;
|
||||
}
|
||||
log.info('stopCapturing()');
|
||||
this.captureOptions = undefined;
|
||||
if (this.mediaStream) {
|
||||
for (const track of this.mediaStream.getVideoTracks()) {
|
||||
// Make the light turn off faster
|
||||
track.stop();
|
||||
}
|
||||
this.mediaStream = undefined;
|
||||
}
|
||||
|
||||
this.updateLocalPreviewSourceObject();
|
||||
}
|
||||
|
||||
private startSending(sender: VideoFrameSender): void {
|
||||
if (this.sender === sender) {
|
||||
return;
|
||||
}
|
||||
if (this.sender) {
|
||||
// If we're replacing an existing sender, make sure we stop the
|
||||
// current setInterval loop before starting another one.
|
||||
this.stopSending();
|
||||
}
|
||||
this.sender = sender;
|
||||
|
||||
if (!this.spawnedSenderRunning && this.mediaStream !== undefined) {
|
||||
this.spawnSender(this.mediaStream, this.sender);
|
||||
}
|
||||
}
|
||||
|
||||
private spawnSender(mediaStream: MediaStream, sender: VideoFrameSender) {
|
||||
const track = mediaStream.getVideoTracks()[0];
|
||||
if (track === undefined || this.spawnedSenderRunning) {
|
||||
return;
|
||||
}
|
||||
|
||||
const { onEnded } = this.captureOptions || {};
|
||||
|
||||
if (track.readyState === 'ended') {
|
||||
this.stopCapturing();
|
||||
log.error('spawnSender(): Video track ended before spawning sender');
|
||||
return;
|
||||
}
|
||||
|
||||
const reader = new MediaStreamTrackProcessor({
|
||||
track,
|
||||
}).readable.getReader();
|
||||
const buffer = Buffer.alloc(MAX_VIDEO_CAPTURE_BUFFER_SIZE);
|
||||
this.spawnedSenderRunning = true;
|
||||
// eslint-disable-next-line @typescript-eslint/no-floating-promises
|
||||
(async () => {
|
||||
try {
|
||||
while (mediaStream === this.mediaStream) {
|
||||
const { done, value: frame } = await reader.read();
|
||||
if (done) {
|
||||
break;
|
||||
}
|
||||
if (!frame) {
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
const format = videoPixelFormatToEnum(frame.format ?? 'I420');
|
||||
if (format === undefined) {
|
||||
log.warn(`Unsupported video frame format: ${frame.format}`);
|
||||
break;
|
||||
}
|
||||
|
||||
const { width, height } = frame.visibleRect || {};
|
||||
if (!width || !height) {
|
||||
continue;
|
||||
}
|
||||
|
||||
await frame.copyTo(buffer);
|
||||
if (sender !== this.sender) {
|
||||
break;
|
||||
}
|
||||
|
||||
sender.sendVideoFrame(width, height, format, buffer);
|
||||
} catch (e) {
|
||||
log.error(`sendVideoFrame(): ${e}`);
|
||||
} finally {
|
||||
// This must be called for more frames to come.
|
||||
frame.close();
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
log.error(`spawnSender(): ${e}`);
|
||||
} finally {
|
||||
reader.releaseLock();
|
||||
onEnded?.();
|
||||
}
|
||||
this.spawnedSenderRunning = false;
|
||||
})();
|
||||
}
|
||||
|
||||
private stopSending(): void {
|
||||
// The spawned sender should stop
|
||||
this.sender = undefined;
|
||||
}
|
||||
|
||||
private updateLocalPreviewSourceObject(): void {
|
||||
if (!this.localPreview) {
|
||||
return;
|
||||
}
|
||||
const localPreview = this.localPreview.current;
|
||||
if (!localPreview) {
|
||||
return;
|
||||
}
|
||||
|
||||
const { mediaStream = null } = this;
|
||||
|
||||
if (localPreview.srcObject === mediaStream) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (mediaStream && this.captureOptions) {
|
||||
localPreview.srcObject = mediaStream;
|
||||
if (localPreview.width === 0) {
|
||||
localPreview.width = this.captureOptions.maxWidth;
|
||||
}
|
||||
if (localPreview.height === 0) {
|
||||
localPreview.height = this.captureOptions.maxHeight;
|
||||
}
|
||||
} else {
|
||||
localPreview.srcObject = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export const MAX_VIDEO_CAPTURE_WIDTH = 2880;
|
||||
export const MAX_VIDEO_CAPTURE_HEIGHT = 1800;
|
||||
export const MAX_VIDEO_CAPTURE_AREA =
|
||||
MAX_VIDEO_CAPTURE_WIDTH * MAX_VIDEO_CAPTURE_HEIGHT;
|
||||
export const MAX_VIDEO_CAPTURE_BUFFER_SIZE = MAX_VIDEO_CAPTURE_AREA * 4;
|
||||
|
||||
export class CanvasVideoRenderer {
|
||||
private canvas?: RefObject<HTMLCanvasElement>;
|
||||
private buffer: Buffer;
|
||||
private imageData?: ImageData;
|
||||
private source?: VideoFrameSource;
|
||||
private rafId?: any;
|
||||
|
||||
constructor() {
|
||||
this.buffer = Buffer.alloc(MAX_VIDEO_CAPTURE_BUFFER_SIZE);
|
||||
}
|
||||
|
||||
setCanvas(canvas: RefObject<HTMLCanvasElement> | undefined): void {
|
||||
this.canvas = canvas;
|
||||
}
|
||||
|
||||
enable(source: VideoFrameSource): void {
|
||||
if (this.source === source) {
|
||||
return;
|
||||
}
|
||||
if (this.source) {
|
||||
// If we're replacing an existing source, make sure we stop the
|
||||
// current rAF loop before starting another one.
|
||||
if (this.rafId) {
|
||||
window.cancelAnimationFrame(this.rafId);
|
||||
}
|
||||
}
|
||||
this.source = source;
|
||||
this.requestAnimationFrameCallback();
|
||||
}
|
||||
|
||||
disable(): void {
|
||||
this.renderBlack();
|
||||
this.source = undefined;
|
||||
if (this.rafId) {
|
||||
window.cancelAnimationFrame(this.rafId);
|
||||
}
|
||||
}
|
||||
|
||||
private requestAnimationFrameCallback() {
|
||||
this.renderVideoFrame();
|
||||
this.rafId = window.requestAnimationFrame(
|
||||
this.requestAnimationFrameCallback.bind(this)
|
||||
);
|
||||
}
|
||||
|
||||
private renderBlack() {
|
||||
if (!this.canvas) {
|
||||
return;
|
||||
}
|
||||
const canvas = this.canvas.current;
|
||||
if (!canvas) {
|
||||
return;
|
||||
}
|
||||
const context = canvas.getContext('2d');
|
||||
if (!context) {
|
||||
return;
|
||||
}
|
||||
context.fillStyle = 'black';
|
||||
context.fillRect(0, 0, canvas.width, canvas.height);
|
||||
}
|
||||
|
||||
private renderVideoFrame() {
|
||||
if (!this.source || !this.canvas) {
|
||||
return;
|
||||
}
|
||||
const canvas = this.canvas.current;
|
||||
if (!canvas) {
|
||||
return;
|
||||
}
|
||||
const context = canvas.getContext('2d');
|
||||
if (!context) {
|
||||
return;
|
||||
}
|
||||
|
||||
const frame = this.source.receiveVideoFrame(
|
||||
this.buffer,
|
||||
MAX_VIDEO_CAPTURE_WIDTH,
|
||||
MAX_VIDEO_CAPTURE_HEIGHT
|
||||
);
|
||||
if (!frame) {
|
||||
return;
|
||||
}
|
||||
const [width, height] = frame;
|
||||
|
||||
if (
|
||||
canvas.clientWidth <= 0 ||
|
||||
width <= 0 ||
|
||||
canvas.clientHeight <= 0 ||
|
||||
height <= 0
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
const frameAspectRatio = width / height;
|
||||
const canvasAspectRatio = canvas.clientWidth / canvas.clientHeight;
|
||||
|
||||
let dx = 0;
|
||||
let dy = 0;
|
||||
if (frameAspectRatio > canvasAspectRatio) {
|
||||
// Frame wider than view: We need bars at the top and bottom
|
||||
canvas.width = width;
|
||||
canvas.height = width / canvasAspectRatio;
|
||||
dy = (canvas.height - height) / 2;
|
||||
} else if (frameAspectRatio < canvasAspectRatio) {
|
||||
// Frame narrower than view: We need pillars on the sides
|
||||
canvas.width = height * canvasAspectRatio;
|
||||
canvas.height = height;
|
||||
dx = (canvas.width - width) / 2;
|
||||
} else {
|
||||
// Will stretch perfectly with no bars
|
||||
canvas.width = width;
|
||||
canvas.height = height;
|
||||
}
|
||||
|
||||
if (dx > 0 || dy > 0) {
|
||||
context.fillStyle = 'black';
|
||||
context.fillRect(0, 0, canvas.width, canvas.height);
|
||||
}
|
||||
|
||||
if (this.imageData?.width !== width || this.imageData?.height !== height) {
|
||||
this.imageData = new ImageData(width, height);
|
||||
}
|
||||
this.imageData.data.set(this.buffer.subarray(0, width * height * 4));
|
||||
context.putImageData(this.imageData, dx, dy);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user