From ea27cc09a8ce40585261dcd7622cdd5a838c2bdc Mon Sep 17 00:00:00 2001
From: Benjamin Pasero <benjamin.pasero@microsoft.com>
Date: Fri, 15 Sep 2023 10:05:34 +0200
Subject: [PATCH] voice - implement super simple vad

---
 .../sharedProcess/contrib/voiceTranscriber.ts | 11 +----
 .../voiceTranscriptionWorklet.ts              | 40 +++++++++++++++++--
 .../workbenchVoiceRecognitionService.ts       |  5 ++-
 3 files changed, 43 insertions(+), 13 deletions(-)

diff --git a/src/vs/code/node/sharedProcess/contrib/voiceTranscriber.ts b/src/vs/code/node/sharedProcess/contrib/voiceTranscriber.ts
index 11299d0848e..bc5657a7136 100644
--- a/src/vs/code/node/sharedProcess/contrib/voiceTranscriber.ts
+++ b/src/vs/code/node/sharedProcess/contrib/voiceTranscriber.ts
@@ -69,18 +69,11 @@ class VoiceTranscriber extends Disposable {
 	}
 
 	private async handleRequest(e: MessageEvent, cancellation: CancellationToken): Promise<void> {
-		if (!(Array.isArray(e.data))) {
+		if (!(e.data instanceof Float32Array)) {
 			return;
 		}
 
-		const newData: Float32Array[] = [];
-		for (const channelData of e.data) {
-			if (channelData instanceof Float32Array) {
-				newData.push(channelData);
-			}
-		}
-
-		const dataCandidate = this.joinFloat32Arrays(this.data ? [this.data, ...newData] : newData);
+		const dataCandidate = this.data ? this.joinFloat32Arrays([this.data, e.data]) : e.data;
 
 		if (dataCandidate.length > VoiceTranscriber.MAX_DATA_LENGTH) {
 			this.logService.warn(`[voice] transcriber: refusing to accept more than 30s of audio data`);
diff --git a/src/vs/workbench/services/voiceRecognition/electron-sandbox/voiceTranscriptionWorklet.ts b/src/vs/workbench/services/voiceRecognition/electron-sandbox/voiceTranscriptionWorklet.ts
index fc3acd9eb6a..bd125dd0262 100644
--- a/src/vs/workbench/services/voiceRecognition/electron-sandbox/voiceTranscriptionWorklet.ts
+++ b/src/vs/workbench/services/voiceRecognition/electron-sandbox/voiceTranscriptionWorklet.ts
@@ -13,6 +13,7 @@ declare class AudioWorkletProcessor {
 interface IVoiceTranscriptionWorkletOptions extends AudioWorkletNodeOptions {
 	processorOptions: {
 		readonly bufferTimespan: number;
+		readonly vadThreshold: number;
 	};
 }
 
@@ -22,6 +23,7 @@ class VoiceTranscriptionWorklet extends AudioWorkletProcessor {
 	private stopped: boolean = false;
 
 	private buffer: Float32Array[] = [];
+	private text = '';
 
 	private sharedProcessConnection: MessagePort | undefined = undefined;
 
@@ -43,7 +45,7 @@ class VoiceTranscriptionWorklet extends AudioWorkletProcessor {
 						}
 
 						if (typeof event.data === 'string') {
-							this.port.postMessage(event.data);
+							this.processText(event.data);
 						}
 					};
 
@@ -63,6 +65,11 @@ class VoiceTranscriptionWorklet extends AudioWorkletProcessor {
 		};
 	}
 
+	private processText(text: string = this.text): void {
+		this.text = text;
+		this.port.postMessage(this.text);
+	}
+
 	override process(inputs: [Float32Array[]]): boolean {
 		if (this.startTime === undefined) {
 			this.startTime = Date.now();
@@ -76,16 +83,43 @@ class VoiceTranscriptionWorklet extends AudioWorkletProcessor {
 		this.buffer.push(inputChannelData.slice(0));
 
 		if (Date.now() - this.startTime > this.options.processorOptions.bufferTimespan && this.sharedProcessConnection) {
-			const buffer = this.buffer;
+			const buffer = this.joinFloat32Arrays(this.buffer);
 			this.buffer = [];
 
-			this.sharedProcessConnection.postMessage(buffer);
+			if (!this.appearsToBeSilence(buffer)) {
+				this.sharedProcessConnection.postMessage(buffer);
+			} else {
+				this.processText();
+			}
 
 			this.startTime = Date.now();
 		}
 
 		return !this.stopped;
 	}
+
+	private appearsToBeSilence(data: Float32Array): boolean {
+		let sum = 0;
+		for (let i = 0; i < data.length; i++) {
+			sum += data[i] * data[i];
+		}
+
+		const rms = Math.sqrt(sum / data.length);
+
+		return rms < this.options.processorOptions.vadThreshold;
+	}
+
+	private joinFloat32Arrays(float32Arrays: Float32Array[]): Float32Array {
+		const result = new Float32Array(float32Arrays.reduce((prev, curr) => prev + curr.length, 0));
+
+		let offset = 0;
+		for (const float32Array of float32Arrays) {
+			result.set(float32Array, offset);
+			offset += float32Array.length;
+		}
+
+		return result;
+	}
 }
 
 // @ts-ignore
diff --git a/src/vs/workbench/services/voiceRecognition/electron-sandbox/workbenchVoiceRecognitionService.ts b/src/vs/workbench/services/voiceRecognition/electron-sandbox/workbenchVoiceRecognitionService.ts
index 11aa0dae3e1..0f279ac26ea 100644
--- a/src/vs/workbench/services/voiceRecognition/electron-sandbox/workbenchVoiceRecognitionService.ts
+++ b/src/vs/workbench/services/voiceRecognition/electron-sandbox/workbenchVoiceRecognitionService.ts
@@ -41,6 +41,7 @@ export interface IWorkbenchVoiceRecognitionService {
 interface IVoiceTranscriptionWorkletOptions extends AudioWorkletNodeOptions {
 	processorOptions: {
 		readonly bufferTimespan: number;
+		readonly vadThreshold: number;
 	};
 }
 
@@ -93,6 +94,7 @@ export class WorkbenchVoiceRecognitionService implements IWorkbenchVoiceRecognit
 	private static readonly AUDIO_CHANNELS = 1;
 
 	private static readonly BUFFER_TIMESPAN = 1000;
+	private static readonly VAD_THRESHOLD = 0.02;
 
 	constructor(
 		@IProgressService private readonly progressService: IProgressService,
@@ -172,7 +174,8 @@ export class WorkbenchVoiceRecognitionService implements IWorkbenchVoiceRecognit
 					channelCount: WorkbenchVoiceRecognitionService.AUDIO_CHANNELS,
 					channelCountMode: 'explicit',
 					processorOptions: {
-						bufferTimespan: WorkbenchVoiceRecognitionService.BUFFER_TIMESPAN
+						bufferTimespan: WorkbenchVoiceRecognitionService.BUFFER_TIMESPAN,
+						vadThreshold: WorkbenchVoiceRecognitionService.VAD_THRESHOLD
 					}
 				}, onDidTranscribe, this.sharedProcessService);
 				await voiceTranscriptionTarget.start(cts.token);