From 17c535565298adcabfdee866d7914955e8c63d2b Mon Sep 17 00:00:00 2001
From: Benjamin Pasero <benjamin.pasero@microsoft.com>
Date: Fri, 15 Sep 2023 11:14:35 +0200
Subject: [PATCH] voice - still handle silence on the transcriber

---
 .../sharedProcess/contrib/voiceTranscriber.ts | 18 +++++++++++--
 .../voiceTranscriptionWorklet.ts              | 25 ++++++++++---------
 2 files changed, 29 insertions(+), 14 deletions(-)

diff --git a/src/vs/code/node/sharedProcess/contrib/voiceTranscriber.ts b/src/vs/code/node/sharedProcess/contrib/voiceTranscriber.ts
index bc5657a7136..c9cd0b411eb 100644
--- a/src/vs/code/node/sharedProcess/contrib/voiceTranscriber.ts
+++ b/src/vs/code/node/sharedProcess/contrib/voiceTranscriber.ts
@@ -38,6 +38,9 @@ class VoiceTranscriber extends Disposable {
 
 	private data: Float32Array | undefined = undefined;
 
+	private transcribedDataLength = 0;
+	private transcribedResult = '';
+
 	constructor(
 		private readonly port: MessagePortMain,
 		private readonly voiceRecognitionService: IVoiceRecognitionService,
@@ -74,7 +77,6 @@ class VoiceTranscriber extends Disposable {
 		}
 
 		const dataCandidate = this.data ? this.joinFloat32Arrays([this.data, e.data]) : e.data;
-
 		if (dataCandidate.length > VoiceTranscriber.MAX_DATA_LENGTH) {
 			this.logService.warn(`[voice] transcriber: refusing to accept more than 30s of audio data`);
 			return;
@@ -95,7 +97,19 @@ class VoiceTranscriber extends Disposable {
 			return;
 		}
 
-		const result = await this.voiceRecognitionService.transcribe(data, cancellation);
+		let result: string;
+		if (data.length === this.transcribedDataLength) {
+			// Optimization: if the data is the same as the last time
+			// we transcribed, don't transcribe again, just return the
+			// same result as we had last time.
+			this.logService.info(`[voice] transcriber: silence detected, reusing previous transcription result`);
+			result = this.transcribedResult;
+		} else {
+			result = await this.voiceRecognitionService.transcribe(data, cancellation);
+		}
+
+		this.transcribedResult = result;
+		this.transcribedDataLength = data.length;
 
 		if (cancellation.isCancellationRequested) {
 			return;
diff --git a/src/vs/workbench/services/voiceRecognition/electron-sandbox/voiceTranscriptionWorklet.ts b/src/vs/workbench/services/voiceRecognition/electron-sandbox/voiceTranscriptionWorklet.ts
index bd125dd0262..cb0083c81eb 100644
--- a/src/vs/workbench/services/voiceRecognition/electron-sandbox/voiceTranscriptionWorklet.ts
+++ b/src/vs/workbench/services/voiceRecognition/electron-sandbox/voiceTranscriptionWorklet.ts
@@ -23,7 +23,6 @@ class VoiceTranscriptionWorklet extends AudioWorkletProcessor {
 	private stopped: boolean = false;
 
 	private buffer: Float32Array[] = [];
-	private text = '';
 
 	private sharedProcessConnection: MessagePort | undefined = undefined;
 
@@ -45,7 +44,7 @@ class VoiceTranscriptionWorklet extends AudioWorkletProcessor {
 						}
 
 						if (typeof event.data === 'string') {
-							this.processText(event.data);
+							this.port.postMessage(event.data);
 						}
 					};
 
@@ -65,11 +64,6 @@ class VoiceTranscriptionWorklet extends AudioWorkletProcessor {
 		};
 	}
 
-	private processText(text: string = this.text): void {
-		this.text = text;
-		this.port.postMessage(this.text);
-	}
-
 	override process(inputs: [Float32Array[]]): boolean {
 		if (this.startTime === undefined) {
 			this.startTime = Date.now();
@@ -86,11 +80,12 @@ class VoiceTranscriptionWorklet extends AudioWorkletProcessor {
 			const buffer = this.joinFloat32Arrays(this.buffer);
 			this.buffer = [];
 
-			if (!this.appearsToBeSilence(buffer)) {
-				this.sharedProcessConnection.postMessage(buffer);
-			} else {
-				this.processText();
-			}
+			// Send buffer to shared process for transcription.
+			// Send an empty buffer if it appears to be silence
+			// so that we can still trigger the transcription
+			// service and let it know about this.
+
+			this.sharedProcessConnection.postMessage(this.appearsToBeSilence(buffer) ? new Float32Array(0) : buffer);
 
 			this.startTime = Date.now();
 		}
@@ -99,6 +94,12 @@ class VoiceTranscriptionWorklet extends AudioWorkletProcessor {
 	}
 
 	private appearsToBeSilence(data: Float32Array): boolean {
+
+		// This is the most simple Voice Activity Detection (VAD)
+		// and it is based on the Root Mean Square (RMS) of the signal
+		// with a certain threshold. Good for testing but probably
+		// not suitable for shipping to stable (TODO@bpasero).
+
 		let sum = 0;
 		for (let i = 0; i < data.length; i++) {
 			sum += data[i] * data[i];