Microsoft/vscode
1
0
Fork 0
mirror of https://github.com/microsoft/vscode.git synced 2026-09-13 05:01:08 +01:00
Code Issues Packages Projects Releases Wiki Activity

voice - add a basic sliding window transcription approach based on silence detection

Browse Source
This commit is contained in:
Benjamin Pasero
2023-09-15 12:40:07 +02:00
parent e8ea1ab584
commit f0a61fd8f6
2 changed files with 95 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/vs/code/node/sharedProcess/contrib/voiceTranscriber.ts
+93 -32
View File
@@ -9,10 +9,12 @@ import { Disposable, toDisposable } from 'vs/base/common/lifecycle';
import { IVoiceRecognitionService } from 'vs/platform/voiceRecognition/node/voiceRecognitionService';
import { ILogService } from 'vs/platform/log/common/log';
import { CancellationToken, CancellationTokenSource } from 'vs/base/common/cancellation';
import { LimitedQueue } from 'vs/base/common/async';
import { LimitedQueue, Queue } from 'vs/base/common/async';
export class VoiceTranscriptionManager extends Disposable {
private static USE_SLIDING_WINDOW = !!process.env.VSCODE_VOICE_USE_SLIDING_WINDOW;
constructor(
private readonly onDidWindowConnectRaw: Event<MessagePortMain>,
@IVoiceRecognitionService private readonly voiceRecognitionService: IVoiceRecognitionService,
@@ -25,26 +27,25 @@ export class VoiceTranscriptionManager extends Disposable {
private registerListeners(): void {
this._register(this.onDidWindowConnectRaw(port => {
this._register(new VoiceTranscriber(port, this.voiceRecognitionService, this.logService));
this.logService.info(`[voice] transcriber: new connection (sliding window: ${VoiceTranscriptionManager.USE_SLIDING_WINDOW})`);
if (VoiceTranscriptionManager.USE_SLIDING_WINDOW) {
this._register(new SlidingWindowVoiceTranscriber(port, this.voiceRecognitionService, this.logService));
} else {
this._register(new FullWindowVoiceTranscriber(port, this.voiceRecognitionService, this.logService));
}
}));
}
}
class VoiceTranscriber extends Disposable {
abstract class VoiceTranscriber extends Disposable {
private static MAX_DATA_LENGTH = 30 /* seconds */ * 16000 /* sampling rate */ * 16 /* bith depth */ * 1 /* channels */ / 8;
private readonly transcriptionQueue = new LimitedQueue();
private data: Float32Array | undefined = undefined;
private transcribedDataLength = 0;
private transcribedResult = '';
protected static MAX_DATA_LENGTH = 30 /* seconds */ * 16000 /* sampling rate */ * 16 /* bith depth */ * 1 /* channels */ / 8;
constructor(
private readonly port: MessagePortMain,
private readonly voiceRecognitionService: IVoiceRecognitionService,
private readonly logService: ILogService
protected readonly port: MessagePortMain,
protected readonly voiceRecognitionService: IVoiceRecognitionService,
protected readonly logService: ILogService
) {
super();
@@ -52,12 +53,16 @@ class VoiceTranscriber extends Disposable {
}
private registerListeners(): void {
this.logService.info(`[voice] transcriber: new connection`);
const cts = new CancellationTokenSource();
this._register(toDisposable(() => cts.dispose(true)));
const requestHandler = (e: MessageEvent) => this.handleRequest(e, cts.token);
const requestHandler = (e: MessageEvent) => {
if (!(e.data instanceof Float32Array)) {
return;
}
this.handleRequest(e.data, cts.token);
};
this.port.on('message', requestHandler);
this._register(toDisposable(() => this.port.off('message', requestHandler)));
@@ -71,12 +76,79 @@ class VoiceTranscriber extends Disposable {
});
}
private async handleRequest(e: MessageEvent, cancellation: CancellationToken): Promise<void> {
if (!(e.data instanceof Float32Array)) {
protected abstract handleRequest(data: Float32Array, cancellation: CancellationToken): Promise<void>;
protected joinFloat32Arrays(float32Arrays: Float32Array[]): Float32Array {
const result = new Float32Array(float32Arrays.reduce((prev, curr) => prev + curr.length, 0));
let offset = 0;
for (const float32Array of float32Arrays) {
result.set(float32Array, offset);
offset += float32Array.length;
}
return result;
}
}
class SlidingWindowVoiceTranscriber extends VoiceTranscriber {
private readonly transcriptionQueue = new Queue();
private transcribedResults: string[] = [];
private data: Float32Array = new Float32Array(0);
protected async handleRequest(data: Float32Array, cancellation: CancellationToken): Promise<void> {
if (data.length > 0) {
this.logService.info(`[voice] transcriber: voice detected, storing in buffer`);
this.data = this.data ? this.joinFloat32Arrays([this.data, data]) : data;
} else if (this.data) {
this.logService.info(`[voice] transcriber: silence detected, transcribing window...`);
const data = this.data.slice(0);
this.data = new Float32Array(0);
this.transcriptionQueue.queue(() => this.transcribe(data, cancellation));
}
}
private async transcribe(data: Float32Array, cancellation: CancellationToken): Promise<void> {
if (cancellation.isCancellationRequested) {
return;
}
const dataCandidate = this.data ? this.joinFloat32Arrays([this.data, e.data]) : e.data;
if (data.length > VoiceTranscriber.MAX_DATA_LENGTH) {
this.logService.warn(`[voice] transcriber: refusing to accept more than 30s of audio data`);
return;
}
if (data.length !== 0) {
const result = await this.voiceRecognitionService.transcribe(data, cancellation);
if (result) {
this.transcribedResults.push(result);
}
}
if (cancellation.isCancellationRequested) {
return;
}
this.port.postMessage(this.transcribedResults.join(' '));
}
}
class FullWindowVoiceTranscriber extends VoiceTranscriber {
private readonly transcriptionQueue = new LimitedQueue();
private data: Float32Array | undefined = undefined;
private transcribedDataLength = 0;
private transcribedResult = '';
protected async handleRequest(data: Float32Array, cancellation: CancellationToken): Promise<void> {
const dataCandidate = this.data ? this.joinFloat32Arrays([this.data, data]) : data;
if (dataCandidate.length > VoiceTranscriber.MAX_DATA_LENGTH) {
this.logService.warn(`[voice] transcriber: refusing to accept more than 30s of audio data`);
return;
@@ -105,6 +177,7 @@ class VoiceTranscriber extends Disposable {
this.logService.info(`[voice] transcriber: silence detected, reusing previous transcription result`);
result = this.transcribedResult;
} else {
this.logService.info(`[voice] transcriber: voice detected, transcribing everything...`);
result = await this.voiceRecognitionService.transcribe(data, cancellation);
}
@@ -117,16 +190,4 @@ class VoiceTranscriber extends Disposable {
this.port.postMessage(result);
}
private joinFloat32Arrays(float32Arrays: Float32Array[]): Float32Array {
const result = new Float32Array(float32Arrays.reduce((prev, curr) => prev + curr.length, 0));
let offset = 0;
for (const float32Array of float32Arrays) {
result.set(float32Array, offset);
offset += float32Array.length;
}
return result;
}
}