diff --git a/src/vs/code/node/sharedProcess/contrib/voiceTranscriber.ts b/src/vs/code/node/sharedProcess/contrib/voiceTranscriber.ts index c9cd0b411eb..23a790e6109 100644 --- a/src/vs/code/node/sharedProcess/contrib/voiceTranscriber.ts +++ b/src/vs/code/node/sharedProcess/contrib/voiceTranscriber.ts @@ -9,10 +9,12 @@ import { Disposable, toDisposable } from 'vs/base/common/lifecycle'; import { IVoiceRecognitionService } from 'vs/platform/voiceRecognition/node/voiceRecognitionService'; import { ILogService } from 'vs/platform/log/common/log'; import { CancellationToken, CancellationTokenSource } from 'vs/base/common/cancellation'; -import { LimitedQueue } from 'vs/base/common/async'; +import { LimitedQueue, Queue } from 'vs/base/common/async'; export class VoiceTranscriptionManager extends Disposable { + private static USE_SLIDING_WINDOW = !!process.env.VSCODE_VOICE_USE_SLIDING_WINDOW; + constructor( private readonly onDidWindowConnectRaw: Event, @IVoiceRecognitionService private readonly voiceRecognitionService: IVoiceRecognitionService, @@ -25,26 +27,25 @@ export class VoiceTranscriptionManager extends Disposable { private registerListeners(): void { this._register(this.onDidWindowConnectRaw(port => { - this._register(new VoiceTranscriber(port, this.voiceRecognitionService, this.logService)); + this.logService.info(`[voice] transcriber: new connection (sliding window: ${VoiceTranscriptionManager.USE_SLIDING_WINDOW})`); + + if (VoiceTranscriptionManager.USE_SLIDING_WINDOW) { + this._register(new SlidingWindowVoiceTranscriber(port, this.voiceRecognitionService, this.logService)); + } else { + this._register(new FullWindowVoiceTranscriber(port, this.voiceRecognitionService, this.logService)); + } })); } } -class VoiceTranscriber extends Disposable { +abstract class VoiceTranscriber extends Disposable { - private static MAX_DATA_LENGTH = 30 /* seconds */ * 16000 /* sampling rate */ * 16 /* bith depth */ * 1 /* channels */ / 8; - - private readonly transcriptionQueue = new LimitedQueue(); - - private data: Float32Array | undefined = undefined; - - private transcribedDataLength = 0; - private transcribedResult = ''; + protected static MAX_DATA_LENGTH = 30 /* seconds */ * 16000 /* sampling rate */ * 16 /* bith depth */ * 1 /* channels */ / 8; constructor( - private readonly port: MessagePortMain, - private readonly voiceRecognitionService: IVoiceRecognitionService, - private readonly logService: ILogService + protected readonly port: MessagePortMain, + protected readonly voiceRecognitionService: IVoiceRecognitionService, + protected readonly logService: ILogService ) { super(); @@ -52,12 +53,16 @@ class VoiceTranscriber extends Disposable { } private registerListeners(): void { - this.logService.info(`[voice] transcriber: new connection`); - const cts = new CancellationTokenSource(); this._register(toDisposable(() => cts.dispose(true))); - const requestHandler = (e: MessageEvent) => this.handleRequest(e, cts.token); + const requestHandler = (e: MessageEvent) => { + if (!(e.data instanceof Float32Array)) { + return; + } + + this.handleRequest(e.data, cts.token); + }; this.port.on('message', requestHandler); this._register(toDisposable(() => this.port.off('message', requestHandler))); @@ -71,12 +76,79 @@ class VoiceTranscriber extends Disposable { }); } - private async handleRequest(e: MessageEvent, cancellation: CancellationToken): Promise { - if (!(e.data instanceof Float32Array)) { + protected abstract handleRequest(data: Float32Array, cancellation: CancellationToken): Promise; + + protected joinFloat32Arrays(float32Arrays: Float32Array[]): Float32Array { + const result = new Float32Array(float32Arrays.reduce((prev, curr) => prev + curr.length, 0)); + + let offset = 0; + for (const float32Array of float32Arrays) { + result.set(float32Array, offset); + offset += float32Array.length; + } + + return result; + } +} + +class SlidingWindowVoiceTranscriber extends VoiceTranscriber { + + private readonly transcriptionQueue = new Queue(); + + private transcribedResults: string[] = []; + private data: Float32Array = new Float32Array(0); + + protected async handleRequest(data: Float32Array, cancellation: CancellationToken): Promise { + if (data.length > 0) { + this.logService.info(`[voice] transcriber: voice detected, storing in buffer`); + + this.data = this.data ? this.joinFloat32Arrays([this.data, data]) : data; + } else if (this.data) { + this.logService.info(`[voice] transcriber: silence detected, transcribing window...`); + + const data = this.data.slice(0); + this.data = new Float32Array(0); + + this.transcriptionQueue.queue(() => this.transcribe(data, cancellation)); + } + } + + private async transcribe(data: Float32Array, cancellation: CancellationToken): Promise { + if (cancellation.isCancellationRequested) { return; } - const dataCandidate = this.data ? this.joinFloat32Arrays([this.data, e.data]) : e.data; + if (data.length > VoiceTranscriber.MAX_DATA_LENGTH) { + this.logService.warn(`[voice] transcriber: refusing to accept more than 30s of audio data`); + return; + } + + if (data.length !== 0) { + const result = await this.voiceRecognitionService.transcribe(data, cancellation); + if (result) { + this.transcribedResults.push(result); + } + } + + if (cancellation.isCancellationRequested) { + return; + } + + this.port.postMessage(this.transcribedResults.join(' ')); + } +} + +class FullWindowVoiceTranscriber extends VoiceTranscriber { + + private readonly transcriptionQueue = new LimitedQueue(); + + private data: Float32Array | undefined = undefined; + + private transcribedDataLength = 0; + private transcribedResult = ''; + + protected async handleRequest(data: Float32Array, cancellation: CancellationToken): Promise { + const dataCandidate = this.data ? this.joinFloat32Arrays([this.data, data]) : data; if (dataCandidate.length > VoiceTranscriber.MAX_DATA_LENGTH) { this.logService.warn(`[voice] transcriber: refusing to accept more than 30s of audio data`); return; @@ -105,6 +177,7 @@ class VoiceTranscriber extends Disposable { this.logService.info(`[voice] transcriber: silence detected, reusing previous transcription result`); result = this.transcribedResult; } else { + this.logService.info(`[voice] transcriber: voice detected, transcribing everything...`); result = await this.voiceRecognitionService.transcribe(data, cancellation); } @@ -117,16 +190,4 @@ class VoiceTranscriber extends Disposable { this.port.postMessage(result); } - - private joinFloat32Arrays(float32Arrays: Float32Array[]): Float32Array { - const result = new Float32Array(float32Arrays.reduce((prev, curr) => prev + curr.length, 0)); - - let offset = 0; - for (const float32Array of float32Arrays) { - result.set(float32Array, offset); - offset += float32Array.length; - } - - return result; - } } diff --git a/src/vs/platform/voiceRecognition/node/voiceRecognitionService.ts b/src/vs/platform/voiceRecognition/node/voiceRecognitionService.ts index 9213c0798b5..03f7a16ca95 100644 --- a/src/vs/platform/voiceRecognition/node/voiceRecognitionService.ts +++ b/src/vs/platform/voiceRecognition/node/voiceRecognitionService.ts @@ -37,8 +37,6 @@ export class VoiceRecognitionService implements IVoiceRecognitionService { ) { } async transcribe(channelData: Float32Array, cancellation: CancellationToken): Promise { - this.logService.info(`[voice] transcribe(${channelData.length}): Begin`); - const modulePath = process.env.VSCODE_VOICE_MODULE_PATH; // TODO@bpasero package if (!modulePath || this.productService.quality === 'stable') { this.logService.error(`[voice] transcribe(${channelData.length}): Voice recognition not yet supported`); @@ -47,8 +45,6 @@ export class VoiceRecognitionService implements IVoiceRecognitionService { const now = Date.now(); - this.logService.info(`[voice] transcribe(${channelData.length}): Getting module from ${modulePath}`); - try { const voiceModule: { transcribe: ( @@ -73,11 +69,11 @@ export class VoiceRecognitionService implements IVoiceRecognitionService { signal: abortController.signal }); - this.logService.info(`[voice] transcribe(${channelData.length}): End (text: "${text}", took: ${Date.now() - now}ms)`); + this.logService.info(`[voice] transcribe(${channelData.length}): Text "${text}", took ${Date.now() - now}ms)`); return text; } catch (error) { - this.logService.error(`[voice] transcribe(${channelData.length}): Failed (error: "${error}", took: ${Date.now() - now}ms)`); + this.logService.error(`[voice] transcribe(${channelData.length}): Failed width "${error}", took ${Date.now() - now}ms)`); throw error; }