From 03ffebe2fcd06034091f47d996c1ebae7329e327 Mon Sep 17 00:00:00 2001 From: Benjamin Pasero Date: Thu, 10 Aug 2023 12:43:37 +0200 Subject: [PATCH] voice - drop `VSFloat32Array` and encode `float32` into a `VSBuffer` --- src/vs/base/common/buffer.ts | 32 ------------ src/vs/base/common/marshalling.ts | 3 +- src/vs/base/test/common/buffer.test.ts | 21 +------- .../node/sharedProcess/sharedProcessMain.ts | 1 - .../common/voiceRecognitionService.ts | 17 +++---- .../node/voiceRecognitionService.ts | 40 +++++++++++---- .../bufferInputAudioProcessor.js | 50 +++++++++++++------ .../workbenchVoiceRecognitionService.ts | 18 +++---- 8 files changed, 81 insertions(+), 101 deletions(-) diff --git a/src/vs/base/common/buffer.ts b/src/vs/base/common/buffer.ts index 1981217fd1c..08736ab8c0b 100644 --- a/src/vs/base/common/buffer.ts +++ b/src/vs/base/common/buffer.ts @@ -4,7 +4,6 @@ *--------------------------------------------------------------------------------------------*/ import { Lazy } from 'vs/base/common/lazy'; -import { MarshalledId } from 'vs/base/common/marshallingIds'; import * as streams from 'vs/base/common/stream'; declare const Buffer: any; @@ -440,34 +439,3 @@ export function encodeBase64({ buffer }: VSBuffer, padded = true, urlSafe = fals return output; } - -export interface VSFloat32ArrayComponents { - readonly $mid: MarshalledId.Float32Array; - readonly values: number[]; -} - -export class VSFloat32Array { - - readonly buffer: Float32Array; - readonly byteLength: number; - - static wrap(actual: Float32Array): VSFloat32Array { - return new VSFloat32Array(actual); - } - - private constructor(buffer: Float32Array) { - this.buffer = buffer; - this.byteLength = this.buffer.byteLength; - } - - toJSON(): VSFloat32ArrayComponents { - return { - $mid: MarshalledId.Float32Array, - values: Array.from(this.buffer.map(value => value)) - }; - } - - static revive(raw: VSFloat32ArrayComponents): VSFloat32Array { - return VSFloat32Array.wrap(new Float32Array(raw.values)); - } -} diff --git a/src/vs/base/common/marshalling.ts b/src/vs/base/common/marshalling.ts index 087ad5d0f58..67a29703cdc 100644 --- a/src/vs/base/common/marshalling.ts +++ b/src/vs/base/common/marshalling.ts @@ -3,7 +3,7 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import { VSBuffer, VSFloat32Array } from 'vs/base/common/buffer'; +import { VSBuffer } from 'vs/base/common/buffer'; import { regExpFlags } from 'vs/base/common/strings'; import { URI, UriComponents } from 'vs/base/common/uri'; import { MarshalledId } from './marshallingIds'; @@ -54,7 +54,6 @@ export function revive(obj: any, depth = 0): Revived { case MarshalledId.Uri: return URI.revive(obj); case MarshalledId.Regexp: return new RegExp(obj.source, obj.flags); case MarshalledId.Date: return new Date(obj.source); - case MarshalledId.Float32Array: return VSFloat32Array.revive(obj); } if ( diff --git a/src/vs/base/test/common/buffer.test.ts b/src/vs/base/test/common/buffer.test.ts index 969a96d9303..5a37943b658 100644 --- a/src/vs/base/test/common/buffer.test.ts +++ b/src/vs/base/test/common/buffer.test.ts @@ -5,8 +5,7 @@ import * as assert from 'assert'; import { timeout } from 'vs/base/common/async'; -import { bufferedStreamToBuffer, bufferToReadable, bufferToStream, decodeBase64, encodeBase64, newWriteableBufferStream, readableToBuffer, streamToBuffer, VSBuffer, VSFloat32Array } from 'vs/base/common/buffer'; -import { parse, stringify } from 'vs/base/common/marshalling'; +import { bufferedStreamToBuffer, bufferToReadable, bufferToStream, decodeBase64, encodeBase64, newWriteableBufferStream, readableToBuffer, streamToBuffer, VSBuffer } from 'vs/base/common/buffer'; import { peekStream } from 'vs/base/common/stream'; suite('Buffer', () => { @@ -478,22 +477,4 @@ suite('Buffer', () => { assert.throws(() => decodeBase64('invalid!')); }); }); - - suite('Float32Array', () => { - - test('serialization', () => { - const array = new Float32Array(10); - for (let i = 0; i < array.length; i++) { - array[i] = i === 0 ? 0 : Math.random(); - } - - const buffer = VSFloat32Array.wrap(array); - const serialized = stringify(buffer); - const deserialized = parse(serialized); - - assert.ok(deserialized instanceof VSFloat32Array); - assert.deepStrictEqual(array, deserialized.buffer); - assert.deepStrictEqual(array.byteLength, deserialized.byteLength); - }); - }); }); diff --git a/src/vs/code/node/sharedProcess/sharedProcessMain.ts b/src/vs/code/node/sharedProcess/sharedProcessMain.ts index 7092cc2b8ec..85559f93147 100644 --- a/src/vs/code/node/sharedProcess/sharedProcessMain.ts +++ b/src/vs/code/node/sharedProcess/sharedProcessMain.ts @@ -3,7 +3,6 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -/* eslint-disable local/code-layering, local/code-import-patterns */ import { hostname, release } from 'os'; import { toErrorMessage } from 'vs/base/common/errorMessage'; import { onUnexpectedError, setUnexpectedErrorHandler } from 'vs/base/common/errors'; diff --git a/src/vs/platform/voiceRecognition/common/voiceRecognitionService.ts b/src/vs/platform/voiceRecognition/common/voiceRecognitionService.ts index 3accb99ed0a..34f7c9add07 100644 --- a/src/vs/platform/voiceRecognition/common/voiceRecognitionService.ts +++ b/src/vs/platform/voiceRecognition/common/voiceRecognitionService.ts @@ -3,18 +3,11 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import { VSFloat32Array } from 'vs/base/common/buffer'; +import { VSBuffer } from 'vs/base/common/buffer'; import { createDecorator } from 'vs/platform/instantiation/common/instantiation'; export const IVoiceRecognitionService = createDecorator('voiceRecognitionService'); -export interface IAudioBuffer { - readonly sampleRate: 16000; - readonly sampleSize: 16; - readonly channelCount: 1; - readonly channelData: VSFloat32Array; -} - export interface IVoiceRecognitionService { readonly _serviceBrand: undefined; @@ -24,8 +17,10 @@ export interface IVoiceRecognitionService { * transcribe the spoken words into text. * * @param buffer the audio data obtained from - * the microphone as PCM 32-bit float mono in - * 16khz. + * the microphone as uncompressed PCM data: + * - 1 channel (mono) + * - 16khz sampling rate + * - 16bit sample size */ - transcribe(buffer: IAudioBuffer): Promise; + transcribe(buffer: VSBuffer): Promise; } diff --git a/src/vs/platform/voiceRecognition/node/voiceRecognitionService.ts b/src/vs/platform/voiceRecognition/node/voiceRecognitionService.ts index a9c750ec97d..5cc49cae3ab 100644 --- a/src/vs/platform/voiceRecognition/node/voiceRecognitionService.ts +++ b/src/vs/platform/voiceRecognition/node/voiceRecognitionService.ts @@ -3,8 +3,9 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ +import { VSBuffer } from 'vs/base/common/buffer'; import { ILogService } from 'vs/platform/log/common/log'; -import { IAudioBuffer, IVoiceRecognitionService } from 'vs/platform/voiceRecognition/common/voiceRecognitionService'; +import { IVoiceRecognitionService } from 'vs/platform/voiceRecognition/common/voiceRecognitionService'; export class VoiceRecognitionService implements IVoiceRecognitionService { @@ -14,29 +15,50 @@ export class VoiceRecognitionService implements IVoiceRecognitionService { @ILogService private readonly logService: ILogService ) { } - async transcribe(buffer: IAudioBuffer): Promise { - const now = Date.now(); - this.logService.info(`[voice] transcribe(${buffer.channelData.buffer.length}): Begin`); + async transcribe(buffer: VSBuffer): Promise { + this.logService.info(`[voice] transcribe(${buffer.buffer.length / 4}): Begin`); const modulePath = process.env.VSCODE_VOICE_MODULE_PATH; if (!modulePath) { throw new Error('Voice recognition not yet supported!'); } + const now = Date.now(); + const channelData = this.toFloat32Array(buffer); + const conversionTime = Date.now() - now; + const voiceModule: { transcribe: (audioBuffer: { channelCount: 1; sampleRate: 16000; sampleSize: 16; channelData: Float32Array }, options: { language: string | 'auto'; suppressNonSpeechTokens: boolean }) => Promise } = require.__$__nodeRequire(modulePath); const text = await voiceModule.transcribe({ - sampleRate: buffer.sampleRate, - sampleSize: buffer.sampleSize, - channelCount: buffer.channelCount, - channelData: buffer.channelData.buffer + sampleRate: 16000, + sampleSize: 16, + channelCount: 1, + channelData }, { language: 'en', suppressNonSpeechTokens: true }); - this.logService.info(`[voice] transcribe(${buffer.channelData.buffer.length}): End (text: "${text}", took: ${Date.now() - now}ms))`); + this.logService.info(`[voice] transcribe(${buffer.buffer.length / 4}): End (text: "${text}", took: ${Date.now() - now}ms total, ${conversionTime}ms uint8->float32 conversion)`); return text; } + + private toFloat32Array({ buffer: uint8Array }: VSBuffer): Float32Array { + const float32Array = new Float32Array(uint8Array.length / 4); + let offset = 0; + + for (let i = 0; i < float32Array.length; i++) { + const buffer = new ArrayBuffer(4); + const view = new DataView(buffer); + + for (let j = 0; j < 4; j++) { + view.setUint8(j, uint8Array[offset++]); + } + + float32Array[i] = view.getFloat32(0, true); + } + + return float32Array; + } } diff --git a/src/vs/workbench/services/voiceRecognition/electron-sandbox/bufferInputAudioProcessor.js b/src/vs/workbench/services/voiceRecognition/electron-sandbox/bufferInputAudioProcessor.js index bed2b6ea63c..bc87ec89076 100644 --- a/src/vs/workbench/services/voiceRecognition/electron-sandbox/bufferInputAudioProcessor.js +++ b/src/vs/workbench/services/voiceRecognition/electron-sandbox/bufferInputAudioProcessor.js @@ -16,8 +16,8 @@ class BufferInputAudioProcessor extends AudioWorkletProcessor { this.bufferTimespan = 4000; this.startTime = undefined; - this.allInputChannelDataBuffer = undefined; - this.currentInputChannelDataBuffer = []; // buffer over the duration of bufferTimespan + this.allInputUint8Array = undefined; + this.currentInputUint8Arrays = []; // buffer over the duration of bufferTimespan } /** @@ -29,16 +29,16 @@ class BufferInputAudioProcessor extends AudioWorkletProcessor { } const inputChannelData = inputs[0][0]; - this.currentInputChannelDataBuffer.push(inputChannelData.slice(0)); + this.currentInputUint8Arrays.push(this.float32ArrayToUint8Array(inputChannelData.slice(0))); if (Date.now() - this.startTime > this.bufferTimespan) { - const currentInputChannelDataBuffer = this.currentInputChannelDataBuffer; - this.currentInputChannelDataBuffer = []; + const currentInputUint8Arrays = this.currentInputUint8Arrays; + this.currentInputUint8Arrays = []; - this.allInputChannelDataBuffer = this._joinFloat32Arrays(this.allInputChannelDataBuffer ? [this.allInputChannelDataBuffer, ...currentInputChannelDataBuffer] : currentInputChannelDataBuffer); + this.allInputUint8Array = this.joinUint8Arrays(this.allInputUint8Array ? [this.allInputUint8Array, ...currentInputUint8Arrays] : currentInputUint8Arrays); // @ts-ignore - this.port.postMessage(this.allInputChannelDataBuffer); + this.port.postMessage(this.allInputUint8Array); this.startTime = Date.now(); } @@ -47,20 +47,42 @@ class BufferInputAudioProcessor extends AudioWorkletProcessor { } /** - * @param {Float32Array[]} float32Arrays - * @returns {Float32Array} + * @param {Uint8Array[]} uint8Arrays + * @returns {Uint8Array} */ - _joinFloat32Arrays(float32Arrays) { - const result = new Float32Array(float32Arrays.reduce((acc, curr) => acc + curr.length, 0)); + joinUint8Arrays(uint8Arrays) { + const result = new Uint8Array(uint8Arrays.reduce((acc, curr) => acc + curr.length, 0)); let offset = 0; - for (const float32Array of float32Arrays) { - result.set(float32Array, offset); - offset += float32Array.length; + for (const uint8Array of uint8Arrays) { + result.set(uint8Array, offset); + offset += uint8Array.length; } return result; } + + /** + * + * @param {Float32Array} float32Array + * @returns {Uint8Array} + */ + float32ArrayToUint8Array(float32Array) { + const uint8Array = new Uint8Array(float32Array.length * 4); + let offset = 0; + + for (let i = 0; i < float32Array.length; i++) { + const buffer = new ArrayBuffer(4); + const view = new DataView(buffer); + view.setFloat32(0, float32Array[i], true); + + for (let j = 0; j < 4; j++) { + uint8Array[offset++] = view.getUint8(j); + } + } + + return uint8Array; + } } // @ts-ignore diff --git a/src/vs/workbench/services/voiceRecognition/electron-sandbox/workbenchVoiceRecognitionService.ts b/src/vs/workbench/services/voiceRecognition/electron-sandbox/workbenchVoiceRecognitionService.ts index 1e3c986de15..11dfe1c3d33 100644 --- a/src/vs/workbench/services/voiceRecognition/electron-sandbox/workbenchVoiceRecognitionService.ts +++ b/src/vs/workbench/services/voiceRecognition/electron-sandbox/workbenchVoiceRecognitionService.ts @@ -4,7 +4,7 @@ *--------------------------------------------------------------------------------------------*/ import { localize } from 'vs/nls'; -import { VSFloat32Array } from 'vs/base/common/buffer'; +import { VSBuffer } from 'vs/base/common/buffer'; import { CancellationToken, CancellationTokenSource } from 'vs/base/common/cancellation'; import { InstantiationType, registerSingleton } from 'vs/platform/instantiation/common/extensions'; import { createDecorator } from 'vs/platform/instantiation/common/instantiation'; @@ -36,12 +36,12 @@ class BufferInputAudioNode extends AudioWorkletNode { } // TODO@voice -// - load `navigator.mediaDevices.getUserMedia` lazily on startup? or would it trigger a permission prompt? // - how to prevent data processing accumulation when processing is slow? // - how to make this a singleton service that enables ref-counting on multiple callers? // - cancellation should flow to the shared process // - voice module should directly transcribe the PCM32 data -// - we should transfer the Float32Array directly without serialisation overhead +// - we should transfer the Float32Array directly without serialisation overhead maybe from AudioWorklet? +// - the audio worklet should be a TS file (try without any import/export?) export class WorkbenchVoiceRecognitionService implements IWorkbenchVoiceRecognitionService { @@ -115,7 +115,7 @@ export class WorkbenchVoiceRecognitionService implements IWorkbenchVoiceRecognit progress.report({ message: localize('voiceTranscriptionRecording', "Recording from microphone...") }); bufferInputAudioTarget.port.onmessage = async e => { - if (e.data instanceof Float32Array) { + if (e.data instanceof Uint8Array) { this.doTranscribeChunk(e.data, emitter, token); } }; @@ -124,18 +124,12 @@ export class WorkbenchVoiceRecognitionService implements IWorkbenchVoiceRecognit }); } - private async doTranscribeChunk(data: Float32Array, emitter: Emitter, token: CancellationToken): Promise { + private async doTranscribeChunk(data: Uint8Array, emitter: Emitter, token: CancellationToken): Promise { if (token.isCancellationRequested) { return; } - const text = await this.voiceRecognitionService.transcribe({ - sampleRate: WorkbenchVoiceRecognitionService.AUDIO_SAMPLE_RATE, - sampleSize: WorkbenchVoiceRecognitionService.AUDIO_SAMPLE_SIZE, - channelCount: WorkbenchVoiceRecognitionService.AUDIO_CHANNELS, - channelData: VSFloat32Array.wrap(data) - }); - + const text = await this.voiceRecognitionService.transcribe(VSBuffer.wrap(data)); if (token.isCancellationRequested) { return; }