From 70eecdc300c3769b2f2781dc87a145dacd6238b7 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sat, 3 Oct 2026 09:46:29 -0700 Subject: [PATCH] perf(mobile): defer audio recorder creation until dictation --- .../voice-input/VoiceInputProvider.tsx | 53 +++-- .../voice-input/lazyVoiceRecorder.test.ts | 181 ++++++++++++++++++ .../features/voice-input/lazyVoiceRecorder.ts | 88 +++++++++ 3 files changed, 307 insertions(+), 15 deletions(-) create mode 100644 apps/mobile/src/features/voice-input/lazyVoiceRecorder.test.ts create mode 100644 apps/mobile/src/features/voice-input/lazyVoiceRecorder.ts diff --git a/apps/mobile/src/features/voice-input/VoiceInputProvider.tsx b/apps/mobile/src/features/voice-input/VoiceInputProvider.tsx index f6deefcea87b..ebd628e4bad4 100644 --- a/apps/mobile/src/features/voice-input/VoiceInputProvider.tsx +++ b/apps/mobile/src/features/voice-input/VoiceInputProvider.tsx @@ -1,9 +1,10 @@ import { + AudioModule, RecordingPresets, requestRecordingPermissionsAsync, setAudioModeAsync, setIsAudioActiveAsync, - useAudioRecorder, + type RecorderState, type RecordingStatus, } from "expo-audio"; import { File } from "expo-file-system"; @@ -18,7 +19,7 @@ import { useState, type ReactNode, } from "react"; -import { AppState } from "react-native"; +import { AppState, Platform } from "react-native"; import { useSharedValue } from "react-native-reanimated"; import { getLocalVoiceTranscriber } from "../../native/voiceTranscription"; @@ -28,14 +29,23 @@ import { voiceInputBlocksSubmission, type VoiceInputState, } from "@t3tools/client-runtime/voice-input"; +import { createLazyVoiceRecorder, type LazyVoiceRecorder } from "./lazyVoiceRecorder"; import { normalizeVoiceInputDecibels, VOICE_WAVEFORM_SAMPLE_COUNT } from "./voiceInputMetering"; import { VoiceInputSession } from "./voiceInputSession"; const INITIAL_STATE: VoiceInputState = { phase: "idle", error: null, errorAction: null }; const VOICE_METERING_INTERVAL_MS = 80; +// The native constructor takes platform-flattened options, as `useAudioRecorder` +// builds them with expo-audio's internal `createRecordingOptions`. +const { ios: IOS_RECORDING_OPTIONS, android: ANDROID_RECORDING_OPTIONS } = + RecordingPresets.HIGH_QUALITY; const VOICE_RECORDING_OPTIONS = { - ...RecordingPresets.HIGH_QUALITY, + extension: RecordingPresets.HIGH_QUALITY.extension, + sampleRate: RecordingPresets.HIGH_QUALITY.sampleRate, + numberOfChannels: RecordingPresets.HIGH_QUALITY.numberOfChannels, + bitRate: RecordingPresets.HIGH_QUALITY.bitRate, isMeteringEnabled: true, + ...(Platform.OS === "ios" ? IOS_RECORDING_OPTIONS : ANDROID_RECORDING_OPTIONS), }; async function releaseVoiceRecordingAudio(): Promise { @@ -102,17 +112,22 @@ function useVoiceInputRuntime() { const audioLevelsRef = useRef(Array(VOICE_WAVEFORM_SAMPLE_COUNT).fill(0)); const audioLevels = useSharedValue(audioLevelsRef.current); const sessionRef = useRef(null); - const handleRecorderStatus = useCallback((status: RecordingStatus) => { - sessionRef.current?.controller.handleRecorderStatus({ - isFinished: status.isFinished, - hasError: status.hasError || status.mediaServicesDidReset === true, - error: status.error, - url: status.url, + const recorderRef = useRef | null>(null); + + if (!sessionRef.current || !recorderRef.current) { + // The native recorder is created when dictation starts, not on app launch. + const recorder = createLazyVoiceRecorder({ + create: () => new AudioModule.AudioRecorder(VOICE_RECORDING_OPTIONS), + onStatus: (status: RecordingStatus) => { + sessionRef.current?.controller.handleRecorderStatus({ + isFinished: status.isFinished, + hasError: status.hasError || status.mediaServicesDidReset === true, + error: status.error, + url: status.url, + }); + }, }); - }, []); - const recorder = useAudioRecorder(VOICE_RECORDING_OPTIONS, handleRecorderStatus); - - if (!sessionRef.current) { + recorderRef.current = recorder; sessionRef.current = new VoiceInputSession({ recorder, getTranscriber: getLocalVoiceTranscriber, @@ -134,6 +149,7 @@ function useVoiceInputRuntime() { const session = sessionRef.current; const controller = session.controller; + const recorder = recorderRef.current; useEffect(() => { const subscription = AppState.addEventListener("change", (nextState) => { @@ -145,7 +161,14 @@ function useVoiceInputRuntime() { return () => subscription.remove(); }, [controller]); - useEffect(() => () => controller.dispose(), [controller]); + useEffect( + () => () => { + // Dispose first so an active recording is stopped before the recorder is released. + controller.dispose(); + recorder.release(); + }, + [controller, recorder], + ); useEffect(() => { if (state.phase !== "recording") return; @@ -175,7 +198,7 @@ function useVoiceInputRuntime() { const sampleRecording = () => { if (controller.currentState.phase !== "recording") return; const status = recorder.getStatus(); - if (!status.isRecording) return; + if (!status?.isRecording) return; const level = normalizeVoiceInputDecibels(status.metering); const history = audioLevelsRef.current; diff --git a/apps/mobile/src/features/voice-input/lazyVoiceRecorder.test.ts b/apps/mobile/src/features/voice-input/lazyVoiceRecorder.test.ts new file mode 100644 index 000000000000..ffba8faefffd --- /dev/null +++ b/apps/mobile/src/features/voice-input/lazyVoiceRecorder.test.ts @@ -0,0 +1,181 @@ +import { describe, expect, it, vi } from "vite-plus/test"; + +import { VoiceInputController } from "@t3tools/client-runtime/voice-input"; + +import { createLazyVoiceRecorder } from "./lazyVoiceRecorder"; + +type Status = { readonly isFinished: boolean }; +type State = { readonly isRecording: boolean }; + +function deferred() { + let resolve!: () => void; + const promise = new Promise((resolvePromise) => { + resolve = resolvePromise; + }); + return { promise, resolve }; +} + +function createNativeRecorder(uri: string) { + const listeners = new Set<(status: Status) => void>(); + return { + uri: null as string | null, + prepared: uri, + prepareToRecordAsync: vi.fn(async function (this: { uri: string | null; prepared: string }) { + this.uri = this.prepared; + }), + record: vi.fn(), + stop: vi.fn(async (): Promise => undefined), + getStatus: vi.fn((): State => ({ isRecording: true })), + addListener: vi.fn((_event: "recordingStatusUpdate", listener: (status: Status) => void) => { + listeners.add(listener); + return { remove: () => listeners.delete(listener) }; + }), + release: vi.fn(), + emit: (status: Status) => listeners.forEach((listener) => listener(status)), + }; +} + +function createHarness() { + const created: Array> = []; + const statuses: Status[] = []; + const recorder = createLazyVoiceRecorder({ + create: () => { + const native = createNativeRecorder(`file:///voice-${created.length}.m4a`); + created.push(native); + return native; + }, + onStatus: (status) => statuses.push(status), + }); + return { recorder, created, statuses }; +} + +describe("createLazyVoiceRecorder", () => { + it("does not create a native recorder until a recording prepares", async () => { + const { recorder, created } = createHarness(); + + expect(recorder.uri).toBeNull(); + expect(recorder.getStatus()).toBeNull(); + await recorder.stop(); + recorder.release(); + expect(created).toHaveLength(0); + + await recorder.prepareToRecordAsync(); + recorder.record({ forDuration: 300 }); + + expect(created).toHaveLength(1); + expect(created[0]!.record).toHaveBeenCalledWith({ forDuration: 300 }); + expect(recorder.uri).toBe("file:///voice-0.m4a"); + expect(recorder.getStatus()).toEqual({ isRecording: true }); + }); + + it("reuses one native recorder across recordings", async () => { + const { recorder, created } = createHarness(); + + await recorder.prepareToRecordAsync(); + await recorder.stop(); + await recorder.prepareToRecordAsync(); + + expect(created).toHaveLength(1); + expect(created[0]!.prepareToRecordAsync).toHaveBeenCalledTimes(2); + }); + + it("forwards status events until the recorder is released", async () => { + const { recorder, created, statuses } = createHarness(); + await recorder.prepareToRecordAsync(); + const native = created[0]!; + + native.emit({ isFinished: true }); + recorder.release(); + native.emit({ isFinished: false }); + + expect(statuses).toEqual([{ isFinished: true }]); + }); + + it("releases only after a pending stop settles and keeps the final uri", async () => { + const { recorder, created } = createHarness(); + await recorder.prepareToRecordAsync(); + const native = created[0]!; + const stopped = deferred(); + native.stop.mockReturnValueOnce(stopped.promise); + + const stopping = recorder.stop(); + recorder.release(); + await Promise.resolve(); + expect(native.release).not.toHaveBeenCalled(); + expect(recorder.uri).toBe("file:///voice-0.m4a"); + + stopped.resolve(); + await stopping; + await vi.waitFor(() => expect(native.release).toHaveBeenCalledTimes(1)); + expect(recorder.uri).toBe("file:///voice-0.m4a"); + expect(recorder.getStatus()).toBeNull(); + }); + + it("releases after a failed stop and creates a fresh recorder for the next start", async () => { + const { recorder, created } = createHarness(); + await recorder.prepareToRecordAsync(); + const first = created[0]!; + first.stop.mockRejectedValueOnce(new Error("not recording")); + + await expect(recorder.stop()).rejects.toThrow("not recording"); + recorder.release(); + await vi.waitFor(() => expect(first.release).toHaveBeenCalledTimes(1)); + expect(() => recorder.record({ forDuration: 1 })).toThrow("not prepared"); + + await recorder.prepareToRecordAsync(); + expect(created).toHaveLength(2); + expect(recorder.uri).toBe("file:///voice-1.m4a"); + }); + + describe("with the voice input controller", () => { + function createController(granted: boolean) { + const harness = createHarness(); + const deleted: string[] = []; + const controller = new VoiceInputController({ + recorder: harness.recorder, + getTranscriber: () => ({ + prepare: async () => ({ locale: "en-US", transcribe: async () => "text" }), + }), + requestPermission: async () => ({ granted, canAskAgain: true }), + configureRecording: async () => undefined, + releaseRecording: async () => undefined, + deleteRecording: (uri) => deleted.push(uri), + readDraft: () => ({ + ownerKey: "thread", + text: "", + selection: { start: 0, end: 0 }, + revision: 0, + }), + commitDraft: () => undefined, + onStateChange: () => undefined, + }); + return { ...harness, controller, deleted }; + } + + it("does not create a native recorder when microphone permission is denied", async () => { + const { controller, created } = createController(false); + + await controller.start(); + + expect(controller.currentState.phase).toBe("error"); + expect(created).toHaveLength(0); + }); + + it("stops, deletes, and releases the recorder created by a cancelled recording", async () => { + const { controller, recorder, created, deleted } = createController(true); + + await controller.start(); + expect(controller.currentState.phase).toBe("recording"); + expect(created).toHaveLength(1); + const native = created[0]!; + + controller.cancel(); + recorder.release(); + await vi.waitFor(() => expect(native.release).toHaveBeenCalledTimes(1)); + + expect(native.stop).toHaveBeenCalledTimes(1); + expect(deleted).toEqual(["file:///voice-0.m4a"]); + expect(controller.currentState.phase).toBe("idle"); + }); + }); +}); diff --git a/apps/mobile/src/features/voice-input/lazyVoiceRecorder.ts b/apps/mobile/src/features/voice-input/lazyVoiceRecorder.ts new file mode 100644 index 000000000000..d6deb2163c63 --- /dev/null +++ b/apps/mobile/src/features/voice-input/lazyVoiceRecorder.ts @@ -0,0 +1,88 @@ +import type { VoiceRecorder } from "@t3tools/client-runtime/voice-input"; + +type Subscription = { remove(): void }; + +// The subset of expo-audio's native `AudioRecorder` the voice input hook uses. +export interface NativeVoiceRecorder { + readonly uri: string | null; + prepareToRecordAsync(): Promise; + record(options: { readonly forDuration: number }): void; + stop(): Promise; + getStatus(): TRecorderState; + addListener( + eventName: "recordingStatusUpdate", + listener: (status: TStatus) => void, + ): Subscription; + release(): void; +} + +export interface LazyVoiceRecorder extends VoiceRecorder { + /** Status of the native recorder, or null before a recording has prepared one. */ + getStatus(): TRecorderState | null; + /** + * Releases the native recorder after its pending prepare and stop calls settle. + * A later recording creates a new one, so replayed effects keep working. + */ + release(): void; +} + +/** + * Creates expo-audio's native `AudioRecorder` when a recording first prepares, + * instead of on composer render. Construction blocks the JS thread for tens of + * milliseconds on a cold composer, and most composer mounts never record. + */ +export function createLazyVoiceRecorder(input: { + readonly create: () => NativeVoiceRecorder; + readonly onStatus: (status: TStatus) => void; +}): LazyVoiceRecorder { + let recorder: NativeVoiceRecorder | null = null; + let subscription: Subscription | null = null; + // A released recorder still answers `uri` so the controller can delete its file. + let retired: NativeVoiceRecorder | null = null; + let retiredUri: string | null = null; + let pending: Promise = Promise.resolve(); + + const track = (promise: Promise): Promise => { + pending = Promise.allSettled([pending, promise]).then(() => undefined); + return promise; + }; + + return { + get uri() { + if (recorder) return recorder.uri; + return retired ? retired.uri : retiredUri; + }, + prepareToRecordAsync() { + if (!recorder) { + recorder = input.create(); + subscription = recorder.addListener("recordingStatusUpdate", input.onStatus); + } + return track(recorder.prepareToRecordAsync()); + }, + record(options) { + if (!recorder) throw new Error("Voice recorder is not prepared."); + recorder.record(options); + }, + stop() { + return recorder ? track(recorder.stop()) : Promise.resolve(); + }, + getStatus() { + return recorder ? recorder.getStatus() : null; + }, + release() { + const released = recorder; + if (!released) return; + recorder = null; + subscription?.remove(); + subscription = null; + retired = released; + void pending.then(() => { + if (retired === released) { + retiredUri = released.uri; + retired = null; + } + released.release(); + }); + }, + }; +}