Skip to content

Commit c9b3b97

Browse files
neozhuneozhu
committed
feat: add audio transcription functionality with ASR integration and update VoiceInputButton component
1 parent e31856d commit c9b3b97

7 files changed

Lines changed: 284 additions & 11 deletions

File tree

‎.env.local.example‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,4 +8,6 @@ PUBLIC_SUPABASE_PUBLISHABLE_DEFAULT_KEY=
88
SUPABASE_SERVICE_ROLE_KEY=
99
TTS_API_KEY=
1010
TTS_API_URL=
11-
TTS_VOICE="af_heart"
11+
TTS_VOICE="af_heart"
12+
ASR_API_KEY=
13+
ASR_API_URL=

‎app/api/audio/transcriptions/route.ts‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
import {
2+
buildAsrTranscriptionRequest,
3+
getAsrTranscriptionConfig,
4+
} from "@/lib/botchat/asr";
5+
6+
export async function POST(request: Request) {
7+
const formData = await request.formData().catch(() => null);
8+
const file = formData?.get("file");
9+
10+
if (!(file instanceof File)) {
11+
return Response.json({ error: "Missing audio file." }, { status: 400 });
12+
}
13+
14+
let transcriptionRequest: ReturnType<typeof buildAsrTranscriptionRequest>;
15+
try {
16+
transcriptionRequest = buildAsrTranscriptionRequest(
17+
file,
18+
getAsrTranscriptionConfig()
19+
);
20+
} catch (error) {
21+
console.error("ASR configuration is incomplete", error);
22+
return Response.json({ error: "ASR service is not configured." }, { status: 500 });
23+
}
24+
25+
const upstreamResponse = await fetch(
26+
transcriptionRequest.url,
27+
transcriptionRequest.init
28+
);
29+
30+
if (!upstreamResponse.ok) {
31+
const upstreamError = await upstreamResponse.text().catch(() => "");
32+
console.error("ASR service request failed", {
33+
status: upstreamResponse.status,
34+
error: upstreamError,
35+
});
36+
return Response.json({ error: "ASR service request failed." }, { status: 502 });
37+
}
38+
39+
const transcription = await upstreamResponse.json().catch(() => null);
40+
if (
41+
!transcription ||
42+
typeof transcription !== "object" ||
43+
typeof (transcription as { text?: unknown }).text !== "string"
44+
) {
45+
console.error("ASR service returned an invalid transcription response");
46+
return Response.json({ error: "ASR service returned an invalid response." }, { status: 502 });
47+
}
48+
49+
return Response.json({ text: transcription.text.trim() });
50+
}

‎components/botchat/VoiceInputButton.tsx‎

Lines changed: 128 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,8 @@ export interface VoiceInputButtonProps {
3131
onRecordingChange?: (recording: boolean) => void;
3232
onStart?: () => void;
3333
onStop?: (elapsedSeconds: number) => void;
34+
onTranscription?: (text: string) => void;
35+
onError?: (error: Error) => void;
3436
}
3537

3638
function formatElapsedTime(totalSeconds: number): string {
@@ -52,6 +54,8 @@ export function VoiceInputButton({
5254
onRecordingChange,
5355
onStart,
5456
onStop,
57+
onTranscription,
58+
onError,
5559
}: VoiceInputButtonProps) {
5660
const isControlled = recording !== undefined;
5761
const [internalRecording, setInternalRecording] = useState(defaultRecording);
@@ -62,20 +66,135 @@ export function VoiceInputButton({
6266
);
6367
const startedAtRef = useRef<number | null>(null);
6468
const elapsedSecondsRef = useRef(0);
69+
const mediaRecorderRef = useRef<MediaRecorder | null>(null);
70+
const audioChunksRef = useRef<Blob[]>([]);
71+
const [isStarting, setIsStarting] = useState(false);
72+
const [isTranscribing, setIsTranscribing] = useState(false);
73+
74+
const reportError = useCallback(
75+
(error: unknown) => {
76+
const nextError =
77+
error instanceof Error ? error : new Error("Voice input failed.");
78+
console.error("Voice input failed", nextError);
79+
onError?.(nextError);
80+
},
81+
[onError]
82+
);
83+
84+
const transcribeAudio = useCallback(
85+
async (audio: Blob) => {
86+
const formData = new FormData();
87+
formData.append("file", audio, "voice-input.webm");
88+
89+
const response = await fetch("/api/audio/transcriptions", {
90+
method: "POST",
91+
body: formData,
92+
});
93+
94+
if (!response.ok) {
95+
throw new Error(`ASR request failed with status ${response.status}`);
96+
}
97+
98+
const transcription = (await response.json()) as { text?: unknown };
99+
const text =
100+
typeof transcription.text === "string" ? transcription.text.trim() : "";
101+
102+
if (text) onTranscription?.(text);
103+
},
104+
[onTranscription]
105+
);
106+
107+
const startRecording = useCallback(async () => {
108+
if (
109+
disabled ||
110+
isRecording ||
111+
isStarting ||
112+
isTranscribing ||
113+
!navigator.mediaDevices?.getUserMedia
114+
) {
115+
if (!navigator.mediaDevices?.getUserMedia) {
116+
reportError(new Error("Audio recording is not supported in this browser."));
117+
}
118+
return;
119+
}
120+
121+
setIsStarting(true);
122+
123+
try {
124+
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
125+
const mediaRecorder = new MediaRecorder(stream);
126+
127+
audioChunksRef.current = [];
128+
mediaRecorder.ondataavailable = (event) => {
129+
if (event.data.size > 0) audioChunksRef.current.push(event.data);
130+
};
131+
mediaRecorder.onstop = () => {
132+
stream.getTracks().forEach((track) => track.stop());
133+
mediaRecorderRef.current = null;
134+
135+
const audio = new Blob(audioChunksRef.current, {
136+
type: mediaRecorder.mimeType || "audio/webm",
137+
});
138+
audioChunksRef.current = [];
139+
140+
if (audio.size === 0) return;
141+
142+
setIsTranscribing(true);
143+
void transcribeAudio(audio)
144+
.catch(reportError)
145+
.finally(() => setIsTranscribing(false));
146+
};
147+
148+
mediaRecorderRef.current = mediaRecorder;
149+
mediaRecorder.start();
150+
151+
if (!isControlled) setInternalRecording(true);
152+
onRecordingChange?.(true);
153+
onStart?.();
154+
} catch (error) {
155+
reportError(error);
156+
} finally {
157+
setIsStarting(false);
158+
}
159+
}, [
160+
disabled,
161+
isControlled,
162+
isRecording,
163+
isStarting,
164+
isTranscribing,
165+
onRecordingChange,
166+
onStart,
167+
reportError,
168+
transcribeAudio,
169+
]);
65170

66171
const setRecording = useCallback(
67172
(nextRecording: boolean) => {
68-
if ((disabled && nextRecording) || nextRecording === isRecording) return;
173+
if (nextRecording) {
174+
void startRecording();
175+
return;
176+
}
177+
178+
if (!isRecording) return;
69179

70-
if (!isControlled) setInternalRecording(nextRecording);
71-
onRecordingChange?.(nextRecording);
180+
if (!isControlled) setInternalRecording(false);
181+
onRecordingChange?.(false);
182+
onStop?.(elapsedSecondsRef.current);
72183

73-
if (nextRecording) onStart?.();
74-
else onStop?.(elapsedSecondsRef.current);
184+
const mediaRecorder = mediaRecorderRef.current;
185+
if (mediaRecorder?.state !== "inactive") mediaRecorder?.stop();
75186
},
76-
[disabled, isControlled, isRecording, onRecordingChange, onStart, onStop]
187+
[isControlled, isRecording, onRecordingChange, onStop, startRecording]
77188
);
78189

190+
useEffect(() => {
191+
return () => {
192+
const mediaRecorder = mediaRecorderRef.current;
193+
if (mediaRecorder?.state !== "inactive") mediaRecorder?.stop();
194+
mediaRecorder?.stream.getTracks().forEach((track) => track.stop());
195+
};
196+
}, []);
197+
79198
useEffect(() => {
80199
if (!isRecording) {
81200
startedAtRef.current = null;
@@ -136,15 +255,15 @@ export function VoiceInputButton({
136255
className={cn(
137256
styles.voiceControl,
138257
isRecording && styles.recording,
139-
disabled && !isRecording && styles.disabled,
258+
(disabled || isStarting || isTranscribing) && !isRecording && styles.disabled,
140259
className
141260
)}
142261
data-recording={isRecording}
143262
>
144263
<button
145264
className={styles.voiceTrigger}
146265
type="button"
147-
disabled={disabled && !isRecording}
266+
disabled={(disabled || isStarting || isTranscribing) && !isRecording}
148267
aria-label={isRecording ? activeLabel : startLabel}
149268
aria-pressed={isRecording}
150269
title={isRecording ? activeLabel : startLabel}
@@ -189,4 +308,4 @@ export function VoiceInputButton({
189308
</div>
190309
</div>
191310
);
192-
}
311+
}

‎components/botchat/chat-panel.tsx‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1131,7 +1131,12 @@ export function ChatPanel({
11311131
<div className="mt-3 flex flex-wrap items-center justify-between gap-2.5">
11321132
<div className="flex flex-wrap items-center gap-1.5">
11331133
<ToolbarIcon icon={MessageCircle} label="Message type" />
1134-
<VoiceInputButton disabled={!canSend} />
1134+
<VoiceInputButton
1135+
disabled={!canSend}
1136+
onTranscription={(text) =>
1137+
setInput(input + (input ? " " : "") + text)
1138+
}
1139+
/>
11351140
<ToolbarIcon
11361141
icon={Brain}
11371142
label={isHighReasoning ? "Reasoning: High" : "Reasoning: Low"}

‎components/botchat/voice-input-button.test.mts‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
import test from "node:test";
2+
import assert from "node:assert/strict";
3+
import { readFileSync } from "node:fs";
4+
5+
const voiceInputSource = readFileSync(
6+
new URL("./VoiceInputButton.tsx", import.meta.url),
7+
"utf8"
8+
);
9+
const chatPanelSource = readFileSync(
10+
new URL("./chat-panel.tsx", import.meta.url),
11+
"utf8"
12+
);
13+
14+
test("voice input records audio, transcribes it, and appends the result to the chat draft", () => {
15+
assert.match(voiceInputSource, /new MediaRecorder\(/);
16+
assert.match(voiceInputSource, /fetch\("\/api\/audio\/transcriptions"/);
17+
assert.match(voiceInputSource, /onTranscription\?\./);
18+
assert.match(chatPanelSource, /<VoiceInputButton[\s\S]*onTranscription=/);
19+
assert.match(chatPanelSource, /setInput\(input \+ \(input \? " " : ""\) \+ text\)/);
20+
});

‎lib/botchat/asr.test.mts‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
import test from "node:test";
2+
import assert from "node:assert/strict";
3+
4+
import {
5+
buildAsrTranscriptionRequest,
6+
getAsrTranscriptionConfig,
7+
} from "./asr.ts";
8+
9+
test("getAsrTranscriptionConfig requires the ASR endpoint and key", () => {
10+
assert.throws(
11+
() => getAsrTranscriptionConfig({ ASR_API_KEY: "key" }),
12+
/ASR_API_URL/
13+
);
14+
});
15+
16+
test("buildAsrTranscriptionRequest prepares an OpenAI-compatible upload", async () => {
17+
const file = new File(["audio"], "voice-input.webm", { type: "audio/webm" });
18+
const request = buildAsrTranscriptionRequest(file, {
19+
apiKey: "secret",
20+
apiUrl: "https://asr.test/v1/audio/transcriptions",
21+
});
22+
23+
assert.equal(request.url, "https://asr.test/v1/audio/transcriptions");
24+
assert.equal(request.init.method, "POST");
25+
assert.equal(request.init.headers.Authorization, "Bearer secret");
26+
assert.equal(request.init.body.get("file"), file);
27+
assert.equal(request.init.body.get("model"), "whisper-1");
28+
assert.equal(request.init.body.get("response_format"), "json");
29+
});

‎lib/botchat/asr.ts‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
type AsrTranscriptionEnv = {
2+
[key: string]: string | undefined;
3+
ASR_API_KEY?: string;
4+
ASR_API_URL?: string;
5+
};
6+
7+
export type AsrTranscriptionConfig = {
8+
apiKey: string;
9+
apiUrl: string;
10+
};
11+
12+
export function getAsrTranscriptionConfig(
13+
env: AsrTranscriptionEnv = process.env
14+
): AsrTranscriptionConfig {
15+
const apiKey = env.ASR_API_KEY?.trim();
16+
const apiUrl = env.ASR_API_URL?.trim();
17+
18+
if (!apiKey) {
19+
throw new Error("Missing ASR_API_KEY environment variable.");
20+
}
21+
22+
if (!apiUrl) {
23+
throw new Error("Missing ASR_API_URL environment variable.");
24+
}
25+
26+
return { apiKey, apiUrl };
27+
}
28+
29+
export function buildAsrTranscriptionRequest(
30+
file: File,
31+
config: AsrTranscriptionConfig
32+
) {
33+
const body = new FormData();
34+
body.append("file", file);
35+
body.append("model", "whisper-1");
36+
body.append("response_format", "json");
37+
38+
return {
39+
url: config.apiUrl,
40+
init: {
41+
method: "POST",
42+
headers: {
43+
Authorization: `Bearer ${config.apiKey}`,
44+
},
45+
body,
46+
},
47+
};
48+
}

0 commit comments

Comments
 (0)