diff --git a/apps/docs/content/docs/api-reference/api.mdx b/apps/docs/content/docs/api-reference/api.mdx index 9c7f4ada4..786fdf4c8 100644 --- a/apps/docs/content/docs/api-reference/api.mdx +++ b/apps/docs/content/docs/api-reference/api.mdx @@ -113,6 +113,78 @@ curl -X GET 'https://sexyvoice.ai/api/v1/billing' \ -H 'Authorization: Bearer sk_live_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx' ``` +## Voice Cloning + +Clone a voice from a short reference recording and synthesize new speech with it +via `POST /api/v1/clone`. Provide the reference audio as **either** a public URL +(`reference_audio_url`) **or** base64 (`reference_audio`) — not both. The +language (`locale`, default `en`) selects the cloning provider automatically. + +Reference audio of **3–25 seconds** works best. Longer clips are trimmed +automatically. Set `enhance_reference_audio: true` to denoise the reference +before cloning (bills additional credits per second). + +### curl (reference audio URL) + +```bash +curl -X POST 'https://sexyvoice.ai/api/v1/clone' \ + -H 'Authorization: Bearer sk_live_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx' \ + -H 'Content-Type: application/json' \ + -d '{ + "input": "Hello from my cloned voice!", + "locale": "en", + "reference_audio_url": "https://example.com/reference-sample.wav" + }' +``` + +### TypeScript (base64 reference audio) + +```ts +import { readFileSync } from 'node:fs'; + +const referenceAudio = readFileSync('sample.wav').toString('base64'); + +const response = await fetch('https://sexyvoice.ai/api/v1/clone', { + method: 'POST', + headers: { + Authorization: `Bearer ${process.env.SEXYVOICE_API_KEY}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + input: 'Hello from TypeScript', + locale: 'en', + reference_audio: referenceAudio, + reference_audio_format: 'audio/wav', + }), +}); + +const json = await response.json(); +// json.url contains the generated cloned-voice audio URL +``` + +### Python (with enhancement) + +```python +import requests + +response = requests.post( + 'https://sexyvoice.ai/api/v1/clone', + headers={ + 'Authorization': f'Bearer {API_KEY}', + 'Content-Type': 'application/json', + }, + json={ + 'input': 'Hello from Python', + 'locale': 'en', + 'reference_audio_url': 'https://example.com/reference-sample.wav', + 'enhance_reference_audio': True, + }, + timeout=120, +) + +print(response.status_code, response.json()) +``` + ### Python (gpro) ```python diff --git a/apps/web/app/[lang]/(dashboard)/dashboard/api-keys/api-keys.tsx b/apps/web/app/[lang]/(dashboard)/dashboard/api-keys/api-keys.tsx index 761c10b87..a84b0b2f0 100644 --- a/apps/web/app/[lang]/(dashboard)/dashboard/api-keys/api-keys.tsx +++ b/apps/web/app/[lang]/(dashboard)/dashboard/api-keys/api-keys.tsx @@ -5,6 +5,7 @@ import { Copy, ExternalLink, KeyRound, + Mic, Plus, Terminal, Trash2, @@ -322,6 +323,14 @@ export function ApiKeys({ isPaidUser }: { isPaidUser: boolean }) { {t('infoCard.generateDescription')} + + {t('infoCard.cloneDescription')} + [0]['type']; +} + +const CLONE_ERROR_MAP: Record = { + unsupported_locale: { + status: 400, + code: 'unsupported_locale', + type: 'invalid_request_error', + param: 'locale', + }, + audio_duration_unknown: { + status: 400, + code: 'invalid_reference_audio', + type: 'invalid_request_error', + param: 'reference_audio', + }, + audio_duration_too_short: { + status: 400, + code: 'reference_audio_too_short', + type: 'invalid_request_error', + param: 'reference_audio', + }, + audio_conversion_failed: { + status: 400, + code: 'unsupported_audio_format', + type: 'invalid_request_error', + param: 'reference_audio', + }, + unsupported_audio_format: { + status: 400, + code: 'unsupported_audio_format', + type: 'invalid_request_error', + param: 'reference_audio', + }, + guardrail_violation: { + status: 422, + code: 'content_policy_violation', + type: 'invalid_request_error', + param: 'input', + }, + provider_unavailable: { + status: 503, + code: 'provider_unavailable', + type: 'server_error', + param: null, + }, +}; + +function isPrivateIPv4(ip: string): boolean { + const [a, b] = ip.split('.').map(Number); + if (a === 0 || a === 10 || a === 127) return true; + if (a === 169 && b === 254) return true; // link-local + if (a === 172 && b >= 16 && b <= 31) return true; + if (a === 192 && b === 168) return true; + if (a === 100 && b >= 64 && b <= 127) return true; // CGNAT + return a >= 224; // multicast / reserved +} + +function isPrivateIPv6(ip: string): boolean { + const lower = ip.toLowerCase(); + if (lower === '::1' || lower === '::') return true; + if (lower.startsWith('fe80')) return true; // link-local + if (lower.startsWith('fc') || lower.startsWith('fd')) return true; // unique local + const mapped = lower.match(/^::ffff:(\d+\.\d+\.\d+\.\d+)$/); + if (mapped) return isPrivateIPv4(mapped[1]); + return false; +} + +// Reject loopback, link-local, private, CGNAT, and reserved address ranges so a +// user-supplied URL cannot be used to reach internal services (SSRF). +function isPrivateAddress(ip: string): boolean { + return isIP(ip) === 4 ? isPrivateIPv4(ip) : isPrivateIPv6(ip); +} + +async function assertPublicHttpUrl(rawUrl: string): Promise { + let url: URL; + try { + url = new URL(rawUrl); + } catch { + throw new CloneServiceError( + 'unsupported_audio_format', + 'reference_audio_url is not a valid URL.', + { param: 'reference_audio_url' }, + ); + } + + if (url.protocol !== 'http:' && url.protocol !== 'https:') { + throw new CloneServiceError( + 'unsupported_audio_format', + 'reference_audio_url must use http or https.', + { param: 'reference_audio_url' }, + ); + } + if (url.username || url.password) { + throw new CloneServiceError( + 'unsupported_audio_format', + 'reference_audio_url must not contain credentials.', + { param: 'reference_audio_url' }, + ); + } + + const hostname = url.hostname.replace(/^\[|\]$/g, ''); + let addresses: { address: string }[]; + if (isIP(hostname)) { + addresses = [{ address: hostname }]; + } else { + try { + addresses = await lookup(hostname, { all: true }); + } catch { + throw new CloneServiceError( + 'unsupported_audio_format', + 'reference_audio_url host could not be resolved.', + { param: 'reference_audio_url' }, + ); + } + } + + if ( + addresses.length === 0 || + addresses.some((a) => isPrivateAddress(a.address)) + ) { + throw new CloneServiceError( + 'unsupported_audio_format', + 'reference_audio_url must point to a public host.', + { param: 'reference_audio_url' }, + ); + } + + return url; +} + +async function fetchReferenceAudioFromUrl( + rawUrl: string, +): Promise<{ buffer: Buffer; mimeType: string; filename: string }> { + const url = await assertPublicHttpUrl(rawUrl); + + let response: Response; + try { + response = await fetch(url, { + // Block redirect-based SSRF bypasses: a public URL must not bounce to an + // internal host. Reference audio is expected to be a direct link. + redirect: 'error', + signal: AbortSignal.timeout(REFERENCE_AUDIO_FETCH_TIMEOUT_MS), + }); + } catch (error) { + const timedOut = error instanceof Error && error.name === 'TimeoutError'; + throw new CloneServiceError( + 'unsupported_audio_format', + timedOut + ? 'Timed out downloading reference_audio_url.' + : 'Failed to download reference_audio_url. Provide a direct, publicly reachable link.', + { param: 'reference_audio_url' }, + ); + } + + if (!response.ok) { + throw new CloneServiceError( + 'unsupported_audio_format', + `Failed to download reference audio (status ${response.status})`, + { param: 'reference_audio_url' }, + ); + } + + const contentLength = response.headers.get('content-length'); + if ( + contentLength && + Number.isFinite(Number(contentLength)) && + Number(contentLength) > REFERENCE_AUDIO_MAX_FETCH_BYTES + ) { + throw new CloneServiceError( + 'unsupported_audio_format', + 'Reference audio exceeds the maximum allowed size.', + { param: 'reference_audio_url' }, + ); + } + + // Stream the body and abort once the hard byte cap is exceeded, so a missing + // or spoofed content-length header cannot exhaust memory. + const reader = response.body?.getReader(); + if (!reader) { + throw new CloneServiceError( + 'unsupported_audio_format', + 'Failed to read reference audio stream.', + { param: 'reference_audio_url' }, + ); + } + + const chunks: Uint8Array[] = []; + let totalBytes = 0; + while (true) { + const { done, value } = await reader.read(); + if (done) break; + if (value) { + totalBytes += value.byteLength; + if (totalBytes > REFERENCE_AUDIO_MAX_FETCH_BYTES) { + await reader.cancel(); + throw new CloneServiceError( + 'unsupported_audio_format', + 'Reference audio exceeds the maximum allowed size.', + { param: 'reference_audio_url' }, + ); + } + chunks.push(value); + } + } + + const mimeType = + response.headers.get('content-type')?.split(';')[0]?.trim() || 'audio/wav'; + const filename = url.pathname.split('/').pop() || 'reference-audio'; + + return { buffer: Buffer.concat(chunks), mimeType, filename }; +} + +async function resolveReferenceAudio(data: { + reference_audio_url?: string; + reference_audio?: string; + reference_audio_format?: string; +}): Promise<{ buffer: Buffer; mimeType: string; filename: string }> { + if (data.reference_audio_url) { + return await fetchReferenceAudioFromUrl(data.reference_audio_url); + } + + // base64 payload + const base64 = (data.reference_audio ?? '').replace( + /^data:[^;]+;base64,/, + '', + ); + const buffer = Buffer.from(base64, 'base64'); + if (buffer.length === 0) { + throw new CloneServiceError( + 'unsupported_audio_format', + 'reference_audio is not valid base64-encoded audio.', + { param: 'reference_audio' }, + ); + } + // Enforce the same hard size cap as the URL path so a large base64 body + // cannot allocate unbounded memory. + if (buffer.length > REFERENCE_AUDIO_MAX_FETCH_BYTES) { + throw new CloneServiceError( + 'unsupported_audio_format', + 'Reference audio exceeds the maximum allowed size.', + { param: 'reference_audio' }, + ); + } + const mimeType = + data.reference_audio_format?.split(';')[0]?.trim() || 'audio/wav'; + return { buffer, mimeType, filename: 'reference-audio' }; +} + +// biome-ignore lint/complexity/noExcessiveCognitiveComplexity: Single entrypoint orchestrates validation, billing, cloning and persistence. +export async function POST(request: Request) { + const requestId = getExternalApiRequestId(); + const log = createLogger({ requestId, endpoint: ENDPOINT }); + + const authHeader = request.headers.get('authorization'); + if (!authHeader) { + await log({ status: 401, errorCode: 'missing_authorization_header' }); + return externalApiErrorResponse({ + key: 'missing_authorization_header', + requestId, + }); + } + + const authResult = await validateApiKey(authHeader); + if (!authResult) { + await log({ status: 401, errorCode: 'invalid_api_key' }); + return externalApiErrorResponse({ key: 'invalid_api_key', requestId }); + } + + const rateLimit = await consumeRateLimit(authResult.keyHash); + if (!rateLimit.allowed) { + await log({ + status: 429, + errorCode: 'rate_limit_exceeded', + userId: authResult.userId, + apiKeyId: authResult.apiKeyId, + }); + return externalApiErrorResponse({ + key: 'rate_limit_exceeded', + rateLimit, + requestId, + }); + } + + const respond = ( + body: unknown, + init: ResponseInit = {}, + rateLimitState = rateLimit, + ) => jsonWithRateLimitHeaders(body, init, rateLimitState, requestId); + + const speechApiBucket = process.env.R2_SPEECH_API_BUCKET_NAME; + if (!speechApiBucket) { + await log({ status: 500, errorCode: 'missing_r2_bucket_config' }); + return respond( + createApiError({ + message: 'Storage is not configured. Please contact support.', + type: 'server_error', + code: 'server_error', + }), + { status: 500 }, + ); + } + + const userId = authResult.userId; + + try { + const payload = await request.json(); + + const parsed = VoiceCloneRequestSchema.safeParse(payload); + if (!parsed.success) { + await log({ + status: 400, + errorCode: 'validation_error', + error: parsed.error.message, + userId, + apiKeyId: authResult.apiKeyId, + }); + return respond(zodErrorToApiError(parsed.error), { status: 400 }); + } + + const { input, enhance_reference_audio } = parsed.data; + const locale = parsed.data.locale ?? 'en'; + const enhancementEnabled = enhance_reference_audio === true; + + // Validate locale early (cheap, before any I/O). + validateLocale(locale); + const provider = resolveCloneProvider(locale); + + const userHasPaid = await hasUserPaidAdmin(userId); + if (isCloneTextOverLimit({ text: input, locale, userHasPaid })) { + const maxLength = getCloneTextMaxLength(locale, userHasPaid); + await log({ + status: 400, + errorCode: 'input_too_long', + userId, + apiKeyId: authResult.apiKeyId, + textLength: input.length, + }); + return respond( + createApiError({ + message: `The input text exceeds the maximum length of ${maxLength} characters`, + type: 'invalid_request_error', + code: 'input_too_long', + param: 'input', + }), + { status: 400 }, + ); + } + + const reference = await resolveReferenceAudio(parsed.data); + + const processed = await processCloneReferenceAudio({ + buffer: reference.buffer, + mimeType: reference.mimeType, + filename: reference.filename, + locale, + enhancementEnabled, + }); + + let cloneBuffer = processed.buffer; + let cloneMimeType = processed.mimeType; + let cloneAudioHash = processed.audioHash; + let cloneDuration = processed.duration; + + const baseCloneCredits = estimateCredits(input, 'clone'); + let creditsUsed = baseCloneCredits; + let referenceAudioEnhanced = false; + let enhancementCredits = 0; + let enhancementDollarAmount = 0; + let enhancementDurationSeconds: number | null = null; + let enhancementModelUsed: string | null = null; + let enhancementRequestId: string | null = null; + + // Reference audio must satisfy the provider's minimum length before we + // spend anything (credits or a paid enhancement call). + validateAudioDuration(processed.duration, provider); + + if (enhancementEnabled) { + if ( + processed.duration !== null && + processed.duration > REFERENCE_AUDIO_ENHANCEMENT_MAX_DURATION + ) { + await log({ + status: 400, + errorCode: 'reference_audio_too_long', + userId, + apiKeyId: authResult.apiKeyId, + textLength: input.length, + }); + return respond( + createApiError({ + message: `Reference audio enhancement supports clips up to ${REFERENCE_AUDIO_ENHANCEMENT_MAX_DURATION} seconds`, + type: 'invalid_request_error', + code: 'reference_audio_too_long', + param: 'reference_audio', + }), + { status: 400 }, + ); + } + if ( + processed.buffer.length > REFERENCE_AUDIO_ENHANCEMENT_MAX_INPUT_BYTES + ) { + await log({ + status: 400, + errorCode: 'reference_audio_too_large', + userId, + apiKeyId: authResult.apiKeyId, + textLength: input.length, + }); + return respond( + createApiError({ + message: 'Reference audio enhancement input exceeds size limit', + type: 'invalid_request_error', + code: 'reference_audio_too_large', + param: 'reference_audio', + }), + { status: 400 }, + ); + } + + enhancementDurationSeconds = processed.duration; + enhancementCredits = calculateReferenceAudioEnhancementCredits( + enhancementDurationSeconds, + ); + enhancementDollarAmount = getReferenceAudioEnhancementDollarCost( + enhancementDurationSeconds, + ); + // Provisionally bill the enhancement; affordability is verified below + // before the paid enhancement call is made. + creditsUsed = baseCloneCredits + enhancementCredits; + } + + // Verify the user can afford the maximum cost BEFORE invoking any paid + // provider call (including the expensive enhancement step). This prevents a + // user with insufficient credits from forcing billable work that then 402s. + const currentCredits = await getCreditsAdmin(userId); + if (currentCredits < creditsUsed) { + await log({ + status: 402, + errorCode: 'insufficient_credits', + userId, + apiKeyId: authResult.apiKeyId, + textLength: input.length, + }); + return respond( + createApiError({ + message: 'Insufficient credits', + type: 'permission_error', + code: 'insufficient_credits', + }), + { status: 402 }, + ); + } + + if (enhancementEnabled) { + try { + const enhanced = await enhanceReferenceAudio({ + abortSignal: request.signal, + buffer: processed.buffer, + filename: reference.filename, + mimeType: processed.mimeType, + }); + cloneBuffer = enhanced.buffer; + cloneMimeType = enhanced.mimeType; + cloneAudioHash = generateBufferHash(enhanced.buffer); + cloneDuration = + (await getAudioDuration(enhanced.buffer, enhanced.mimeType)) ?? + cloneDuration; + referenceAudioEnhanced = true; + enhancementModelUsed = enhanced.modelUsed; + enhancementRequestId = enhanced.requestId; + } catch (enhancementError) { + // Enhancement is best-effort: fall back to the original reference audio + // and only bill the base clone credits. + captureException(enhancementError, { + extra: { requestId, endpoint: ENDPOINT, locale }, + }); + enhancementCredits = 0; + enhancementDollarAmount = 0; + enhancementDurationSeconds = null; + creditsUsed = baseCloneCredits; + } + } + + const folder = userHasPaid ? 'generated-audio' : 'generated-audio-free'; + const filename = `${folder}/clone-${cloneAudioHash.slice(0, 16)}-${Date.now()}.wav`; + + let modelUsed: string; + let predictionId: string; + let outputBuffer: Buffer; + + if (provider === 'mistral') { + const result = await generateVoiceWithMistral(input, cloneBuffer); + outputBuffer = result.buffer; + modelUsed = result.modelUsed; + predictionId = result.requestId; + } else { + // Replicate needs a public URL to the reference audio. + const refFilename = sanitizeFilename(reference.filename); + const refKey = `clone-voice-input-api/${userId}-${cloneAudioHash}-${refFilename}`; + const referenceAudioUrl = await uploadFileToR2( + refKey, + cloneBuffer, + cloneMimeType, + speechApiBucket, + process.env.R2_SPEECH_API_PUBLIC_URL, + ); + const result = await cloneVoiceWithReplicate( + input, + locale, + referenceAudioUrl, + ); + outputBuffer = Buffer.from(await result.blob.arrayBuffer()); + modelUsed = result.modelUsed; + predictionId = result.requestId; + } + + const uploadUrl = await uploadFileToR2( + filename, + outputBuffer, + 'audio/wav', + speechApiBucket, + process.env.R2_SPEECH_API_PUBLIC_URL, + ); + + if (!uploadUrl) { + throw new Error('uploadUrl is empty after cloning — this is a bug'); + } + + await reduceCreditsAdmin({ userId, amount: creditsUsed }); + + const dollarAmount = calculateGenerateApiDollarAmount({ + sourceType: 'api_voice_cloning', + provider, + inputChars: input.length, + }); + + const audioFileResult = await saveAudioFileAdmin({ + userId, + filename, + text: input, + url: uploadUrl, + model: modelUsed, + predictionId, + isPublic: false, + voiceId: CLONE_VOICE_ID, + duration: (cloneDuration ?? -1).toFixed(3), + credits_used: creditsUsed, + usage: { + userHasPaid, + apiKeyId: authResult.apiKeyId, + sourceType: 'api_voice_cloning', + dollarAmount, + referenceAudioEnhanced, + }, + }); + + await insertUsageEvent({ + userId, + sourceType: 'api_voice_cloning', + sourceId: audioFileResult.data?.id ?? null, + apiKeyId: authResult.apiKeyId, + model: modelUsed, + inputChars: input.length, + unit: 'operation', + quantity: 1, + creditsUsed: baseCloneCredits, + dollarAmount, + requestId, + metadata: { + provider, + model: modelUsed, + locale, + textPreview: input.slice(0, 100), + textLength: input.length, + audioDuration: cloneDuration, + referenceAudioEnhanced, + predictionId, + userHasPaid, + }, + }); + + if (referenceAudioEnhanced && enhancementCredits > 0) { + await insertUsageEvent({ + userId, + sourceType: 'audio_processing', + sourceId: audioFileResult.data?.id ?? null, + apiKeyId: authResult.apiKeyId, + requestId: enhancementRequestId ?? undefined, + model: enhancementModelUsed ?? undefined, + unit: 'secs', + quantity: enhancementDurationSeconds ?? 0, + durationSeconds: enhancementDurationSeconds ?? 0, + creditsUsed: enhancementCredits, + dollarAmount: enhancementDollarAmount, + metadata: { + operation: 'reference_audio_enhancement', + provider: 'fal', + model: enhancementModelUsed, + voiceCloningModel: modelUsed, + locale, + userHasPaid, + }, + }); + } + + // Fire-and-forget: do not await the log on the success path. A flush failure + // must never return a 500 after audio has been generated and credits have + // already been deducted. (Mirrors /api/v1/speech.) + log({ + status: 200, + userId, + apiKeyId: authResult.apiKeyId, + model: modelUsed, + textLength: input.length, + provider, + creditsUsed, + dollarAmount, + referenceAudioEnhanced, + userHasPaid, + }).catch((err) => { + console.error('[clone] success-path log failed:', err); + }); + + return respond( + { + url: uploadUrl, + credits_used: creditsUsed, + credits_remaining: Math.max(0, currentCredits - creditsUsed), + usage: { + input_characters: input.length, + model: modelUsed, + }, + }, + { status: 200 }, + ); + } catch (error) { + if (error instanceof CloneServiceError) { + const mapping = CLONE_ERROR_MAP[error.code]; + // Prefer a more specific param from the error (e.g. reference_audio_url) + // so the failure is attributed to the field the caller actually sent. + const detailParam = + typeof error.details?.param === 'string' ? error.details.param : null; + await log({ + status: mapping.status, + errorCode: error.code, + error: error.message, + userId, + apiKeyId: authResult.apiKeyId, + }); + return respond( + createApiError({ + message: error.message, + type: mapping.type, + code: mapping.code, + param: detailParam ?? mapping.param ?? null, + }), + { status: mapping.status }, + ); + } + + if (error instanceof SyntaxError) { + await log({ + status: 400, + errorCode: 'invalid_json', + error: error.message, + userId, + apiKeyId: authResult.apiKeyId, + }); + return externalApiErrorResponse({ + key: 'invalid_json', + rateLimit, + requestId, + }); + } + + captureException(error, { + extra: { + requestId, + endpoint: ENDPOINT, + userId, + apiKeyId: authResult.apiKeyId, + }, + }); + await log({ + status: 500, + errorCode: 'server_error', + error: error instanceof Error ? error.message : String(error), + userId, + apiKeyId: authResult.apiKeyId, + }); + return externalApiErrorResponse({ + key: 'server_error', + rateLimit, + requestId, + }); + } finally { + await updateApiKeyLastUsed(authResult.keyHash); + } +} + +export const runtime = 'nodejs'; diff --git a/apps/web/lib/api/logger.ts b/apps/web/lib/api/logger.ts index 403e50eba..a59bc1fd2 100644 --- a/apps/web/lib/api/logger.ts +++ b/apps/web/lib/api/logger.ts @@ -19,6 +19,7 @@ export interface LogFields { provider?: string; providerCode?: number; providerStatus?: string; + referenceAudioEnhanced?: boolean; status: number; textLength?: number; userHasPaid?: boolean; diff --git a/apps/web/lib/api/openapi.ts b/apps/web/lib/api/openapi.ts index 5ee3eae47..4d6257147 100644 --- a/apps/web/lib/api/openapi.ts +++ b/apps/web/lib/api/openapi.ts @@ -4,6 +4,8 @@ import { BillingResponseSchema, ErrorResponseSchema, ModelsResponseSchema, + VoiceCloneRequestSchema, + VoiceCloneResponseSchema, VoiceGenerationRequestSchema, VoiceGenerationResponseSchema, VoicesResponseSchema, @@ -177,8 +179,8 @@ export function createExternalApiOpenApiDocument() { openapi: '3.1.0', info: { title: 'SexyVoice API', - version: '1.0.1', - description: 'API for text-to-speech generation.', + version: '1.1.0', + description: 'API for text-to-speech generation and voice cloning.', }, components: { securitySchemes: { @@ -191,6 +193,8 @@ export function createExternalApiOpenApiDocument() { schemas: { VoiceGenerationRequest: VoiceGenerationRequestSchema, VoiceGenerationResponse: VoiceGenerationResponseSchema, + VoiceCloneRequest: VoiceCloneRequestSchema, + VoiceCloneResponse: VoiceCloneResponseSchema, ErrorResponse: ErrorResponseSchema, VoicesResponse: VoicesResponseSchema, ModelsResponse: ModelsResponseSchema, @@ -330,6 +334,102 @@ export function createExternalApiOpenApiDocument() { }, }, }, + '/api/v1/clone': { + post: { + security: [{ BearerAuth: [] }], + summary: 'Clone a voice from reference audio', + requestBody: { + required: true, + content: { + 'application/json': { + schema: VoiceCloneRequestSchema, + examples: { + url: { + summary: 'Clone from a reference audio URL', + value: { + input: 'Hello from the cloned voice!', + locale: 'en', + reference_audio_url: + 'https://example.com/reference-sample.wav', + }, + }, + base64: { + summary: 'Clone from base64-encoded reference audio', + value: { + input: 'Hola desde la voz clonada!', + locale: 'es', + reference_audio: 'UklGRiQAAABXQVZF...', + reference_audio_format: 'audio/wav', + }, + }, + enhanced: { + summary: 'Clone with reference-audio enhancement', + value: { + input: 'Hello with cleaner reference audio.', + locale: 'en', + reference_audio_url: + 'https://example.com/noisy-sample.wav', + enhance_reference_audio: true, + }, + }, + }, + }, + }, + }, + responses: { + 200: { + description: 'Voice cloned', + content: { + 'application/json': { + schema: VoiceCloneResponseSchema, + }, + }, + }, + 400: { + description: 'Bad request', + content: { + 'application/json': { schema: ErrorResponseSchema }, + }, + }, + 401: { + description: 'Authentication failed', + content: { + 'application/json': { schema: ErrorResponseSchema }, + }, + }, + 402: { + description: 'Insufficient credits', + content: { + 'application/json': { schema: ErrorResponseSchema }, + }, + }, + 422: { + description: 'Content policy violation', + content: { + 'application/json': { schema: ErrorResponseSchema }, + }, + }, + 429: { + description: 'Rate limit exceeded', + content: { + 'application/json': { schema: ErrorResponseSchema }, + }, + }, + 500: { + description: 'Server error', + content: { + 'application/json': { schema: ErrorResponseSchema }, + }, + }, + 503: { + description: 'Provider temporarily unavailable', + content: { + 'application/json': { schema: ErrorResponseSchema }, + }, + }, + }, + }, + }, '/api/v1/voices': { get: { security: [{ BearerAuth: [] }], diff --git a/apps/web/lib/api/pricing.ts b/apps/web/lib/api/pricing.ts index 48166255b..6e1d6c4f1 100644 --- a/apps/web/lib/api/pricing.ts +++ b/apps/web/lib/api/pricing.ts @@ -1,5 +1,5 @@ type ExternalSourceType = 'api_tts' | 'api_voice_cloning' | 'tts'; -type ExternalProvider = 'google' | 'replicate' | 'xai'; +type ExternalProvider = 'google' | 'replicate' | 'xai' | 'mistral'; type ExternalModel = 'gpro' | 'orpheus' | string; interface PricingInput { @@ -43,6 +43,21 @@ const PRICING_TABLE: Record = { perInputToken: 0.000_000_5, perOutputToken: 0.000_01, }, + // Voice cloning API + 'api_voice_cloning:mistral:*': { + // Mistral Voxtral — $0.016 per 1k input characters + perInputChar: 0.000_016, + perOutputChar: 0, + perMinute: 0, + perRequest: 0, + }, + 'api_voice_cloning:replicate:*': { + // resemble-ai/chatterbox-multilingual — ~$0.0121 average per prediction + perInputChar: 0, + perOutputChar: 0, + perMinute: 0, + perRequest: 0.0121, + }, 'api_tts:replicate:lucataco/orpheus-3b-0.1-ft:79f2a473e6a9720716a473d9b2f2951437dbf91dc02ccb7079fb3d89b881207f': { perInputChar: 0.000_015, @@ -54,7 +69,6 @@ const PRICING_TABLE: Record = { 'api_tts:xai:xai': { perInputChar: 0.000_015, }, - 'api_voice_cloning:replicate:*': {}, }; function getPriceConfig({ diff --git a/apps/web/lib/api/schemas.ts b/apps/web/lib/api/schemas.ts index 8375a23f1..a3a637764 100644 --- a/apps/web/lib/api/schemas.ts +++ b/apps/web/lib/api/schemas.ts @@ -115,6 +115,84 @@ export const VoiceGenerationRequestSchema = z export const VoiceGenerationRequestOpenApiSchema = VoiceGenerationRequestSchema; +export const VoiceCloneRequestSchema = z + .strictObject({ + input: z + .string() + .min(1) + .max(4000) + .describe( + 'The text to synthesize with the cloned voice (max 1000 chars on the free tier, 4000 for paid Voxtral locales, 300 for other languages)', + ), + locale: z + .string() + .min(2) + .max(10) + .optional() + .describe( + 'Language of the input text (e.g. "en", "es", "fr"). Defaults to "en". Determines the cloning provider.', + ), + reference_audio_url: z + .string() + .url() + .optional() + .describe( + 'Public URL to the reference audio to clone (MP3, WAV, OGG/Opus). 3-25s recommended.', + ), + reference_audio: z + .string() + .min(1) + .optional() + .describe( + 'Base64-encoded reference audio (alternative to reference_audio_url).', + ), + reference_audio_format: z + .string() + .optional() + .describe( + 'MIME type of the base64 reference_audio (e.g. "audio/wav", "audio/mpeg"). Defaults to audio/wav.', + ), + enhance_reference_audio: z + .boolean() + .optional() + .describe( + 'Denoise/enhance the reference audio before cloning. Bills additional credits per second.', + ), + response_format: z + .enum(['wav']) + .optional() + .describe( + 'Audio format of the generated output. Only "wav" is supported.', + ), + }) + .refine( + (data) => + Boolean(data.reference_audio_url) !== Boolean(data.reference_audio), + { + message: + 'Provide exactly one of "reference_audio_url" or "reference_audio"', + path: ['reference_audio_url'], + }, + ); + +export const VoiceCloneResponseSchema = z.object({ + url: z.string().url().describe('URL to the generated cloned-voice audio'), + credits_used: z + .number() + .int() + .nonnegative() + .describe('Credits consumed for this clone'), + credits_remaining: z + .number() + .int() + .nonnegative() + .describe('Remaining credits'), + usage: z.object({ + input_characters: z.number().int().describe('Input characters processed'), + model: z.string().describe('Cloning model used'), + }), +}); + export const VoiceGenerationResponseSchema = z.object({ url: z.url().describe('URL to generated audio'), credits_used: z diff --git a/apps/web/lib/clone/clone-service.ts b/apps/web/lib/clone/clone-service.ts new file mode 100644 index 000000000..c04b4d887 --- /dev/null +++ b/apps/web/lib/clone/clone-service.ts @@ -0,0 +1,526 @@ +import { createHash, randomUUID } from 'node:crypto'; +import { Mistral } from '@mistralai/mistralai'; +import { logger } from '@sentry/nextjs'; +import { parseBuffer } from 'music-metadata'; +import Replicate, { type Prediction } from 'replicate'; + +import { + AudioDecodeError, + convertToWav, + isConversionSupported, + needsConversion, + trimWavBuffer, +} from '@/lib/audio-converter'; +import { + type CloneProvider, + VOXTRAL_SUPPORTED_LOCALE_CODES, +} from '@/lib/clone/constants'; + +// ============================================================================ +// Constants (shared with the internal /api/clone-voice route) +// ============================================================================ + +export const FALLBACK_MIN_DURATION = 10; +export const REPLICATE_REFERENCE_AUDIO_MAX_DURATION = 10; +export const VOXTRAL_MIN_DURATION = 3; +export const VOXTRAL_MAX_DURATION = 25; +export const REFERENCE_AUDIO_ENHANCEMENT_MAX_DURATION = 60; +export const REFERENCE_AUDIO_ENHANCEMENT_MAX_INPUT_BYTES = 25 * 1024 * 1024; +export const REFERENCE_AUDIO_ENHANCEMENT_CREDITS_PER_SECOND = 10; +export const REFERENCE_AUDIO_ENHANCEMENT_DOLLARS_PER_SECOND = 0.001; + +// Replicate multilingual supports the following languages +// https://replicate.com/resemble-ai/chatterbox-multilingual/api/schema +export const SUPPORTED_LOCALE_CODES = [ + { code: 'ar', value: 'arabic' }, + { code: 'da', value: 'danish' }, + { code: 'de', value: 'german' }, + { code: 'el', value: 'greek' }, + { code: 'en', value: 'english' }, + { code: 'en-multi', value: 'english' }, + { code: 'es', value: 'spanish' }, + { code: 'fi', value: 'finnish' }, + { code: 'fr', value: 'french' }, + { code: 'he', value: 'hebrew' }, + { code: 'hi', value: 'hindi' }, + { code: 'it', value: 'italian' }, + { code: 'ja', value: 'japanese' }, + { code: 'ko', value: 'korean' }, + { code: 'ms', value: 'malay' }, + { code: 'nl', value: 'dutch' }, + { code: 'no', value: 'norwegian' }, + { code: 'pl', value: 'polish' }, + { code: 'pt', value: 'portuguese' }, + { code: 'ru', value: 'russian' }, + { code: 'sv', value: 'swedish' }, + { code: 'sw', value: 'swahili' }, + { code: 'tr', value: 'turkish' }, + { code: 'zh', value: 'chinese' }, +]; + +// ============================================================================ +// Typed errors — provider/validation failures map to API errors in the route. +// Kept free of any HTTP/i18n coupling so the module stays reusable. +// ============================================================================ + +export type CloneServiceErrorCode = + | 'unsupported_locale' + | 'audio_duration_unknown' + | 'audio_duration_too_short' + | 'audio_conversion_failed' + | 'unsupported_audio_format' + | 'guardrail_violation' + | 'provider_unavailable'; + +export class CloneServiceError extends Error { + code: CloneServiceErrorCode; + details?: Record; + + constructor( + code: CloneServiceErrorCode, + message: string, + details?: Record, + ) { + super(message); + this.name = 'CloneServiceError'; + this.code = code; + this.details = details; + } +} + +interface CloneProviderConstraints { + minDurationSeconds: number; +} + +export interface ProcessedCloneInputAudio { + audioHash: string; + buffer: Buffer; + duration: number | null; + mimeType: string; + originalDuration: number | null; + wasTrimmed: boolean; +} + +interface MistralSdkErrorLike { + body?: unknown; + message?: unknown; + statusCode?: unknown; +} + +// ============================================================================ +// Provider / locale resolution +// ============================================================================ + +export function isVoxtralCloneLocale(locale: string): boolean { + return VOXTRAL_SUPPORTED_LOCALE_CODES.has(locale); +} + +export function resolveCloneProvider(locale: string): CloneProvider { + return isVoxtralCloneLocale(locale) ? 'mistral' : 'replicate'; +} + +export function getCloneProviderConstraints( + provider: CloneProvider, +): CloneProviderConstraints { + if (provider === 'mistral') { + return { minDurationSeconds: VOXTRAL_MIN_DURATION }; + } + return { minDurationSeconds: FALLBACK_MIN_DURATION }; +} + +export function validateLocale(locale: string): void { + const localeConfig = SUPPORTED_LOCALE_CODES.find((l) => l.code === locale); + if (!localeConfig) { + throw new CloneServiceError( + 'unsupported_locale', + `Unsupported language for voice cloning: ${locale}. Supported languages are: ${SUPPORTED_LOCALE_CODES.map((l) => l.code).join(', ')}`, + { locale }, + ); + } +} + +export function validateAudioDuration( + duration: number | null, + provider: CloneProvider, +): void { + if (duration === null) { + throw new CloneServiceError( + 'audio_duration_unknown', + 'Could not determine audio duration.', + ); + } + + const constraints = getCloneProviderConstraints(provider); + if (duration < constraints.minDurationSeconds) { + throw new CloneServiceError( + 'audio_duration_too_short', + `Reference audio must be at least ${constraints.minDurationSeconds} seconds for voice cloning.`, + { MIN: constraints.minDurationSeconds }, + ); + } +} + +// ============================================================================ +// Enhancement billing helpers +// ============================================================================ + +export function calculateReferenceAudioEnhancementCredits( + durationSeconds: number | null, +): number { + if (durationSeconds === null) { + throw new CloneServiceError( + 'audio_duration_unknown', + 'Could not determine audio duration.', + ); + } + return Math.ceil( + Math.max(0, durationSeconds) * + REFERENCE_AUDIO_ENHANCEMENT_CREDITS_PER_SECOND, + ); +} + +export function getReferenceAudioEnhancementDollarCost( + durationSeconds: number | null, +): number { + if (durationSeconds === null) { + return 0; + } + return ( + Math.max(0, durationSeconds) * + REFERENCE_AUDIO_ENHANCEMENT_DOLLARS_PER_SECOND + ); +} + +// ============================================================================ +// Audio utilities +// ============================================================================ + +export function sanitizeFilename(filename: string): string { + return filename + .normalize('NFD') + .replace(/[\u0300-\u036f]/g, '') // Remove diacritics + .replace(/[^a-zA-Z0-9.-]/g, '_'); // Replace special chars with underscore +} + +export function generateBufferHash(buffer: Buffer): string { + // Use Node's optimized, synchronous SHA-256 (node:crypto is already imported) + // instead of the Web Crypto API + manual hex formatting. + return createHash('sha256').update(buffer).digest('hex'); +} + +export async function getAudioDuration( + fileBuffer: Buffer, + mimeType: string, +): Promise { + try { + const metadata = await parseBuffer( + fileBuffer, + { mimeType, size: fileBuffer.length }, + { duration: true }, + ); + return metadata.format.duration ?? null; + } catch { + return null; + } +} + +/** + * Normalize a reference-audio buffer for voice cloning: convert to WAV when the + * provider needs it, trim to the provider's maximum reference duration, and + * report the resulting duration. + * + * This is the buffer-based counterpart of the internal route's `processAudioFile` + * (which operates on a `File`), so the external API can feed it either a fetched + * URL or a decoded base64 payload. + */ +// biome-ignore lint/complexity/noExcessiveCognitiveComplexity: Mirrors the internal route's audio normalization (convert + trim + duration). +export async function processCloneReferenceAudio({ + buffer, + mimeType, + filename, + locale, + enhancementEnabled, +}: { + buffer: Buffer; + mimeType: string; + filename: string; + locale: string; + enhancementEnabled: boolean; +}): Promise { + const normalizedMimeType = mimeType.split(';')[0]?.trim().toLowerCase() || ''; + const provider = resolveCloneProvider(locale); + + let processedBuffer: Buffer = buffer; + let processedMimeType = normalizedMimeType; + let sourceDuration = await getAudioDuration(buffer, normalizedMimeType); + let wasTrimmed = false; + + const canNormalizeToWav = isConversionSupported(normalizedMimeType, filename); + const shouldNormalizeToWav = + provider === 'mistral' || enhancementEnabled || canNormalizeToWav; + + if (shouldNormalizeToWav && needsConversion(normalizedMimeType)) { + if (!isConversionSupported(normalizedMimeType, filename)) { + throw new CloneServiceError( + 'unsupported_audio_format', + 'Unsupported audio format for voice cloning. Please use MP3, OGG/OPUS, or WAV.', + { mimeType: normalizedMimeType }, + ); + } + + try { + const wavBuffer = await convertToWav( + buffer, + normalizedMimeType, + filename, + ); + if (wavBuffer) { + processedBuffer = wavBuffer as Buffer; + processedMimeType = 'audio/wav'; + } + } catch (conversionError) { + if (!(conversionError instanceof AudioDecodeError)) { + logger.warn('Clone API audio conversion failed', { + extra: { + error: + conversionError instanceof Error + ? conversionError.message + : String(conversionError), + normalizedMimeType, + locale, + filename, + }, + }); + } + const isWebm = + normalizedMimeType === 'audio/webm' || + normalizedMimeType === 'video/webm'; + throw new CloneServiceError( + 'audio_conversion_failed', + isWebm + ? 'WebM audio must be converted to WAV before sending. Please upload MP3, OGG, Opus, or WAV reference audio.' + : 'Failed to convert audio format to WAV. Reference audio must be MP3, OGG, Opus, or WAV.', + { mimeType: normalizedMimeType }, + ); + } + } + + let duration = await getAudioDuration(processedBuffer, processedMimeType); + sourceDuration ??= duration; + + const referenceAudioMaxDuration = + provider === 'mistral' + ? VOXTRAL_MAX_DURATION + : REPLICATE_REFERENCE_AUDIO_MAX_DURATION; + + if (duration !== null && duration > referenceAudioMaxDuration) { + const trimmedBuffer = trimWavBuffer( + processedBuffer, + referenceAudioMaxDuration, + ); + + if (trimmedBuffer && trimmedBuffer !== processedBuffer) { + processedBuffer = trimmedBuffer; + processedMimeType = 'audio/wav'; + duration = + (await getAudioDuration(processedBuffer, processedMimeType)) ?? + referenceAudioMaxDuration; + wasTrimmed = true; + } else if (provider === 'mistral') { + throw new CloneServiceError( + 'audio_conversion_failed', + 'Failed to trim reference audio for voice cloning.', + ); + } + } + + const audioHash = generateBufferHash(processedBuffer); + + return { + audioHash, + buffer: processedBuffer, + mimeType: processedMimeType, + duration, + originalDuration: sourceDuration, + wasTrimmed, + }; +} + +// ============================================================================ +// Voice cloning providers +// ============================================================================ + +let mistralClient: Mistral | null = null; + +function getMistralClient(): Mistral { + const apiKey = process.env.MISTRAL_API_KEY; + if (!apiKey) { + throw new Error('MISTRAL_API_KEY is not configured'); + } + if (!mistralClient) { + mistralClient = new Mistral({ apiKey }); + } + return mistralClient; +} + +function getMistralErrorPayload(error: unknown): { + code?: string; + message?: string; + rawStatusCode?: number; + statusCode?: number; + type?: string; +} { + if (!(error && typeof error === 'object')) { + return {}; + } + + const sdkError = error as MistralSdkErrorLike; + const statusCode = + typeof sdkError.statusCode === 'number' ? sdkError.statusCode : undefined; + const body = typeof sdkError.body === 'string' ? sdkError.body : undefined; + const message = + typeof sdkError.message === 'string' ? sdkError.message : undefined; + + if (!body) { + return { message, statusCode }; + } + + try { + const parsed = JSON.parse(body) as Record; + return { + code: typeof parsed.code === 'string' ? parsed.code : undefined, + message: typeof parsed.message === 'string' ? parsed.message : message, + rawStatusCode: + typeof parsed.raw_status_code === 'number' + ? parsed.raw_status_code + : undefined, + statusCode, + type: typeof parsed.type === 'string' ? parsed.type : undefined, + }; + } catch { + return { message, statusCode }; + } +} + +function isMistralGuardrailError(error: unknown): boolean { + const payload = getMistralErrorPayload(error); + const statusCode = payload.statusCode ?? payload.rawStatusCode; + return ( + statusCode === 403 && + (payload.type === 'guardrail_violation' || + payload.code === '1920' || + payload.message?.toLowerCase().includes('guardrail') === true) + ); +} + +export async function generateVoiceWithMistral( + text: string, + referenceAudioBuffer: Buffer, +): Promise<{ buffer: Buffer; modelUsed: string; requestId: string }> { + const model = 'voxtral-mini-tts-2603'; + const client = getMistralClient(); + + let response: Awaited>; + try { + response = await client.audio.speech.complete({ + model, + input: text, + refAudio: referenceAudioBuffer.toString('base64'), + responseFormat: 'wav', + }); + } catch (error) { + if (isMistralGuardrailError(error)) { + throw new CloneServiceError( + 'guardrail_violation', + 'This request was blocked by a third-party voice cloning safety policy. Please try different text or a different reference audio.', + { provider: 'mistral' }, + ); + } + throw error; + } + + const audioData = response.audioData; + if (!audioData) { + throw new Error('Mistral Voxtral response did not include audio data'); + } + + const buffer = + typeof audioData === 'string' + ? Buffer.from(audioData, 'base64') + : Buffer.from(audioData); + + if (buffer.length === 0) { + throw new Error('Mistral Voxtral response returned empty audio data'); + } + + const riffHeader = buffer.subarray(0, 4).toString('ascii'); + const waveHeader = buffer.subarray(8, 12).toString('ascii'); + if (riffHeader !== 'RIFF' || waveHeader !== 'WAVE') { + throw new Error('Mistral Voxtral response did not return a valid WAV file'); + } + + return { buffer, modelUsed: model, requestId: randomUUID() }; +} + +interface ReplicateOutput { + blob: () => Promise; + url: () => string; +} + +export async function cloneVoiceWithReplicate( + text: string, + locale: string, + audioReferenceUrl: string, +): Promise<{ blob: Blob; modelUsed: string; requestId: string }> { + const localeConfig = SUPPORTED_LOCALE_CODES.find((l) => l.code === locale); + if (!localeConfig) { + throw new CloneServiceError( + 'unsupported_locale', + `Unsupported locale: ${locale}`, + { locale }, + ); + } + + const language = locale === 'en-multi' ? 'en' : locale; + + const replicate = new Replicate(); + let replicateResponse: Prediction | undefined; + const onProgress = (prediction: Prediction) => { + replicateResponse = prediction; + }; + + const model = + 'resemble-ai/chatterbox-multilingual:9cfba4c265e685f840612be835424f8c33bdee685d7466ece7684b0d9d4c0b1c' as `${string}/${string}`; + const input = { + seed: 0, + text, + language, + cfg_weight: 0.5, + temperature: 0.8, + exaggeration: 0.5, + reference_audio: audioReferenceUrl, + }; + + const output = (await replicate.run(model, { input }, onProgress)) as + | ReplicateOutput + | { error?: string }; + + if (output && typeof output === 'object' && 'error' in output) { + logger.warn('Replicate voice cloning provider failed', { + extra: { locale, language, model, errorMessage: output.error || null }, + }); + throw new CloneServiceError( + 'provider_unavailable', + 'Voice cloning provider is temporarily unavailable. Please try again.', + { provider: 'replicate' }, + ); + } + + const audioBlob = await (output as ReplicateOutput).blob(); + + return { + blob: audioBlob, + modelUsed: model.split(':')[0], + requestId: replicateResponse?.id || 'unknown', + }; +} diff --git a/apps/web/messages/da.json b/apps/web/messages/da.json index 5af30a5ca..b70f21a72 100644 --- a/apps/web/messages/da.json +++ b/apps/web/messages/da.json @@ -762,6 +762,8 @@ "title": "Hvad kan du gøre med SexyVoice API?", "generateTitle": "Generer", "generateDescription": "Konverter tekst til naturlig tale med AI-stemmer. Vælg mellem flere sprog og tilføj følelsesmæssigt udtryk (på understøttede stemmer).", + "cloneTitle": "Klon", + "cloneDescription": "Klon en stemme ud fra en kort referenceoptagelse, og generer derefter ny tale med den på mere end 20 sprog via API'et.", "cliTitle": "CLI", "cliDescription": "Brug SexyVoice CLI til browserbaseret login, stemmeopdagelse, talegenerering, curl-output og lokal afspilning direkte fra din terminal.", "moreInfo": "Mere info:", diff --git a/apps/web/messages/de.json b/apps/web/messages/de.json index 2361f071e..a90b957dc 100644 --- a/apps/web/messages/de.json +++ b/apps/web/messages/de.json @@ -762,6 +762,8 @@ "title": "Was können Sie mit der SexyVoice API machen?", "generateTitle": "Generieren", "generateDescription": "Wandeln Sie Text mit KI-Stimmen in natürlich klingende Sprache um. Wählen Sie aus mehreren Sprachen und fügen Sie emotionale Ausdrucksformen hinzu (bei unterstützten Stimmen).", + "cloneTitle": "Klonen", + "cloneDescription": "Klonen Sie eine Stimme aus einer kurzen Referenzaufnahme und erzeugen Sie damit über die API neue Sprache in mehr als 20 Sprachen.", "cliTitle": "CLI", "cliDescription": "Verwenden Sie die SexyVoice-CLI für browserbasierten Login, Stimmenübersicht, Sprachgenerierung, Curl-Ausgabe und lokale Wiedergabe direkt im Terminal.", "moreInfo": "Mehr Infos:", diff --git a/apps/web/messages/en.json b/apps/web/messages/en.json index 083c4f76b..f363c6c64 100644 --- a/apps/web/messages/en.json +++ b/apps/web/messages/en.json @@ -295,9 +295,7 @@ "description": "For bigger creative needs and bolder content.", "creditsWithAudio": "{numCredits} credits (~125 minutes of audio)", "creditsWithPromo": "{numCredits} credits", - "features": [ - "Everything in Standard, just cheaper" - ] + "features": ["Everything in Standard, just cheaper"] } }, "subscriptionScheduledToCancel": { @@ -762,6 +760,8 @@ "title": "What can you do with SexyVoice API?", "generateTitle": "Generate", "generateDescription": "Convert text to natural speech with AI voices. Choose from multiple languages and add emotional expression (on supported voices).", + "cloneTitle": "Clone", + "cloneDescription": "Clone a voice from a short reference recording, then synthesize new speech with it in 20+ languages via the API.", "cliTitle": "CLI", "cliDescription": "Use the SexyVoice CLI for browser-based login, voice discovery, speech generation, curl output, and local playback from your terminal.", "moreInfo": "More info:", diff --git a/apps/web/messages/es.json b/apps/web/messages/es.json index 18c619fa4..45c99da4e 100644 --- a/apps/web/messages/es.json +++ b/apps/web/messages/es.json @@ -762,6 +762,8 @@ "title": "¿Qué puedes hacer con la API de SexyVoice?", "generateTitle": "Generar", "generateDescription": "Convierte texto en voz natural con voces de IA. Elige entre varios idiomas y añade expresión emocional (en las voces compatibles).", + "cloneTitle": "Clonar", + "cloneDescription": "Clona una voz a partir de una breve grabación de referencia y luego sintetiza nueva voz con ella en más de 20 idiomas a través de la API.", "cliTitle": "CLI", "cliDescription": "Usa la CLI de SexyVoice para iniciar sesión desde el navegador, descubrir voces, generar audio, obtener comandos curl y reproducir audio localmente desde tu terminal.", "moreInfo": "Más información:", diff --git a/apps/web/messages/fr.json b/apps/web/messages/fr.json index 568f59f57..a349002e8 100644 --- a/apps/web/messages/fr.json +++ b/apps/web/messages/fr.json @@ -762,6 +762,8 @@ "title": "Que pouvez-vous faire avec l'API SexyVoice ?", "generateTitle": "Générer", "generateDescription": "Convertissez du texte en parole naturelle avec des voix IA. Choisissez parmi plusieurs langues et ajoutez de l'expression émotionnelle (sur les voix compatibles).", + "cloneTitle": "Cloner", + "cloneDescription": "Clonez une voix à partir d'un court enregistrement de référence, puis synthétisez une nouvelle parole avec elle dans plus de 20 langues via l'API.", "cliTitle": "CLI", "cliDescription": "Utilisez la CLI SexyVoice pour vous connecter via le navigateur, découvrir les voix, générer de l'audio, produire des commandes curl et lire localement depuis votre terminal.", "moreInfo": "Plus d'infos :", diff --git a/apps/web/messages/it.json b/apps/web/messages/it.json index 451272d96..ba99026e7 100644 --- a/apps/web/messages/it.json +++ b/apps/web/messages/it.json @@ -762,6 +762,8 @@ "title": "Cosa puoi fare con l'API di SexyVoice?", "generateTitle": "Genera", "generateDescription": "Converti il testo in parlato naturale con voci AI. Scegli tra più lingue e aggiungi espressività emotiva (nelle voci supportate).", + "cloneTitle": "Clona", + "cloneDescription": "Clona una voce da una breve registrazione di riferimento, quindi sintetizza nuovo parlato con essa in oltre 20 lingue tramite l'API.", "cliTitle": "CLI", "cliDescription": "Usa la CLI di SexyVoice per il login dal browser, la scoperta delle voci, la generazione del parlato, l'output curl e la riproduzione locale dal terminale.", "moreInfo": "Più info:", diff --git a/apps/web/tests/api-pricing.test.ts b/apps/web/tests/api-pricing.test.ts index 4df45cbb5..a49b0cebe 100644 --- a/apps/web/tests/api-pricing.test.ts +++ b/apps/web/tests/api-pricing.test.ts @@ -82,12 +82,21 @@ describe('external API pricing', () => { expect(amount).toBe(0); }); - it('supports api_voice_cloning pricing key (currently zero)', () => { + it('charges a per-request price for replicate voice cloning', () => { const amount = calculateGenerateApiDollarAmount({ sourceType: 'api_voice_cloning', provider: 'replicate', model: 'future-model', }); - expect(amount).toBe(0); + expect(amount).toBe(0.0121); + }); + + it('charges per input character for mistral voice cloning', () => { + const amount = calculateGenerateApiDollarAmount({ + sourceType: 'api_voice_cloning', + provider: 'mistral', + inputChars: 1000, + }); + expect(amount).toBe(0.016); }); }); diff --git a/apps/web/tests/api-v1-clone.test.ts b/apps/web/tests/api-v1-clone.test.ts new file mode 100644 index 000000000..372656b26 --- /dev/null +++ b/apps/web/tests/api-v1-clone.test.ts @@ -0,0 +1,361 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { POST } from '@/app/api/v1/clone/route'; +import { + getCreditsAdmin, + insertUsageEvent, + reduceCreditsAdmin, + saveAudioFileAdmin, +} from '@/lib/supabase/queries'; +import { + flushPromises, + mockFalSubscribe, + mockMistralSpeechComplete, + mockParseBuffer, + mockRatelimitLimit, + mockReplicateRun, + mockUploadFileToR2, +} from './setup'; + +const TEST_API_KEY = 'sk_live_Abc123Def456Ghi789Jkl012Mno345Pq'; +const TEST_AUTH_HEADER = `Bearer ${TEST_API_KEY}`; + +// Build a minimal-but-valid WAV buffer (RIFF/WAVE with fmt + data chunks). +const createWavBase64 = (dataSize = 1024): string => { + const buffer = Buffer.alloc(44 + dataSize); + buffer.write('RIFF', 0, 'ascii'); + buffer.writeUInt32LE(36 + dataSize, 4); + buffer.write('WAVE', 8, 'ascii'); + buffer.write('fmt ', 12, 'ascii'); + buffer.writeUInt32LE(16, 16); + buffer.writeUInt16LE(1, 20); // PCM + buffer.writeUInt16LE(1, 22); // mono + buffer.writeUInt32LE(44_100, 24); + buffer.writeUInt32LE(88_200, 28); + buffer.writeUInt16LE(2, 32); + buffer.writeUInt16LE(16, 34); + buffer.write('data', 36, 'ascii'); + buffer.writeUInt32LE(dataSize, 40); + return buffer.toString('base64'); +}; + +const cloneRequest = (body: unknown, headers: Record = {}) => + new Request('http://localhost/api/v1/clone', { + method: 'POST', + headers: { + 'content-type': 'application/json', + authorization: TEST_AUTH_HEADER, + ...headers, + }, + body: typeof body === 'string' ? body : JSON.stringify(body), + }); + +describe('/api/v1/clone', () => { + beforeEach(() => { + vi.clearAllMocks(); + // Default reference-audio duration valid for both Voxtral (min 3s) and the + // Replicate fallback (min 10s). + mockParseBuffer.mockResolvedValue({ format: { duration: 12 } } as never); + }); + + it('returns 401 when API key is missing', async () => { + const request = new Request('http://localhost/api/v1/clone', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ + input: 'Hello world', + reference_audio: createWavBase64(), + }), + }); + + const response = await POST(request); + const json = await response.json(); + + expect(response.status).toBe(401); + expect(json.error.code).toBe('invalid_api_key'); + expect(response.headers.get('request-id')).toBeTruthy(); + }); + + it('returns 401 when API key is malformed', async () => { + const request = cloneRequest( + { input: 'Hello', reference_audio: createWavBase64() }, + { authorization: 'Bearer not-a-valid-key' }, + ); + + const response = await POST(request); + const json = await response.json(); + + expect(response.status).toBe(401); + expect(json.error.code).toBe('invalid_api_key'); + }); + + it('returns 429 when rate limit is exceeded', async () => { + mockRatelimitLimit.mockResolvedValueOnce({ + success: false, + limit: 60, + remaining: 0, + reset: Date.now() + 60_000, + }); + + const response = await POST( + cloneRequest({ input: 'Hi', reference_audio: createWavBase64() }), + ); + const json = await response.json(); + + expect(response.status).toBe(429); + expect(json.error.code).toBe('rate_limit_exceeded'); + expect(response.headers.get('x-ratelimit-remaining-requests')).toBe('0'); + }); + + it('returns 400 when neither reference_audio nor reference_audio_url is provided', async () => { + const response = await POST(cloneRequest({ input: 'Hello world' })); + const json = await response.json(); + + expect(response.status).toBe(400); + expect(json.error.code).toBe('invalid_request'); + }); + + it('returns 400 when both reference inputs are provided', async () => { + const response = await POST( + cloneRequest({ + input: 'Hello world', + reference_audio: createWavBase64(), + reference_audio_url: 'https://example.com/ref.wav', + }), + ); + const json = await response.json(); + + expect(response.status).toBe(400); + expect(json.error.code).toBe('invalid_request'); + }); + + it('returns 400 when input is empty', async () => { + const response = await POST( + cloneRequest({ input: '', reference_audio: createWavBase64() }), + ); + const json = await response.json(); + + expect(response.status).toBe(400); + expect(json.error.type).toBe('invalid_request_error'); + }); + + it('returns 400 input_too_long when free-tier Voxtral text exceeds the limit', async () => { + const response = await POST( + cloneRequest({ + input: 'a'.repeat(1001), + locale: 'en', + reference_audio: createWavBase64(), + }), + ); + const json = await response.json(); + + expect(response.status).toBe(400); + expect(json.error.code).toBe('input_too_long'); + expect(json.error.param).toBe('input'); + }); + + it('returns 400 for an unsupported locale', async () => { + const response = await POST( + cloneRequest({ + input: 'Hello world', + locale: 'xyz', + reference_audio: createWavBase64(), + }), + ); + const json = await response.json(); + + expect(response.status).toBe(400); + expect(json.error.code).toBe('unsupported_locale'); + expect(json.error.param).toBe('locale'); + }); + + it('returns 402 when credits are insufficient', async () => { + vi.mocked(getCreditsAdmin).mockResolvedValueOnce(1); + + const response = await POST( + cloneRequest({ + input: 'Hello world this is a long enough sentence', + locale: 'en', + reference_audio: createWavBase64(), + }), + ); + const json = await response.json(); + + expect(response.status).toBe(402); + expect(json.error.code).toBe('insufficient_credits'); + }); + + it('clones a voice with Mistral Voxtral for an en locale (happy path)', async () => { + const response = await POST( + cloneRequest({ + input: 'Hello from the clone API', + locale: 'en', + reference_audio: createWavBase64(), + }), + ); + const json = await response.json(); + + expect(response.status).toBe(200); + expect(json.url).toContain('files.sexyvoice.ai'); + expect(json.credits_used).toBeGreaterThan(0); + expect(json.credits_remaining).toBeDefined(); + expect(json.usage.model).toBe('voxtral-mini-tts-2603'); + expect(json.usage.input_characters).toBe('Hello from the clone API'.length); + + expect(vi.mocked(reduceCreditsAdmin)).toHaveBeenCalledWith({ + userId: 'test-user-id', + amount: json.credits_used, + }); + expect(vi.mocked(saveAudioFileAdmin)).toHaveBeenCalled(); + expect(vi.mocked(insertUsageEvent)).toHaveBeenCalledWith( + expect.objectContaining({ + sourceType: 'api_voice_cloning', + apiKeyId: 'test-api-key-id', + unit: 'operation', + quantity: 1, + }), + ); + }); + + it('clones a voice with Replicate for a non-Voxtral locale', async () => { + const response = await POST( + cloneRequest({ + input: 'こんにちは', + locale: 'ja', + reference_audio: createWavBase64(), + }), + ); + const json = await response.json(); + + expect(response.status).toBe(200); + expect(json.url).toContain('files.sexyvoice.ai'); + expect(mockReplicateRun).toHaveBeenCalled(); + expect(mockMistralSpeechComplete).not.toHaveBeenCalled(); + // Reference audio upload + generated output upload. + expect(mockUploadFileToR2).toHaveBeenCalledWith( + expect.stringContaining('clone-voice-input-api/'), + expect.any(Buffer), + expect.any(String), + 'test-speech-bucket', + undefined, + ); + }); + + it('bills a second usage event when reference audio enhancement is enabled', async () => { + const response = await POST( + cloneRequest({ + input: 'Hello world', + locale: 'en', + reference_audio: createWavBase64(), + enhance_reference_audio: true, + }), + ); + const json = await response.json(); + await flushPromises(); + + expect(response.status).toBe(200); + expect(json.url).toContain('files.sexyvoice.ai'); + // Base clone credits + 120 enhancement credits (12s * 10/sec). + expect(json.credits_used).toBeGreaterThan(120); + expect(mockFalSubscribe).toHaveBeenCalled(); + expect(vi.mocked(insertUsageEvent)).toHaveBeenCalledTimes(2); + expect(vi.mocked(insertUsageEvent)).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ + sourceType: 'audio_processing', + unit: 'secs', + creditsUsed: 120, // 12s * 10 credits/sec + }), + ); + }); + + it('rejects a reference_audio_url that resolves to a private address (SSRF guard)', async () => { + const response = await POST( + cloneRequest({ + input: 'Hello world', + locale: 'en', + reference_audio_url: 'http://169.254.169.254/latest/meta-data/', + }), + ); + const json = await response.json(); + + expect(response.status).toBe(400); + expect(json.error.code).toBe('unsupported_audio_format'); + expect(json.error.param).toBe('reference_audio_url'); + }); + + it('does not invoke paid enhancement when credits are insufficient', async () => { + // base clone credits + 120 enhancement credits would exceed this balance. + vi.mocked(getCreditsAdmin).mockResolvedValueOnce(50); + + const response = await POST( + cloneRequest({ + input: 'Hello world', + locale: 'en', + reference_audio: createWavBase64(), + enhance_reference_audio: true, + }), + ); + const json = await response.json(); + + expect(response.status).toBe(402); + expect(json.error.code).toBe('insufficient_credits'); + // The expensive enhancement provider must not be called before the check. + expect(mockFalSubscribe).not.toHaveBeenCalled(); + expect(vi.mocked(reduceCreditsAdmin)).not.toHaveBeenCalled(); + }); + + it('returns 422 when Mistral blocks the request via guardrail', async () => { + const guardrailBody = { + object: 'error', + message: 'Request blocked by guardrail policy', + type: 'guardrail_violation', + code: '1920', + raw_status_code: 403, + }; + mockMistralSpeechComplete.mockRejectedValueOnce( + Object.assign(new Error('API error occurred: Status 403'), { + body: JSON.stringify(guardrailBody), + name: 'SDKError', + statusCode: 403, + }), + ); + + const response = await POST( + cloneRequest({ + input: 'blocked text', + locale: 'en', + reference_audio: createWavBase64(), + }), + ); + const json = await response.json(); + + expect(response.status).toBe(422); + expect(json.error.code).toBe('content_policy_violation'); + expect(vi.mocked(reduceCreditsAdmin)).not.toHaveBeenCalled(); + }); + + it('returns 503 when the Replicate provider is unavailable', async () => { + mockReplicateRun.mockResolvedValueOnce({ error: 'API quota exceeded' }); + + const response = await POST( + cloneRequest({ + input: 'こんにちは', + locale: 'ja', + reference_audio: createWavBase64(), + }), + ); + const json = await response.json(); + + expect(response.status).toBe(503); + expect(json.error.code).toBe('provider_unavailable'); + }); + + it('returns 400 for malformed JSON payloads', async () => { + const response = await POST(cloneRequest('{bad-json')); + const json = await response.json(); + + expect(response.status).toBe(400); + expect(json.error.message).toBe('Invalid JSON payload'); + }); +}); diff --git a/apps/web/tests/clone-service.test.ts b/apps/web/tests/clone-service.test.ts new file mode 100644 index 000000000..4be2fa5e3 --- /dev/null +++ b/apps/web/tests/clone-service.test.ts @@ -0,0 +1,98 @@ +import { describe, expect, it } from 'vitest'; + +import { + CloneServiceError, + calculateReferenceAudioEnhancementCredits, + getCloneProviderConstraints, + getReferenceAudioEnhancementDollarCost, + isVoxtralCloneLocale, + resolveCloneProvider, + sanitizeFilename, + validateAudioDuration, + validateLocale, +} from '@/lib/clone/clone-service'; + +describe('clone-service', () => { + describe('resolveCloneProvider', () => { + it('uses Mistral Voxtral for supported Voxtral locales', () => { + expect(resolveCloneProvider('en')).toBe('mistral'); + expect(resolveCloneProvider('es')).toBe('mistral'); + expect(isVoxtralCloneLocale('fr')).toBe(true); + }); + + it('falls back to Replicate for non-Voxtral locales', () => { + expect(resolveCloneProvider('ja')).toBe('replicate'); + expect(isVoxtralCloneLocale('ja')).toBe(false); + }); + }); + + describe('getCloneProviderConstraints', () => { + it('returns the provider-specific minimum reference duration', () => { + expect(getCloneProviderConstraints('mistral').minDurationSeconds).toBe(3); + expect(getCloneProviderConstraints('replicate').minDurationSeconds).toBe( + 10, + ); + }); + }); + + describe('validateLocale', () => { + it('accepts supported locales', () => { + expect(() => validateLocale('en')).not.toThrow(); + expect(() => validateLocale('ja')).not.toThrow(); + }); + + it('throws CloneServiceError for unsupported locales', () => { + expect(() => validateLocale('xyz')).toThrow(CloneServiceError); + try { + validateLocale('xyz'); + } catch (error) { + expect((error as CloneServiceError).code).toBe('unsupported_locale'); + } + }); + }); + + describe('validateAudioDuration', () => { + it('accepts durations at or above the provider minimum', () => { + expect(() => validateAudioDuration(5, 'mistral')).not.toThrow(); + expect(() => validateAudioDuration(10, 'replicate')).not.toThrow(); + }); + + it('throws when the duration is unknown', () => { + try { + validateAudioDuration(null, 'mistral'); + } catch (error) { + expect((error as CloneServiceError).code).toBe( + 'audio_duration_unknown', + ); + } + }); + + it('throws when the reference audio is too short', () => { + try { + validateAudioDuration(2, 'mistral'); + } catch (error) { + expect((error as CloneServiceError).code).toBe( + 'audio_duration_too_short', + ); + } + }); + }); + + describe('enhancement billing helpers', () => { + it('charges 10 credits per second, rounded up', () => { + expect(calculateReferenceAudioEnhancementCredits(12)).toBe(120); + expect(calculateReferenceAudioEnhancementCredits(12.1)).toBe(121); + }); + + it('computes the dollar cost at $0.001 per second', () => { + expect(getReferenceAudioEnhancementDollarCost(12)).toBeCloseTo(0.012); + expect(getReferenceAudioEnhancementDollarCost(null)).toBe(0); + }); + }); + + describe('sanitizeFilename', () => { + it('strips diacritics and special characters', () => { + expect(sanitizeFilename('tëst-äudio!@#.wav')).toBe('test-audio___.wav'); + }); + }); +});