QVAC-23965 feat[api]: describe whisper + parakeet modelConfig fields

packages/inference/src/schemas/transcription-config.ts

@@ -10,71 +10,151 @@ export type AudioFormat = z.infer<typeof audioFormatSchema>
 
 const vadParamsSchema = z
   .object({
-    threshold: z.number().optional(),
-    min_speech_duration_ms: z.number().optional(),
-    min_silence_duration_ms: z.number().optional(),
-    max_speech_duration_s: z.number().optional(),
-    speech_pad_ms: z.number().optional(),
-    samples_overlap: z.number().optional()
+    threshold: z
+      .number()
+      .optional()
+      .describe('VAD probability threshold for classifying a segment as speech.'),
+    min_speech_duration_ms: z
+      .number()
+      .optional()
+      .describe('Minimum duration for a segment to count as speech (ms).'),
+    min_silence_duration_ms: z
+      .number()
+      .optional()
+      .describe('Minimum silence duration required to split speech segments (ms).'),
+    max_speech_duration_s: z
+      .number()
+      .optional()
+      .describe('Maximum duration of a single speech segment (s).'),
+    speech_pad_ms: z
+      .number()
+      .optional()
+      .describe('Padding added before and after each speech segment (ms).'),
+    samples_overlap: z
+      .number()
+      .optional()
+      .describe('Overlap between consecutive speech segments (0–1).')
   })
   .optional()
 
 const contextParamsSchema = z
   .object({
-    model: z.string().optional(),
-    use_gpu: z.boolean().optional(),
-    flash_attn: z.boolean().optional(),
-    gpu_device: z.number().optional()
+    model: z.string().optional().describe('Path to the whisper model file (context override).'),
+    use_gpu: z.boolean().optional().describe('Enable GPU acceleration. Default false.'),
+    flash_attn: z.boolean().optional().describe('Enable flash attention.'),
+    gpu_device: z.number().optional().describe('GPU device index to use.')
   })
   .optional()
 
 const miscConfigSchema = z
   .object({
-    caption_enabled: z.boolean().optional()
+    caption_enabled: z.boolean().optional().describe('Format output segments as captions.')
   })
   .optional()
 
+// TODO(QVAC-23933): the following whisper fields map 1:1 to upstream
+// `whisper_full_params` and have no human-readable semantics documented in-repo,
+// so they are intentionally left without a `.describe()` pending source text
+// from the addon owner: no_context, single_segment, print_realtime,
+// print_timestamps, token_timestamps, thold_ptsum, split_on_word, max_tokens,
+// debug_mode, suppress_blank, length_penalty.
 export const whisperConfigSchema = z.object({
-  strategy: z.enum(['greedy', 'beam_search']).optional(),
-  n_threads: z.number().int().optional(),
-  n_max_text_ctx: z.number().int().optional(),
-  offset_ms: z.number().int().optional(),
-  duration_ms: z.number().int().optional(),
-  audio_ctx: z.number().int().optional(),
-  translate: z.boolean().optional(),
+  strategy: z
+    .enum(['greedy', 'beam_search'])
+    .optional()
+    .describe("Decoding strategy: `'greedy'` or `'beam_search'`."),
+  n_threads: z
+    .number()
+    .int()
+    .optional()
+    .describe('CPU threads for transcription; `0` = auto (half of hardware cores).'),
+  n_max_text_ctx: z
+    .number()
+    .int()
+    .optional()
+    .describe('Maximum text tokens from previous segments used as context.'),
+  offset_ms: z
+    .number()
+    .int()
+    .optional()
+    .describe('Milliseconds to skip at the start of the audio.'),
+  duration_ms: z
+    .number()
+    .int()
+    .optional()
+    .describe('Maximum duration of audio to transcribe, in milliseconds.'),
+  audio_ctx: z
+    .number()
+    .int()
+    .optional()
+    .describe('Audio context window size in samples; `0` = model default.'),
+  translate: z.boolean().optional().describe('Translate the transcribed audio into English.'),
   no_context: z.boolean().optional(),
-  no_timestamps: z.boolean().optional(),
+  no_timestamps: z.boolean().optional().describe('Omit timestamps from the transcription output.'),
   single_segment: z.boolean().optional(),
-  print_special: z.boolean().optional(),
-  print_progress: z.boolean().optional(),
+  print_special: z.boolean().optional().describe('Print special tokens in the output.'),
+  print_progress: z.boolean().optional().describe('Print progress updates during transcription.'),
   print_realtime: z.boolean().optional(),
   print_timestamps: z.boolean().optional(),
   token_timestamps: z.boolean().optional(),
-  thold_pt: z.number().optional(),
+  thold_pt: z
+    .number()
+    .optional()
+    .describe('Word-timestamp probability threshold for accepting a word (0–1).'),
   thold_ptsum: z.number().optional(),
-  max_len: z.number().int().optional(),
+  max_len: z.number().int().optional().describe('Maximum tokens per transcription segment.'),
   split_on_word: z.boolean().optional(),
   max_tokens: z.number().int().optional(),
   debug_mode: z.boolean().optional(),
-  tdrz_enable: z.boolean().optional(),
-  suppress_regex: z.string().optional(),
-  initial_prompt: z.string().optional(),
-  language: z.string().optional(),
-  detect_language: z.boolean().optional(),
+  tdrz_enable: z
+    .boolean()
+    .optional()
+    .describe('Enable tinydiarize (lightweight speaker-turn detection).'),
+  suppress_regex: z
+    .string()
+    .optional()
+    .describe('Regular-expression pattern for tokens to suppress.'),
+  initial_prompt: z
+    .string()
+    .optional()
+    .describe('Initial prompt (context) prepended to the transcription.'),
+  language: z
+    .string()
+    .optional()
+    .describe("Transcription language (ISO 639-1) or `'auto'` to detect."),
+  detect_language: z.boolean().optional().describe('Automatically detect the spoken language.'),
   suppress_blank: z.boolean().optional(),
-  suppress_nst: z.boolean().optional(),
-  temperature: z.number().optional(),
+  suppress_nst: z.boolean().optional().describe('Suppress non-speech tokens (NST).'),
+  temperature: z.number().optional().describe('Sampling temperature (0–1). Default 0.0.'),
   length_penalty: z.number().optional(),
-  temperature_inc: z.number().optional(),
-  entropy_thold: z.number().optional(),
-  logprob_thold: z.number().optional(),
-  greedy_best_of: z.number().int().optional(),
-  beam_search_beam_size: z.number().int().optional(),
+  temperature_inc: z
+    .number()
+    .optional()
+    .describe('Temperature increment applied when sampling fails.'),
+  entropy_thold: z.number().optional().describe('Entropy threshold for filtering uncertain words.'),
+  logprob_thold: z
+    .number()
+    .optional()
+    .describe('Log-probability threshold for filtering words; `-1` disables.'),
+  greedy_best_of: z
+    .number()
+    .int()
+    .optional()
+    .describe('Greedy decoding: number of candidate completions; `-1` = default.'),
+  beam_search_beam_size: z
+    .number()
+    .int()
+    .optional()
+    .describe('Beam size for beam-search decoding; `-1` = default.'),
   vad_params: vadParamsSchema,
-  audio_format: audioFormatSchema.optional(),
+  audio_format: audioFormatSchema
+    .optional()
+    .describe("Interpretation of raw audio bytes: `'f32le'` or `'s16le'`."),
   contextParams: contextParamsSchema,
   miscConfig: miscConfigSchema,
-  vadModelSrc: modelSrcInputSchema.optional()
+  vadModelSrc: modelSrcInputSchema
+    .optional()
+    .describe('Voice-activity-detection (VAD) model source; enables VAD when set.')
 })
 
 export type WhisperConfig = z.infer<typeof whisperConfigSchema>
@@ -94,47 +174,114 @@ export type WhisperConfig = z.infer<typeof whisperConfigSchema>
 // every field on it is already namespaced under `parakeetStreamingConfig`.
 
 export const parakeetRuntimeConfigSchema = z.object({
-  maxThreads: z.number().int().optional(),
-  useGPU: z.boolean().optional(),
-  sampleRate: z.number().int().optional(),
-  channels: z.number().int().optional(),
-  captionEnabled: z.boolean().optional(),
-  timestampsEnabled: z.boolean().optional(),
-  seed: z.number().int().optional(),
-  streaming: z.boolean().optional(),
-  streamingChunkMs: z.number().int().positive().optional(),
-  streamingHistoryMs: z.number().int().positive().optional(),
-  streamingEmitPartials: z.boolean().optional(),
-  /**
-   * CTC/TDT-only energy-based voice-activity hint. Forwarded to
-   * parakeet-cpp's `StreamingOptions::enable_energy_vad`. Influences
-   * how the engine segments speech (segment cadence, partial vs final
-   * emission) but does NOT add new event types to the transcribeStream
-   * output. Use the whisper engine if you need standalone VAD
-   * `speaking`/`probability` events.
-   */
-  streamingEnergyVad: z.boolean().optional(),
-  streamingLeftContextMs: z.number().int().nonnegative().optional(),
-  streamingRightLookaheadMs: z.number().int().nonnegative().optional(),
-  /**
-   * Multilingual CTC language id (e.g. `"hi"`, `"ta"`). Required for Indic
-   * Conformer GGUFs that advertise `parakeet.ctc.lang_*` ranges; ignored on
-   * monolingual CTC such as English `parakeet-ctc-0.6b`.
-   */
-  language: z.string().optional(),
+  maxThreads: z
+    .number()
+    .int()
+    .optional()
+    .describe('CPU threads; `0` defers to hardware concurrency. Default 4.'),
+  useGPU: z.boolean().optional().describe('Enable the ggml GPU backend. Default false.'),
+  sampleRate: z.number().int().optional().describe('Input audio sample rate in Hz. Default 16000.'),
+  channels: z.number().int().optional().describe('Input audio channel count. Default 1 (mono).'),
+  captionEnabled: z
+    .boolean()
+    .optional()
+    .describe('Format output segments as captions. Default false.'),
+  timestampsEnabled: z.boolean().optional().describe('Emit per-segment timestamps. Default true.'),
+  seed: z
+    .number()
+    .int()
+    .optional()
+    .describe('Sampling RNG seed; `-1` picks a random seed. Default -1.'),
+  streaming: z.boolean().optional().describe('Open a long-lived streaming session. Default false.'),
+  streamingChunkMs: z
+    .number()
+    .int()
+    .positive()
+    .optional()
+    .describe('Streaming chunk cadence in ms. Default 2000.'),
+  streamingHistoryMs: z
+    .number()
+    .int()
+    .positive()
+    .optional()
+    .describe('Sortformer rolling-history window in ms. Default 30000.'),
+  streamingEmitPartials: z
+    .boolean()
+    .optional()
+    .describe('Emit partial results before chunk boundaries. Default true.'),
+  streamingEnergyVad: z
+    .boolean()
+    .optional()
+    .describe(
+      'CTC/TDT-only energy-based voice-activity hint; affects speech segmentation but adds no new event types. For standalone VAD `speaking`/`probability` events, use the whisper engine. Default false.'
+    ),
+  streamingLeftContextMs: z
+    .number()
+    .int()
+    .nonnegative()
+    .optional()
+    .describe('ASR encoder left-context window in ms; `-1` keeps the model default (10000).'),
+  streamingRightLookaheadMs: z
+    .number()
+    .int()
+    .nonnegative()
+    .optional()
+    .describe('ASR encoder right-lookahead window in ms; `-1` keeps the model default (2000).'),
+  language: z
+    .string()
+    .optional()
+    .describe(
+      'Multilingual CTC language id (e.g. `hi`, `ta`); required for Indic Conformer GGUFs, ignored on monolingual CTC.'
+    ),
 
   // === AOSC (Audio-Online Speaker Cache; v2.1+ Sortformer only) =========
   // Auto-enabled when the loaded GGUF carries
   // `parakeet.model_variant == "sortformer-streaming-v2.1-aosc"`. Ignored
   // by v1/v2 Sortformer and by non-Sortformer engines.
-  streamingSpkCacheEnable: z.boolean().optional(),
-  streamingSpkCacheLen: z.number().int().positive().optional(),
-  streamingFifoLen: z.number().int().positive().optional(),
-  streamingChunkLeftContextMs: z.number().int().nonnegative().optional(),
-  streamingChunkRightContextMs: z.number().int().nonnegative().optional(),
-  streamingSpkCacheUpdatePeriod: z.number().int().positive().optional(),
-  backendsDir: z.string().optional(),
-  openclCacheDir: z.string().optional()
+  streamingSpkCacheEnable: z
+    .boolean()
+    .optional()
+    .describe('AOSC (Sortformer v2.1): enable speaker-cache streaming. Default true.'),
+  streamingSpkCacheLen: z
+    .number()
+    .int()
+    .positive()
+    .optional()
+    .describe('AOSC: long-term speaker-cache rows (~15 s). Default 188.'),
+  streamingFifoLen: z
+    .number()
+    .int()
+    .positive()
+    .optional()
+    .describe('AOSC: FIFO warmup buffer rows. Default 188.'),
+  streamingChunkLeftContextMs: z
+    .number()
+    .int()
+    .nonnegative()
+    .optional()
+    .describe('AOSC: encoder left-context window in ms. Default 80.'),
+  streamingChunkRightContextMs: z
+    .number()
+    .int()
+    .nonnegative()
+    .optional()
+    .describe('AOSC: encoder right-context window in ms. Default 560.'),
+  streamingSpkCacheUpdatePeriod: z
+    .number()
+    .int()
+    .positive()
+    .optional()
+    .describe('AOSC: FIFO-overflow pop-out count. Default 144.'),
+  backendsDir: z
+    .string()
+    .optional()
+    .describe(
+      'Root directory for dynamically-loaded ggml backend `.so` files. Defaults to `prebuilds/`.'
+    ),
+  openclCacheDir: z
+    .string()
+    .optional()
+    .describe("Persistent directory for ggml-opencl's compiled-program cache (Android only).")
 })
 
 // Parakeet's load-time config currently has no fields beyond the