diff --git a/docs/design/smart-turn-v3-provider-neutral.md b/docs/design/smart-turn-v3-provider-neutral.md index f8a0d4f6ff..7c9af7a70a 100644 --- a/docs/design/smart-turn-v3-provider-neutral.md +++ b/docs/design/smart-turn-v3-provider-neutral.md @@ -28,29 +28,31 @@ also suitable for barge-in and connection lifecycle gating, but it never emits ## Consumer-neutral voice input -`LLMSessionManager.bind_voice_input_consumer()` exposes the high-level final -text boundary needed by later game/plugin integration. A binding is inert: -MicLease remains the sole microphone-ownership authority, and the caller must -bind before changing the lease owner to `game`. - -- `owner=core` keeps the existing Core transcript path. -- `owner=game` accepts PCM only while the exact captured consumer binding is - still registered; otherwise the route remains fail-closed and suspended. -- The binding receives only a route-authorized `VoiceTranscriptEvent`: a - provider-native logical final for streaming ASR, or a Smart Turn-sealed and - aggregated final for segmented ASR. It never receives PCM or partials. -- Consumer replacement or removal is forbidden while that owner holds - MicLease. Lease changes invalidate queued PCM, the active turn, transcript - reservations, and delayed callbacks before another target can become active. -- Consumer callback failure discards that delivery. It never falls back to - Core, Omni, another ASR provider, or a browser speech recognizer. -- Provider selection remains centralized and follows the active Core/ASR route - policy. A game or plugin cannot select Qwen, Soniox, or another provider. - -Phase 3 supplies only this common binding contract and a fake-consumer -integration test. Registering concrete games, changing game UI, and removing -legacy browser `SpeechRecognition` are responsibilities of their own follow-up -integration changes. +`VoiceInputRegistry` owns the high-level transcript boundary used by ordinary +Core chat, the active game route, and a future trusted plugin bridge. MicLease +remains the sole microphone-ownership authority; the Registry never receives +PCM and cannot select a provider or endpoint. + +- `owner=core` activates the built-in `core_chat` consumer, which accepts + identified partials and non-empty finals. +- `owner=game` activates the built-in `game` consumer, which accepts only + non-empty finals while `is_game_route_active(lanlan_name)` is true. An + unavailable game route stays fail-closed and never falls back to chat. +- Every route is pinned by its full `VoiceTurnToken`. Consumer switches, + unregistration, lease changes, PCM holes, aborts, and session teardown + terminate the old route instead of transferring it to the new consumer. +- Final delivery consumes the route before calling business code. A duplicate, + late event, callback failure, or empty final cannot restore or redirect it. +- Empty finals are terminal cleanup events, not transcripts. They reach neither + Core injection nor the game route. +- `core_chat` captures the prepared session and external turn id, so + cancellation after a hot swap abandons the precise original session turn. +- Plugin registration is namespace-bound and exposes no Registry, MicLease, + provider, PCM, process-management, or routing authority. + +Provider-native and Smart Turn-sealed finals therefore share one controlled +Core-side routing contract while provider selection remains centralized below +the independent-ASR runtime boundary. ## Audio contract diff --git a/main_logic/asr_client/detector_runtime.py b/main_logic/asr_client/detector_runtime.py index 0f415cdb67..33bdba2199 100644 --- a/main_logic/asr_client/detector_runtime.py +++ b/main_logic/asr_client/detector_runtime.py @@ -911,6 +911,7 @@ def __init__( vad: SileroVad | None = None, gate: SileroActivityGate | None = None, rnnoise_onset_probability: float = 0.35, + resource_optimization_enabled: bool = True, provider_policy: AsrProviderPolicy | None = None, coordinator: TurnCoordinator | None = None, on_turn_complete: Callable[[], Awaitable[None]] | None = None, @@ -935,6 +936,7 @@ def __init__( self._available = True self._closed = False self._rnnoise_onset_probability = rnnoise_onset_probability + self._resource_optimization_enabled = bool(resource_optimization_enabled) self._speech_active = False self._events: list[SpeechActivityEvent] = [] self._semantic_adapter: _VoiceTurnAdapter | None = None @@ -1383,7 +1385,8 @@ async def feed( if self._closed or not self._available: return DetectorFeedResult((), False) if ( - rnnoise_available + self._resource_optimization_enabled + and rnnoise_available and speech_probability is not None and not self._speech_active and speech_probability < self._rnnoise_onset_probability @@ -1479,7 +1482,8 @@ async def submit_audio( None, ) if ( - rnnoise_available + self._resource_optimization_enabled + and rnnoise_available and speech_probability is not None and not self._candidate_open and speech_probability < self._rnnoise_onset_probability diff --git a/main_logic/asr_client/runtime.py b/main_logic/asr_client/runtime.py index 8306f8a697..6d4ffc58ab 100644 --- a/main_logic/asr_client/runtime.py +++ b/main_logic/asr_client/runtime.py @@ -334,6 +334,7 @@ def _init_asr_runtime_state(self) -> None: self._asr_audio_sequence = 0 self._asr_audio_generation = 0 self._asr_current_ingress_token: VoiceIngressToken | None = None + self._asr_partial_turn_token: VoiceTurnToken | None = None self._asr_accepted_final_keys: OrderedDict[FinalKey, None] = OrderedDict() self._asr_reserved_final_key: FinalKey | None = None self._asr_transcript_dispatcher = TranscriptDispatcher( @@ -378,6 +379,8 @@ def _ensure_asr_runtime_state(self) -> None: self._asr_pending_detector_candidate = None if not hasattr(self, "_asr_overlap_onset_token"): self._asr_overlap_onset_token = None + if not hasattr(self, "_asr_partial_turn_token"): + self._asr_partial_turn_token = None if not hasattr(self, "_asr_overlap_completed_token"): self._asr_overlap_completed_token = None self._asr_overlap_completed_turns = 0 @@ -1182,6 +1185,9 @@ async def on_detector_event(event) -> None: raise RuntimeError("ASR_DETECTOR_CONTROL_BACKPRESSURE") detector_ref = DetectorRuntime( + resource_optimization_enabled=( + self._voice_input_resource_optimization_enabled + ), provider_policy=policy, on_endpointing_failure=( on_detector_endpointing_failure @@ -1274,6 +1280,7 @@ def _reset_asr_turn_state(self) -> None: self._asr_overlap_completed_turns = 0 self._asr_audio_sequence = 0 self._asr_current_ingress_token = None + self._asr_partial_turn_token = None self._asr_accepted_final_keys.clear() self._asr_reserved_final_key = None self._asr_sealed_turn_token = None @@ -2180,6 +2187,11 @@ async def _prepare_independent_asr_turn(self, epoch: int) -> None: self.display_name, ) if accepted and self._runtime_identity_matches(identity): + # The provider callback carries text only. Pin the source identity + # at the ordered prepare boundary; partial delivery later validates + # this exact token instead of relabeling text with whatever turn + # happens to be current at callback time. + self._asr_partial_turn_token = turn_token return transcript_dispatcher.release(final_key) if not self._runtime_identity_matches(identity): @@ -2190,6 +2202,8 @@ async def _prepare_independent_asr_turn(self, epoch: int) -> None: ): self._asr_reserved_final_key = None self._asr_turn_prepared = False + if self._asr_partial_turn_token == turn_token: + self._asr_partial_turn_token = None async def _handle_independent_asr_endpoint(self, epoch: int) -> None: """Seal the current turn immediately at its semantic endpoint.""" @@ -2392,9 +2406,19 @@ async def _send_independent_asr_preview(self, text: str, epoch: int) -> None: if not clean or epoch != self._asr_session_epoch: return lifecycle = self._asr_lifecycle + turn_token = self._asr_partial_turn_token if ( - lifecycle is not None - and not self._asr_first_partial_recorded + lifecycle is None + or turn_token is None + or not self._asr_turn_prepared + or lifecycle.snapshot.state is not VoiceLifecycleState.ACTIVE + or not self._ingress_token_matches(turn_token.ingress) + or lifecycle.snapshot.turn_id != turn_token.turn_id + or self._asr_audio_dispatcher.active_turn != turn_token + ): + return + if ( + not self._asr_first_partial_recorded and self._asr_turn_audio_started_at is not None ): lifecycle.metrics.first_partial_latency_ms = int( @@ -2403,7 +2427,7 @@ async def _send_independent_asr_preview(self, text: str, epoch: int) -> None: self._asr_first_partial_recorded = True try: await self._callbacks.on_partial( - VoicePartialEvent(text=clean, session_epoch=epoch) + VoicePartialEvent(turn_token=turn_token, text=clean) ) except Exception: logger.debug( @@ -2471,6 +2495,8 @@ async def _handle_independent_asr_final( self._asr_turn_prepared = False self._asr_received_audio = False self._asr_sealed_turn_token = None + if self._asr_partial_turn_token == sealed_token.turn: + self._asr_partial_turn_token = None self._asr_turn_endpointed_at = None self._asr_reserved_final_key = None watchdog = self._asr_final_watchdog_task diff --git a/main_logic/core/_shared.py b/main_logic/core/_shared.py index c702eeae3d..f64b1d7e52 100644 --- a/main_logic/core/_shared.py +++ b/main_logic/core/_shared.py @@ -37,6 +37,7 @@ # None collapses both into the same code path and would let recovery / # proactive paths accidentally bind their messages to a newer request_id. _REQUEST_ID_UNSET: Any = object() +_HANDSHAKE_OVERRIDE_UNSET: Any = object() _MAGIC_COMMAND_IMAGE_DROP_REQUEST_MAX = 64 _VOICE_PROACTIVE_ACK_GRACE_S = 0.05 _TEXT_SESSION_INPUT_TYPES = frozenset({"text", "avatar_drop_image", "user_image"}) diff --git a/main_logic/core/asr_runtime.py b/main_logic/core/asr_runtime.py index a679fb18d3..aea70778ad 100644 --- a/main_logic/core/asr_runtime.py +++ b/main_logic/core/asr_runtime.py @@ -11,7 +11,7 @@ import json import struct import time -from dataclasses import dataclass, field, replace +from dataclasses import dataclass, replace from typing import Callable, ClassVar, Literal from websockets import exceptions as web_exceptions @@ -21,6 +21,17 @@ AsrStartStatus, IndependentAsrRuntime, ) +from main_logic.voice_input import ( + BuiltinVoiceInputConsumer, + VoiceInputConsumerCapabilities, + VoiceInputDispatchResult, + VoiceInputRegistry, +) +from main_logic.voice_input.consumers import ( + CoreChatTurnContext, + CoreChatVoiceInputConsumer, + GameVoiceInputConsumer, +) from main_logic.voice_turn.contracts import ( AsrFailureEvent, AsrLifecycleNotification, @@ -28,7 +39,6 @@ AsrSubmitStatus, VoicePartialEvent, VoiceIngressToken, - VoiceTranscriptCallback, VoiceTranscriptEvent, VoiceTurnToken, ) @@ -40,14 +50,6 @@ from ._shared import logger - -@dataclass(frozen=True, slots=True) -class VoiceInputConsumerBinding: - owner: Literal["game"] - on_final: VoiceTranscriptCallback - identity: object = field(default_factory=object, repr=False, compare=False) - - @dataclass(frozen=True, slots=True) class _QueuedMicFrame: # Longest microphone PCM frame accepted at ingress. Bounded by DURATION, @@ -220,10 +222,6 @@ def _init_asr_runtime_state(self) -> None: self._voice_lease_resync_signal_state: tuple[str, int, bool, str] | None = ( None ) - self._voice_input_consumer_bindings: dict[ - str, - VoiceInputConsumerBinding, - ] = {} self._audio_stream_queue = _AudioDurationQueue( capacity_us=2_000_000, max_frames=256, @@ -245,9 +243,16 @@ def _init_asr_runtime_state(self) -> None: self._microphone_route_generation = 0 self._asr_route_operation_generation = 0 self._asr_notification_lock = asyncio.Lock() + # Shared with the hot-swap lifecycle: a prepared final either finishes + # against the still-open old session, or waits until close+promotion + # has atomically exposed the replacement. + self._core_voice_session_swap_lock = asyncio.Lock() + self._core_voice_session_swap_barrier_timeout_s = 5.0 self._independent_asr_provider: str | None = None self._independent_asr_route_key: str | None = None self._independent_asr_handshake_override: bool | None = None + self._voice_input_resource_optimization_handshake_override: bool | None = None + self._voice_input_resource_optimization_session_value: bool | None = None self._voice_input_noise_reduction_enabled = True self._voice_input_audio_pipeline = VoiceInputAudioPipeline( nr_enabled=self._voice_input_noise_reduction_enabled, @@ -260,11 +265,13 @@ def _init_asr_runtime_state(self) -> None: # can tell "my own bubble" from "the next turn already took it over". self._core_asr_preview_turn_id = "" self._core_asr_preview_text = "" + self._core_asr_preview_turn_token: VoiceTurnToken | None = None + self._init_voice_input_registry() callbacks = AsrRuntimeCallbacks( display_name=lambda: str(getattr(self, "lanlan_name", "core")), - on_prepare_turn=self._prepare_core_voice_turn, - on_partial=self._send_core_asr_preview, - on_final=self._dispatch_core_asr_transcript, + on_prepare_turn=self._prepare_voice_input_turn, + on_partial=self._dispatch_voice_input_partial, + on_final=self._dispatch_voice_input_final, on_turn_abandoned=self._handle_core_asr_turn_abandoned, on_failure=self._handle_core_asr_failure, on_status=self._send_core_asr_status, @@ -272,13 +279,62 @@ def _init_asr_runtime_state(self) -> None: ) self._asr_runtime = IndependentAsrRuntime(callbacks) + def _init_voice_input_registry(self) -> None: + """Install the manager-lifetime built-ins exactly once.""" + + if hasattr(self, "_voice_input_registry"): + return + registry = VoiceInputRegistry() + core_chat = CoreChatVoiceInputConsumer( + session_ref=lambda: getattr(self, "session", None), + on_prepare=lambda token, context: self._prepare_core_voice_turn( + token, + session_ref=context.session_ref, + abandon_on_failure=False, + ), + on_partial_event=self._send_core_asr_preview, + on_final_event=lambda event, context: self._dispatch_core_asr_transcript( + event, + session_ref=context.session_ref, + ), + on_cancelled_event=self._cancel_core_chat_voice_turn, + ) + game = GameVoiceInputConsumer( + lanlan_name=lambda: str(getattr(self, "lanlan_name", "core")), + ) + core_registration = registry.register_builtin( + BuiltinVoiceInputConsumer.CORE_CHAT, + core_chat, + capabilities=VoiceInputConsumerCapabilities( + accepts_partial=True, + accepts_final=True, + ), + ) + game_registration = registry.register_builtin( + BuiltinVoiceInputConsumer.GAME, + game, + capabilities=VoiceInputConsumerCapabilities( + accepts_partial=False, + accepts_final=True, + ), + ) + registry.activate(core_registration.handle) + self._voice_input_registry = registry + self._core_chat_voice_input_registration = core_registration + self._game_voice_input_registration = game_registration + def _ensure_asr_runtime_state(self) -> None: if not hasattr(self, "_asr_runtime"): self._init_asr_runtime_state() + self._init_voice_input_registry() if not hasattr(self, "_asr_route_operation_generation"): self._asr_route_operation_generation = 0 if not hasattr(self, "_asr_notification_lock"): self._asr_notification_lock = asyncio.Lock() + if not hasattr(self, "_core_voice_session_swap_lock"): + self._core_voice_session_swap_lock = asyncio.Lock() + if not hasattr(self, "_core_voice_session_swap_barrier_timeout_s"): + self._core_voice_session_swap_barrier_timeout_s = 5.0 if not hasattr(self, "_voice_input_transition_generation"): self._voice_input_transition_generation = 0 if not hasattr(self, "_voice_lease_resync_signal_state"): @@ -289,10 +345,22 @@ def _ensure_asr_runtime_state(self) -> None: self._last_hot_swap_rebind_drop_log_time = 0.0 if not hasattr(self, "_independent_asr_handshake_override"): self._independent_asr_handshake_override = None + if not hasattr( + self, + "_voice_input_resource_optimization_handshake_override", + ): + self._voice_input_resource_optimization_handshake_override = None + if not hasattr( + self, + "_voice_input_resource_optimization_session_value", + ): + self._voice_input_resource_optimization_session_value = None if not hasattr(self, "_core_asr_preview_turn_id"): self._core_asr_preview_turn_id = "" if not hasattr(self, "_core_asr_preview_text"): self._core_asr_preview_text = "" + if not hasattr(self, "_core_asr_preview_turn_token"): + self._core_asr_preview_turn_token = None if not hasattr(self, "_blocked_text_mode_microphone_signalled"): self._blocked_text_mode_microphone_signalled = False if not hasattr(self, "_voice_input_websocket"): @@ -411,9 +479,19 @@ def _ingress_token_matches(self, token: VoiceIngressToken) -> bool: ) def _voice_input_accepts_pcm(self) -> bool: - owner_has_target = self._voice_lease_owner == "core" or ( - self._voice_lease_owner == "game" - and self._voice_input_consumer_bindings.get("game") is not None + owner = self._voice_lease_owner + active_identity = self._voice_input_registry.active_identity + owner_has_target = bool( + owner in {"core", "game"} + and active_identity is not None + and active_identity.namespace == "builtin" + and active_identity.name + == ( + BuiltinVoiceInputConsumer.CORE_CHAT.value + if owner == "core" + else BuiltinVoiceInputConsumer.GAME.value + ) + and self._voice_input_registry.active_accepts_input ) return bool( self._voice_lease_synchronized @@ -423,44 +501,6 @@ def _voice_input_accepts_pcm(self) -> bool: and not self._voice_input_suppressed ) - def bind_voice_input_consumer( - self, - owner: str, - on_final: VoiceTranscriptCallback, - ) -> VoiceInputConsumerBinding: - self._ensure_asr_runtime_state() - normalized_owner = str(owner or "").strip().lower() - if normalized_owner != "game": - raise ValueError("VOICE_INPUT_CONSUMER_OWNER_UNSUPPORTED") - if not callable(on_final): - raise TypeError("VOICE_INPUT_CONSUMER_CALLBACK_REQUIRED") - if self._voice_lease_owner == normalized_owner: - raise RuntimeError("VOICE_INPUT_CONSUMER_BIND_BEFORE_TAKEOVER") - if normalized_owner in self._voice_input_consumer_bindings: - raise RuntimeError("VOICE_INPUT_CONSUMER_ALREADY_BOUND") - binding = VoiceInputConsumerBinding(owner="game", on_final=on_final) - self._voice_input_consumer_bindings[normalized_owner] = binding - return binding - - def unbind_voice_input_consumer( - self, - binding: VoiceInputConsumerBinding, - ) -> bool: - self._ensure_asr_runtime_state() - if not isinstance(binding, VoiceInputConsumerBinding): - return False - if self._voice_lease_owner == binding.owner: - raise RuntimeError("VOICE_INPUT_CONSUMER_RELEASE_LEASE_FIRST") - if self._voice_input_consumer_bindings.get(binding.owner) is not binding: - return False - del self._voice_input_consumer_bindings[binding.owner] - return True - - def _current_voice_input_consumer(self) -> VoiceInputConsumerBinding | None: - if self._voice_lease_owner != "game": - return None - return self._voice_input_consumer_bindings.get("game") - def set_independent_asr_handshake(self, value: object) -> None: # Record the frontend's authoritative independent-ASR toggle carried by # the start_session message (websocket_router). Strictly typed: only a @@ -475,21 +515,31 @@ def set_independent_asr_handshake(self, value: object) -> None: value if isinstance(value, bool) else None ) + def set_voice_input_resource_optimization_handshake( + self, + value: object, + ) -> None: + """Pin one session's authoritative resource-optimization preference.""" + self._ensure_asr_runtime_state() + self._voice_input_resource_optimization_handshake_override = ( + value if isinstance(value, bool) else None + ) + async def _start_independent_asr_if_enabled( self, input_mode: str, *, preserve_hot_swap_audio: bool = False, handshake_override=..., + resource_optimization_override=..., ) -> None: """Resolve the microphone route for one session start. ``handshake_override`` carries the start_session handshake belonging to THIS start operation, snapshotted by ``start_session`` before its first await. Ellipsis means "not supplied" — the internal re-entry paths - (hot-swap, device change) have no request of their own and fall back to - the shared field, which is correct for them: they inherit whatever the - live session was started with. + (hot-swap, device change) have no request of their own and reuse the + accepted live session's optimization choice. """ self._ensure_asr_runtime_state() operation_generation = self._begin_asr_route_operation() @@ -631,11 +681,34 @@ def core_start_is_current() -> bool: # settings POST failed or was still in flight at session start. enabled = handshake_enabled else: - enabled = bool(settings.get("independentAsrEnabled", False)) - optimization_value = settings.get( - "voiceInputResourceOptimizationEnabled", - True, + enabled = bool(settings.get("independentAsrEnabled", True)) + optimization_handshake = resource_optimization_override + if resource_optimization_override is ...: + optimization_handshake = getattr( + self, + "_voice_input_resource_optimization_session_value", + None, + ) + if optimization_handshake is None: + optimization_handshake = getattr( + self, + "_voice_input_resource_optimization_handshake_override", + None, + ) + optimization_value = ( + optimization_handshake + if optimization_handshake is not None + else settings.get("voiceInputResourceOptimizationEnabled", True) ) + resolved_optimization_value = optimization_value is not False + if resource_optimization_override is not ...: + # Only an accepted start_session call supplies this argument. + # Losing/deduplicated requests may still overwrite the manager-level + # handshake field, so internal provider restarts must use this + # session-owned snapshot instead. + self._voice_input_resource_optimization_session_value = ( + resolved_optimization_value + ) if not enabled: self._set_microphone_route("native") await self._send_core_asr_status( @@ -648,7 +721,7 @@ def core_start_is_current() -> bool: return result = await self._asr_runtime.start( route_key=core_type, - resource_optimization_enabled=optimization_value is not False, + resource_optimization_enabled=resolved_optimization_value, # Session language follows the Core-tracked user language; the # asr_client factory maps it per provider and falls back to # automatic detection when it is unset or unsupported. @@ -731,8 +804,9 @@ def _abandon_core_voice_turn( ) async def _abort_independent_asr(self, reason: str) -> None: - self._abandon_core_voice_turn() await self._asr_runtime.abort(reason) + self._invalidate_voice_pcm_sync(reason) + await self._voice_input_registry.wait_idle() async def _reset_native_audio_turn( self, @@ -808,8 +882,9 @@ async def _invalidate_interrupted_voice_turn( await abort async def _suspend_independent_asr(self, reason: str) -> None: - self._abandon_core_voice_turn() await self._asr_runtime.suspend(reason) + self._invalidate_voice_pcm_sync(reason) + await self._voice_input_registry.wait_idle() async def _close_independent_asr( self, @@ -830,13 +905,17 @@ async def _close_independent_asr( self._set_microphone_route("blocked") if not preserve_hot_swap_audio: self._invalidate_voice_pcm_sync("independent_asr_close") + else: + self._voice_input_registry.invalidate_utterance( + reason="independent_asr_close", + ) + await self._voice_input_registry.wait_idle() self._voice_input_audio_pipeline = VoiceInputAudioPipeline( nr_enabled=self._voice_input_noise_reduction_enabled, ) self._voice_input_pipeline_failed = False self._independent_asr_provider = None self._independent_asr_route_key = None - self._abandon_core_voice_turn() await self._asr_runtime.close() try: await pipeline.close() @@ -1723,6 +1802,7 @@ async def replay_frames( ) def _invalidate_voice_pcm_sync(self, reason: str) -> None: + self._voice_input_registry.invalidate_utterance(reason=reason) self._clear_audio_stream_queue(reason) self.hot_swap_audio_cache.clear() @@ -1745,10 +1825,23 @@ async def _apply_voice_lease_state( self._voice_lease_owner = owner self._voice_lease_hard_muted = hard_muted self._voice_lease_focus_suppressed = focus_suppressed + previous_owner = previous[0] + if owner != previous_owner: + if owner == "game": + self._voice_input_registry.activate( + self._game_voice_input_registration.handle, + ) + elif owner == "core": + self._voice_input_registry.activate( + self._core_chat_voice_input_registration.handle, + ) reasons: set[str] = set() if owner == "none": reasons.add("owner_none") - elif owner == "game" and self._current_voice_input_consumer() is None: + elif ( + owner == "game" + and not self._voice_input_registry.active_accepts_input + ): reasons.add("game") if hard_muted: reasons.add("hard_mute") @@ -1762,14 +1855,14 @@ async def _apply_voice_lease_state( force_abort or self._voice_lease_requires_abort or previous != current ) self._voice_lease_requires_abort = False - if reason == "game_takeover" or ( - owner == "game" and self._current_voice_input_consumer() is None - ): - await self._suspend_independent_asr(reason) + if owner == "game" and not self._voice_input_registry.active_accepts_input: + await self._asr_runtime.suspend(reason) + await self._voice_input_registry.wait_idle() elif reason == "game_release": if should_abort: route_operation_snapshot = self._asr_route_operation_generation - await self._abort_independent_asr(reason) + await self._asr_runtime.abort(reason) + await self._voice_input_registry.wait_idle() if ( self._asr_route_operation_generation != route_operation_snapshot or self._voice_lease_owner != "core" @@ -1783,7 +1876,8 @@ async def _apply_voice_lease_state( # the rest of the session. await self._asr_runtime.resume(reason) elif should_abort: - await self._abort_independent_asr(reason) + await self._asr_runtime.abort(reason) + await self._voice_input_registry.wait_idle() async def _suspend_independent_voice_input_for_game(self) -> None: await self._apply_voice_lease_state( @@ -1920,8 +2014,8 @@ async def _revoke_lease_for_blocked_route(self, reason: str) -> bool: Clients that never receive or never honour the teardown notice (older builds, third-party clients, throttled background tabs) keep uploading PCM into a route that discards it. Stop accepting it at ingress too. - The game owner is exempt: the galgame route holds the lease through its - own consumer binding and must not be collaterally revoked. + The game owner is exempt: its built-in Registry consumer holds the + active transcript route and must not be collaterally revoked. NEVER hoist this above the stale/competing-start exits. ``_revoke_voice_input_connection`` calls ``_invalidate_asr_start()`` @@ -2079,24 +2173,98 @@ async def _handle_voice_input_control( return True async def _handle_core_asr_turn_abandoned(self, token: VoiceTurnToken) -> None: - external_turn_id = f"asr-{token.ingress.session_epoch}-{token.turn_id}" - self._abandon_core_voice_turn(external_turn_id) + self._voice_input_registry.invalidate_utterance( + token, + reason="asr_turn_abandoned", + ) + await self._voice_input_registry.wait_idle() - async def _prepare_core_voice_turn(self, token: VoiceTurnToken) -> bool: + async def _prepare_voice_input_turn(self, token: VoiceTurnToken) -> bool: + self._ensure_asr_runtime_state() + # A lease transition activates its next consumer before waiting for + # keyed cancellation callbacks and aborting the ASR transport. Reject + # a prepare arriving in that window before it can pin an old ingress + # token to the newly active consumer. + if ( + not isinstance(token, VoiceTurnToken) + or token.ingress != self._capture_ingress_token() + or not self._voice_input_accepts_pcm() + ): + return False + if not self._voice_input_registry.begin_utterance(token): + return False + return await self._voice_input_registry.prepare_utterance(token) + + async def _dispatch_voice_input_partial( + self, + event: VoicePartialEvent, + ) -> None: + await self._voice_input_registry.dispatch_partial(event) + + async def _dispatch_voice_input_final( + self, + event: VoiceTranscriptEvent, + ) -> None: + result = await self._voice_input_registry.dispatch_final(event) + if result is VoiceInputDispatchResult.CALLBACK_FAILED: + await self._send_core_asr_status( + AsrStatusEvent( + code="ASR_INDEPENDENT_INJECTION_FAILED", + provider=event.provider, + session_epoch=event.turn_token.ingress.session_epoch, + ) + ) + elif result is VoiceInputDispatchResult.REJECTED: + logger.debug( + "[%s] voice input final rejected turn=%s-%s", + self.lanlan_name, + event.turn_token.ingress.session_epoch, + event.turn_token.turn_id, + ) + + async def _cancel_core_chat_voice_turn( + self, + context: CoreChatTurnContext, + reason: str, + ) -> None: + del reason + try: + await self._send_core_asr_preview_clear(context.external_turn_id) + finally: + # Registry cancellation has already consumed the keyed route. Even + # if websocket preview cleanup is itself cancelled, the response + # arbiter pause must be released exactly once here. + self._abandon_core_voice_turn( + context.external_turn_id, + session_ref=context.session_ref, + ) + + async def _prepare_core_voice_turn( + self, + token: VoiceTurnToken, + *, + session_ref: object | None = None, + abandon_on_failure: bool = True, + ) -> bool: if not self._ingress_token_matches(token.ingress): return False - if self._voice_lease_owner == "game": - return self._current_voice_input_consumer() is not None if self._voice_lease_owner != "core": return False - session_ref = self.session + if session_ref is None: + session_ref = getattr(self, "session", None) + if session_ref is None: + return False transition_generation = self._voice_input_transition_generation external_turn_id = f"asr-{token.ingress.session_epoch}-{token.turn_id}" + previous_preview_turn_id = self._core_asr_preview_turn_id + previous_preview_turn_token = self._core_asr_preview_turn_token + previous_preview_text = self._core_asr_preview_text # Turn preparation is the ordered boundary between two turns' partial # streams, so every preview from here on belongs to this turn. Stamping # the owner here is what lets a previous turn's delayed clear be # recognized as stale by the frontend instead of erasing this bubble. self._core_asr_preview_turn_id = external_turn_id + self._core_asr_preview_turn_token = token self._core_asr_preview_text = "" def operation_is_current() -> bool: @@ -2108,6 +2276,7 @@ def operation_is_current() -> bool: ) prepare = getattr(session_ref, "prepare_external_voice_turn", None) + preparation_succeeded = False try: if callable(prepare): await prepare(turn_id=external_turn_id) @@ -2116,30 +2285,35 @@ def operation_is_current() -> bool: if callable(interrupt): await interrupt() if not operation_is_current(): - self._abandon_core_voice_turn( - external_turn_id, - session_ref=session_ref, - ) + if abandon_on_failure: + self._abandon_core_voice_turn( + external_turn_id, + session_ref=session_ref, + ) return False await self.handle_new_message() if operation_is_current(): + preparation_succeeded = True return True - self._abandon_core_voice_turn( - external_turn_id, - session_ref=session_ref, - ) + if abandon_on_failure: + self._abandon_core_voice_turn( + external_turn_id, + session_ref=session_ref, + ) return False except asyncio.CancelledError: - self._abandon_core_voice_turn( - external_turn_id, - session_ref=session_ref, - ) + if abandon_on_failure: + self._abandon_core_voice_turn( + external_turn_id, + session_ref=session_ref, + ) raise except Exception: - self._abandon_core_voice_turn( - external_turn_id, - session_ref=session_ref, - ) + if abandon_on_failure: + self._abandon_core_voice_turn( + external_turn_id, + session_ref=session_ref, + ) if not operation_is_current(): return False logger.warning( @@ -2147,6 +2321,15 @@ def operation_is_current() -> bool: self.lanlan_name, ) return False + finally: + if ( + not preparation_succeeded + and self._core_asr_preview_turn_id == external_turn_id + and self._core_asr_preview_turn_token == token + ): + self._core_asr_preview_turn_id = previous_preview_turn_id + self._core_asr_preview_turn_token = previous_preview_turn_token + self._core_asr_preview_text = previous_preview_text async def _submit_core_voice_turn( self, @@ -2176,28 +2359,21 @@ async def _submit_core_voice_turn( async def _dispatch_core_asr_transcript( self, event: VoiceTranscriptEvent, + *, + session_ref: object | None = None, ) -> None: token = event.turn_token.ingress - if not self._ingress_token_matches(token): - return external_turn_id = f"asr-{token.session_epoch}-{event.turn_token.turn_id}" - binding = self._current_voice_input_consumer() - if binding is not None: - if self._voice_input_consumer_bindings.get(binding.owner) is not binding: - return - if not event.text.strip(): - # Same lingering-preview hazard as the core branch below: a - # preview created before a game takeover would otherwise - # survive this silently consumed empty final. - await self._send_core_asr_preview_clear(external_turn_id) - return - await binding.on_final(event) - return - if self._voice_lease_owner != "core": - return - session_ref = self.session + if session_ref is None: + session_ref = getattr(self, "session", None) transition_generation = self._voice_input_transition_generation try: + if ( + not self._ingress_token_matches(token) + or self._voice_lease_owner != "core" + or session_ref is None + ): + return if not event.text.strip(): # An empty final still completed the turn provider-side (e.g. # the OpenAI/Step stalled-item timeouts): Core deliberately @@ -2231,9 +2407,13 @@ def route_still_core() -> bool: and self._ingress_token_matches(token) ) - operation_still_current = self.session is session_ref and route_still_core() + operation_still_current = route_still_core() if not accepted or not operation_still_current: - if not accepted and operation_still_current: + if ( + not accepted + and operation_still_current + and self.session is session_ref + ): # Rejected text (echo suppression, takeover routing) also # never produces a user_transcript; drop the preview so it # cannot linger. Guarded on an unchanged runtime identity: @@ -2255,16 +2435,53 @@ def route_still_core() -> bool: # order _fail_closed_voice_route owns. if not route_still_core(): return - # Submit through the session validated above, not whatever - # self.session happens to be now: the preview restore awaited a - # websocket send, and a hot swap promoting a new session inside that - # await would otherwise land this transcript in the wrong - # conversation. - await self._submit_core_voice_turn( - event.text, - turn_id=external_turn_id, - session_ref=session_ref, - ) + # Synchronize with close+promotion itself, rather than sampling + # final_swap_task once. A swap may begin during any awaited submit; + # sharing this barrier means it cannot close the prepared session + # until this final finishes, while a final arriving second observes + # the promoted replacement. Bound the wait so the serial transcript + # dispatcher cannot be held forever by a stuck swap. + session_swap_lock = self._core_voice_session_swap_lock + try: + await asyncio.wait_for( + session_swap_lock.acquire(), + timeout=self._core_voice_session_swap_barrier_timeout_s, + ) + except asyncio.TimeoutError: + logger.warning( + "[%s] Timed out waiting for Core voice hot-swap barrier; " + "dropping final fail-closed", + self.lanlan_name, + ) + return + try: + if not route_still_core(): + return + + # Re-read only while close+promotion is excluded. An unchanged + # transition/ingress means a replacement session is the endpoint + # for this same conversation, not an unrelated start. + target_session = getattr(self, "session", None) + if target_session is None: + return + if target_session is not session_ref: + session_ref = target_session + prepare = getattr( + session_ref, + "prepare_external_voice_turn", + None, + ) + if callable(prepare): + await prepare(turn_id=external_turn_id) + if not route_still_core() or self.session is not session_ref: + return + await self._submit_core_voice_turn( + event.text, + turn_id=external_turn_id, + session_ref=session_ref, + ) + finally: + session_swap_lock.release() finally: self._abandon_core_voice_turn( external_turn_id, @@ -2342,9 +2559,15 @@ async def _restore_core_asr_preview_after_final( turn has stopped producing them. """ preview_owner_turn_id = self._core_asr_preview_turn_id + preview_owner_turn_token = getattr( + self, + "_core_asr_preview_turn_token", + None, + ) preview_owner_text = self._core_asr_preview_text if ( not preview_owner_turn_id + or preview_owner_turn_token is None or not preview_owner_text or preview_owner_turn_id == finalized_turn_id ): @@ -2352,8 +2575,8 @@ async def _restore_core_asr_preview_after_final( try: await self._send_core_asr_preview( VoicePartialEvent( + turn_token=preview_owner_turn_token, text=preview_owner_text, - session_epoch=session_epoch, ), remember=False, ) @@ -2383,6 +2606,7 @@ async def _send_core_asr_preview_clear(self, turn_id: str) -> None: return if self._core_asr_preview_turn_id == turn_id: self._core_asr_preview_text = "" + self._core_asr_preview_turn_token = None try: await send_json( { @@ -2401,51 +2625,63 @@ async def _send_core_asr_preview_clear(self, turn_id: str) -> None: async def _handle_core_asr_failure(self, event: AsrFailureEvent) -> None: source_identity = self._capture_core_asr_operation_identity() route_operation_generation = self._asr_route_operation_generation + + def failure_is_current() -> bool: + return bool( + self._core_asr_operation_identity_matches(source_identity) + and event.session_epoch + == self._core_asr_identity_ingress_token( + source_identity + ).session_epoch + ) + async with self._asr_notification_lock: - if ( - not self._core_asr_operation_identity_matches(source_identity) - or event.session_epoch - != self._core_asr_identity_ingress_token(source_identity).session_epoch - ): + if not failure_is_current(): return - self._abandon_core_voice_turn() - self._set_microphone_route("blocked") - self._clear_audio_stream_queue("independent_asr_failure") - self.hot_swap_audio_cache.clear() - # Fail-safe for clients that never receive or never honour the - # teardown notice (an older build, a third-party client, a - # throttled background tab). The route is fail-closed for the rest - # of the session, so stop accepting the PCM at ingress too. The - # game owner is exempt: the galgame route holds the lease through - # its own consumer binding and must not be collaterally revoked. - # No notice of its own -- the BLOCKED lifecycle event that produced - # this failure already reached the client. - # - # Re-captured AFTER this handler's own mutations, and that is the - # whole point: ``source_identity`` was taken while the route was - # still "independent", and the identity tuple carries - # ``_asr_route_mode`` (plus ``_microphone_route_generation``, inside - # the ingress token). Handing that pre-transition tuple to - # ``still_current`` made the predicate false on ENTRY -- against a - # transition this handler had itself performed two lines up -- so - # _fail_closed_voice_route returned before the revoke and left a - # live hardware microphone uploading into a dead route. The fence - # exists to reject a COMPETING newer operation, never our own step. - # Nothing awaits between the identity check at the top of this lock - # and here, so no competing operation can hide in the gap. - # - # The sibling predicates dodge this by testing the route mode - # ABSOLUTELY (``_asr_route_mode == "blocked"``) rather than against - # a captured value; a full-tuple comparison cannot, so it has to be - # re-based here instead. - post_transition_identity = self._capture_core_asr_operation_identity() - await self._fail_closed_voice_route( - "independent_asr_failure", - operation_generation=route_operation_generation, - still_current=lambda: self._core_asr_operation_identity_matches( - post_transition_identity - ), - ) + + # Registry cancellation may execute consumer callbacks. Those + # callbacks are allowed to publish status/lifecycle notifications, + # which acquire _asr_notification_lock themselves. Keep the entire + # cancellation path outside that lock, and re-fence after releasing it + # in case a newer route operation landed while this task was queued. + if not failure_is_current(): + return + self._set_microphone_route("blocked") + self._invalidate_voice_pcm_sync("independent_asr_failure") + post_transition_identity = self._capture_core_asr_operation_identity() + await self._voice_input_registry.wait_idle() + # Fail-safe for clients that never receive or never honour the + # teardown notice (an older build, a third-party client, a + # throttled background tab). The route is fail-closed for the rest + # of the session, so stop accepting the PCM at ingress too. The + # game owner is exempt: the galgame route holds the lease through + # its built-in Registry consumer and must not be collaterally + # revoked. + # No notice of its own -- the BLOCKED lifecycle event that produced + # this failure already reached the client. + # + # Re-captured AFTER this handler's own mutations, and that is the + # whole point: ``source_identity`` was taken while the route was + # still "independent", and the identity tuple carries + # ``_asr_route_mode`` (plus ``_microphone_route_generation``, inside + # the ingress token). Handing that pre-transition tuple to + # ``still_current`` made the predicate false on ENTRY -- against a + # transition this handler had itself performed two lines up -- so + # _fail_closed_voice_route returned before the revoke and left a + # live hardware microphone uploading into a dead route. The fence + # exists to reject a COMPETING newer operation, never our own step. + # + # The sibling predicates dodge this by testing the route mode + # ABSOLUTELY (``_asr_route_mode == "blocked"``) rather than against + # a captured value; a full-tuple comparison cannot, so it has to be + # re-based here instead. + await self._fail_closed_voice_route( + "independent_asr_failure", + operation_generation=route_operation_generation, + still_current=lambda: self._core_asr_operation_identity_matches( + post_transition_identity + ), + ) async def _send_core_asr_status(self, event: AsrStatusEvent) -> None: source_identity = self._capture_core_asr_operation_identity() @@ -2503,3 +2739,4 @@ async def _wait_asr_transcript_dispatch_idle(self) -> None: """ await self._asr_runtime.wait_transcript_idle() + await self._voice_input_registry.wait_idle() diff --git a/main_logic/core/lifecycle.py b/main_logic/core/lifecycle.py index 2cd6e9e7a8..202896a188 100644 --- a/main_logic/core/lifecycle.py +++ b/main_logic/core/lifecycle.py @@ -43,6 +43,7 @@ IDLE_SESSION_RESET_THRESHOLD_SECONDS, IDLE_SESSION_RESET_CHECK_INTERVAL_SECONDS, FRONTEND_START_SESSION_TIMEOUT_SECONDS, + _HANDSHAKE_OVERRIDE_UNSET, _START_LLM_CONCURRENT_ABORTED, _ORPHAN_SESSION_REAPER_TASKS, ) @@ -58,7 +59,6 @@ # facade patch would no longer reach this module's methods. from main_logic import core as _core_facade - class LifecycleMixin: """Session lifecycle methods (see module docstring).""" @@ -683,8 +683,17 @@ async def _maybe_kick_activity_loop_for_context_prompt(self) -> None: except Exception as e: logger.debug("[%s] 活动心跳 kick 失败: %s", self.lanlan_name, e) - async def start_session(self, websocket: WebSocket, new=False, input_mode='audio', - *, user_initiated=False, _allow_cross_mode_restart=True): + async def start_session( + self, + websocket: WebSocket, + new=False, + input_mode='audio', + *, + user_initiated=False, + _allow_cross_mode_restart=True, + handshake_override=_HANDSHAKE_OVERRIDE_UNSET, + resource_optimization_override=_HANDSHAKE_OVERRIDE_UNSET, + ): # user_initiated：True 仅由 websocket_router 的 start_session action 传入， # 标记"用户显式点击启动"。跨模式撞车时只有用户显式请求才会等 in-flight # 落定后改起目标模式；后台 proactive / greeting 的 auto-start 跨模式撞车 @@ -701,8 +710,19 @@ async def start_session(self, websocket: WebSocket, new=False, input_mode='audio # frontend whose field is absent and therefore CLEARS the override -- # replaced the first request's value, so that audio session selected the # persisted or opposite route. Read once here, then carry it down. - session_handshake_override = getattr( - self, "_independent_asr_handshake_override", None + session_handshake_override = ( + getattr(self, "_independent_asr_handshake_override", None) + if handshake_override is _HANDSHAKE_OVERRIDE_UNSET + else handshake_override + ) + session_resource_optimization_handshake_override = ( + getattr( + self, + "_voice_input_resource_optimization_handshake_override", + None, + ) + if resource_optimization_override is _HANDSHAKE_OVERRIDE_UNSET + else resource_optimization_override ) self._start_session_seed_turn_language() # 重置防刷屏标志 @@ -720,6 +740,10 @@ async def start_session(self, websocket: WebSocket, new=False, input_mode='audio websocket, new, input_mode, user_initiated=user_initiated, _allow_cross_mode_restart=_allow_cross_mode_restart, + handshake_override=session_handshake_override, + resource_optimization_override=( + session_resource_optimization_handshake_override + ), ): return @@ -810,6 +834,9 @@ async def start_session(self, websocket: WebSocket, new=False, input_mode='audio llm_result, _diag_start, handshake_override=session_handshake_override, + resource_optimization_override=( + session_resource_optimization_handshake_override + ), ) else: raise Exception("Session not initialized") @@ -838,8 +865,17 @@ async def start_session(self, websocket: WebSocket, new=False, input_mode='audio # Cancellation echo or the prefetch's own error — moot once this start attempt ends. pass - async def _start_session_handle_inflight(self, websocket, new, input_mode, *, - user_initiated, _allow_cross_mode_restart): + async def _start_session_handle_inflight( + self, + websocket, + new, + input_mode, + *, + user_initiated, + _allow_cross_mode_restart, + handshake_override, + resource_optimization_override, + ): """Handle a start request that collides with an in-flight start_session. Returns True when the collision was fully handled here (same-mode dedup @@ -939,8 +975,15 @@ async def _start_session_handle_inflight(self, websocket, new, input_mode, *, # 二次并发撞车回落静默 return 而非无界递归（greptile P2）。guard 检查 # （_starting_session_count 判定）前无 await，count==0 的判定到重入是原子的。 self.reset_session_start_circuit() - await self.start_session(websocket, new, input_mode, - user_initiated=True, _allow_cross_mode_restart=False) + await self.start_session( + websocket, + new, + input_mode, + user_initiated=True, + _allow_cross_mode_restart=False, + handshake_override=handshake_override, + resource_optimization_override=resource_optimization_override, + ) else: logger.warning("⚠️ Session正在启动中（跨模式重复请求），忽略") return True @@ -1518,6 +1561,7 @@ async def _start_session_activate( diag_start, *, handshake_override=..., + resource_optimization_override=..., ): """Post-connect activation: flip the active flags, start the message handler, reset the failure circuit, ack the frontend, and open the @@ -1542,6 +1586,7 @@ async def _start_session_activate( await self._start_independent_asr_if_enabled( input_mode, handshake_override=handshake_override, + resource_optimization_override=resource_optimization_override, ) # 启动成功，重置失败计数器和熔断 @@ -2323,38 +2368,42 @@ async def _perform_final_swap_sequence(self): except Exception as e: logger.warning(f"Final Swap Sequence: Old task exited with error: {e}") - # ── 步骤 2：旧 task 已停，安全关闭旧 session ───────────────────────── - if old_main_session: - try: - await old_main_session.close() - except Exception as e: - logger.error(f"💥 Final Swap Sequence: Error closing old session: {e}") - - # ── promote 前的协作取消检查点 ─────────────────────────────────────── - # Python 3.11 的 asyncio.wait_for（步骤 1）以及部分 session.close()（步骤 2） - # 在外层取消恰好落在其内层 await 已完成之后时，会把该取消“正常返回”式吞掉 - # —— except CancelledError 分支不触发，僵尸带着 cancelling()>0 继续走到 promote。 - # 步骤 1 的 except 只能拦到 wait_for *抛出* 取消的路径，拦不到这条“被吞”的路径。 - # 这里在真正改 self.session 之前补一次显式检查：只要本任务有未确认的取消请求， - # 就 re-raise 交给下面的 CancelledError 处理器关闭 new_session、重置状态。 - # 对正常热切换零影响（无外层取消时 cancelling()==0）。 - _swap_task = asyncio.current_task() - if _swap_task is not None and _swap_task.cancelling() > 0: - raise asyncio.CancelledError() - - # ── 步骤 3：promote 新 session ──────────────────────────────────────── - # 旧 listener 已停、旧 session 已关，现在切换 self.session； - # 此后旧 task 的任何回调若再执行也已看不到旧 ws。 - # 镜像启动侧的强 CAS（_start_session_start_llm 的持锁提升）：整段 swap - # 期间本函数从不改 self.session，正常路径它必然仍是入口快照的 - # old_main_session；任何偏离都意味着并发 start/end_session 已接管会话 - # （典型：swap 被取消但存活成僵尸后，新 start_session 已清场或已就位）， - # 此时覆盖 self.session 会孤儿化赢家 —— 中止 swap 并关闭 new_session。 - # 不回滚共享准备状态：它已属于接管方的新纪元，由接管方管理。 - async with self.lock: - _promote_allowed = self.session is old_main_session - if _promote_allowed: - self.session = new_session + # Exclude Core voice-final delivery across the entire close+promote + # window. The ASR dispatcher shares this lock, so a final either + # completes before the old arbiter closes or sees the replacement + # after promotion; it can no longer land between the two. + core_voice_session_lock = getattr( + self, + "_core_voice_session_swap_lock", + None, + ) + if core_voice_session_lock is None: + core_voice_session_lock = asyncio.Lock() + self._core_voice_session_swap_lock = core_voice_session_lock + async with core_voice_session_lock: + # ── 步骤 2：旧 task 已停，安全关闭旧 session ───────────────────── + if old_main_session: + try: + await old_main_session.close() + except Exception as e: + logger.error(f"💥 Final Swap Sequence: Error closing old session: {e}") + + # ── promote 前的协作取消检查点 ─────────────────────────────────── + # Python 3.11 的 asyncio.wait_for（步骤 1）以及部分 session.close() + # （步骤 2）在外层取消恰好落在其内层 await 已完成之后时，会把该取消 + # “正常返回”式吞掉。只要本任务有未确认的取消请求，就交给下面的 + # CancelledError 处理器关闭 new_session、重置状态。 + _swap_task = asyncio.current_task() + if _swap_task is not None and _swap_task.cancelling() > 0: + raise asyncio.CancelledError() + + # ── 步骤 3：promote 新 session ──────────────────────────────────── + # 镜像启动侧的强 CAS：任何偏离都意味着并发 start/end_session + # 已接管会话，此时覆盖 self.session 会孤儿化赢家。 + async with self.lock: + _promote_allowed = self.session is old_main_session + if _promote_allowed: + self.session = new_session if not _promote_allowed: logger.warning("⚠️ Final Swap Sequence: promote 时 self.session 已被并发接管，中止 swap 并关闭 new_session") try: diff --git a/main_logic/core/notify.py b/main_logic/core/notify.py index fd0d2cace6..c6673e9db7 100644 --- a/main_logic/core/notify.py +++ b/main_logic/core/notify.py @@ -561,7 +561,7 @@ async def send_session_started(self, input_mode: str): # 通知前端session已 # # Game owner exempt, matching send_session_ended_by_server and # _fail_closed_voice_route. The galgame gate owns the mic - # through its own consumer binding and tears down via + # through the built-in game consumer route and tears down via # GAME_ROUTE_ENDED, and websocket_router acknowledges a text # entry during an active game route with a bare # send_session_started("text") -- no ordinary text session, no @@ -652,7 +652,7 @@ async def send_session_ended_by_server(self): # 通知前端session已被服务 # the window holding the hardware is not necessarily the current # socket. Outside the guard on purpose: a dead current socket must # not swallow the lease holder's copy. Game owner exempt -- the - # galgame gate owns the mic through its own consumer binding and + # galgame gate owns the mic through the built-in game consumer and # tears down via GAME_ROUTE_ENDED. if getattr(self, "_voice_lease_owner", "none") != "game": await self._send_to_voice_owner(payload) diff --git a/main_logic/voice_input/__init__.py b/main_logic/voice_input/__init__.py new file mode 100644 index 0000000000..00a7a87f1a --- /dev/null +++ b/main_logic/voice_input/__init__.py @@ -0,0 +1,24 @@ +"""Controlled ASR consumer registration and transcript routing.""" + +from .contracts import ( + BuiltinVoiceInputConsumer, + VoiceInputConsumer, + VoiceInputConsumerCapabilities, + VoiceInputConsumerHandle, + VoiceInputConsumerIdentity, + VoiceInputDispatchResult, + VoiceInputRegistration, +) +from .registry import VoiceInputHandleError, VoiceInputRegistry + +__all__ = [ + "BuiltinVoiceInputConsumer", + "VoiceInputConsumer", + "VoiceInputConsumerCapabilities", + "VoiceInputConsumerHandle", + "VoiceInputConsumerIdentity", + "VoiceInputDispatchResult", + "VoiceInputHandleError", + "VoiceInputRegistration", + "VoiceInputRegistry", +] diff --git a/main_logic/voice_input/consumers/__init__.py b/main_logic/voice_input/consumers/__init__.py new file mode 100644 index 0000000000..6d6698556a --- /dev/null +++ b/main_logic/voice_input/consumers/__init__.py @@ -0,0 +1,10 @@ +"""Built-in transcript consumers owned by the Core process.""" + +from .core_chat import CoreChatTurnContext, CoreChatVoiceInputConsumer +from .game import GameVoiceInputConsumer + +__all__ = [ + "CoreChatTurnContext", + "CoreChatVoiceInputConsumer", + "GameVoiceInputConsumer", +] diff --git a/main_logic/voice_input/consumers/core_chat.py b/main_logic/voice_input/consumers/core_chat.py new file mode 100644 index 0000000000..385d47cbc9 --- /dev/null +++ b/main_logic/voice_input/consumers/core_chat.py @@ -0,0 +1,91 @@ +"""Built-in adapter for the ordinary Core chat voice-input path.""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable +from dataclasses import dataclass, field + +from main_logic.voice_turn.contracts import ( + VoicePartialEvent, + VoiceTranscriptEvent, + VoiceTurnToken, +) + + +@dataclass(frozen=True, slots=True) +class CoreChatTurnContext: + """Core resources pinned when one ASR turn is prepared.""" + + token: VoiceTurnToken + external_turn_id: str + session_ref: object + + +@dataclass(slots=True) +class CoreChatVoiceInputConsumer: + """Keep Core turn cleanup precise across route and session changes.""" + + session_ref: Callable[[], object | None] + on_prepare: Callable[ + [VoiceTurnToken, CoreChatTurnContext], + Awaitable[bool], + ] + on_partial_event: Callable[[VoicePartialEvent], Awaitable[None]] + on_final_event: Callable[ + [VoiceTranscriptEvent, CoreChatTurnContext], + Awaitable[None], + ] + on_cancelled_event: Callable[ + [CoreChatTurnContext, str], + Awaitable[None], + ] + _prepared: dict[VoiceTurnToken, CoreChatTurnContext] = field( + default_factory=dict, + init=False, + repr=False, + ) + + def is_available(self) -> bool: + # Core is the manager-owned default route. Session readiness remains a + # prepare-time fence so native PCM ingress and __new__-constructed test + # managers do not become unavailable merely because no turn can be + # prepared yet. + return True + + async def prepare_turn(self, token: VoiceTurnToken) -> bool: + if token in self._prepared: + return False + session_ref = self.session_ref() + if session_ref is None: + return False + context = CoreChatTurnContext( + token=token, + external_turn_id=( + f"asr-{token.ingress.session_epoch}-{token.turn_id}" + ), + session_ref=session_ref, + ) + self._prepared[token] = context + accepted = bool(await self.on_prepare(token, context)) + # A rejected or failed prepare is terminally cleaned up by the + # registry's keyed on_cancelled callback. Retain the captured session + # context until then so a partially prepared Core turn cannot remain + # paused merely because the prepare result was false. + return accepted and self._prepared.get(token) is context + + async def on_partial(self, event: VoicePartialEvent) -> None: + if event.turn_token not in self._prepared: + return + await self.on_partial_event(event) + + async def on_final(self, event: VoiceTranscriptEvent) -> None: + context = self._prepared.pop(event.turn_token, None) + if context is None: + return + await self.on_final_event(event, context) + + async def on_cancelled(self, token: VoiceTurnToken, reason: str) -> None: + context = self._prepared.pop(token, None) + if context is None: + return + await self.on_cancelled_event(context, reason) diff --git a/main_logic/voice_input/consumers/game.py b/main_logic/voice_input/consumers/game.py new file mode 100644 index 0000000000..3f6a218661 --- /dev/null +++ b/main_logic/voice_input/consumers/game.py @@ -0,0 +1,64 @@ +"""Built-in adapter for the active game-route voice consumer.""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass, field + +from main_logic.voice_turn.contracts import ( + VoicePartialEvent, + VoiceTranscriptEvent, + VoiceTurnToken, +) +from utils.game_route_state import ( + get_active_game_route_identity, + is_game_route_active, + route_external_voice_transcript, +) + + +@dataclass(slots=True) +class GameVoiceInputConsumer: + """Deliver identified non-empty finals to the existing game route.""" + + lanlan_name: Callable[[], str] + _prepared_routes: dict[VoiceTurnToken, tuple[str, str]] = field( + default_factory=dict, + init=False, + repr=False, + ) + + def is_available(self) -> bool: + return is_game_route_active(self.lanlan_name()) + + async def prepare_turn(self, token: VoiceTurnToken) -> bool: + if token in self._prepared_routes: + return False + identity = get_active_game_route_identity(self.lanlan_name()) + if identity is None: + return False + self._prepared_routes[token] = identity + return True + + async def on_partial(self, event: VoicePartialEvent) -> None: + del event + + async def on_final(self, event: VoiceTranscriptEvent) -> None: + token = event.turn_token + route_identity = self._prepared_routes.pop(token, None) + if route_identity is None: + raise RuntimeError("GAME_VOICE_TURN_NOT_PREPARED") + game_type, session_id = route_identity + routed = await route_external_voice_transcript( + self.lanlan_name(), + event.text, + request_id=f"asr-{token.ingress.session_epoch}-{token.turn_id}", + game_type=game_type, + session_id=session_id, + ) + if not routed: + raise RuntimeError("GAME_VOICE_TRANSCRIPT_NOT_ROUTED") + + async def on_cancelled(self, token: VoiceTurnToken, reason: str) -> None: + self._prepared_routes.pop(token, None) + del reason diff --git a/main_logic/voice_input/contracts.py b/main_logic/voice_input/contracts.py new file mode 100644 index 0000000000..5dd343a7ea --- /dev/null +++ b/main_logic/voice_input/contracts.py @@ -0,0 +1,95 @@ +"""Consumer-neutral contracts for routing Core voice-input transcripts.""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass, field +from enum import Enum +from typing import Protocol, runtime_checkable + +from main_logic.voice_turn.contracts import ( + VoicePartialEvent, + VoiceTranscriptEvent, + VoiceTurnToken, +) + + +class BuiltinVoiceInputConsumer(str, Enum): + """Host-owned consumers that may be selected without plugin authority.""" + + CORE_CHAT = "core_chat" + GAME = "game" + + +class VoiceInputDispatchResult(Enum): + """Terminal disposition for one registry dispatch attempt.""" + + REJECTED = "rejected" + CALLBACK_FAILED = "callback_failed" + DELIVERED = "delivered" + EMPTY_CONSUMED = "empty_consumed" + + +@dataclass(frozen=True, slots=True) +class VoiceInputConsumerIdentity: + """Namespaced display identity; routing authority lives in the handle.""" + + namespace: str + name: str + + +@dataclass(frozen=True, slots=True) +class VoiceInputConsumerCapabilities: + """Transcript event kinds accepted by one registered consumer.""" + + accepts_partial: bool = False + accepts_final: bool = True + + +@dataclass(frozen=True, slots=True) +class VoiceInputConsumerHandle: + """Opaque registry-issued capability used for activation.""" + + identity: VoiceInputConsumerIdentity + _registry_token: object = field(repr=False, compare=False) + _registration_token: object = field(repr=False, compare=False) + + +@runtime_checkable +class VoiceInputConsumer(Protocol): + """Delivery SPI shared by built-in and future plugin consumers.""" + + def is_available(self) -> bool: ... + + async def prepare_turn(self, token: VoiceTurnToken) -> bool: ... + + async def on_partial(self, event: VoicePartialEvent) -> None: ... + + async def on_final(self, event: VoiceTranscriptEvent) -> None: ... + + async def on_cancelled(self, token: VoiceTurnToken, reason: str) -> None: ... + + +class VoiceInputRegistration: + """Lifecycle owner for one registry-issued consumer handle.""" + + __slots__ = ("_close_callback", "_closed", "handle") + + def __init__( + self, + handle: VoiceInputConsumerHandle, + close_callback: Callable[[], bool], + ) -> None: + self.handle = handle + self._close_callback = close_callback + self._closed = False + + @property + def closed(self) -> bool: + return self._closed + + def close(self) -> bool: + if self._closed: + return False + self._closed = True + return self._close_callback() diff --git a/main_logic/voice_input/plugin_api.py b/main_logic/voice_input/plugin_api.py new file mode 100644 index 0000000000..48e796fa62 --- /dev/null +++ b/main_logic/voice_input/plugin_api.py @@ -0,0 +1,28 @@ +"""Core-side plugin voice-input SPI; no process wiring lives here.""" + +from __future__ import annotations + +from typing import Protocol, runtime_checkable + +from .contracts import ( + VoiceInputConsumer, + VoiceInputConsumerCapabilities, + VoiceInputRegistration, +) + + +@runtime_checkable +class PluginVoiceInputConsumer(VoiceInputConsumer, Protocol): + """Consumer shape implemented by a future trusted plugin bridge.""" + + +@runtime_checkable +class PluginVoiceInputRegistrar(Protocol): + """Plugin-facing registration surface with a host-fixed identity.""" + + def register_consumer( + self, + consumer: PluginVoiceInputConsumer, + *, + capabilities: VoiceInputConsumerCapabilities | None = None, + ) -> VoiceInputRegistration: ... diff --git a/main_logic/voice_input/registrar.py b/main_logic/voice_input/registrar.py new file mode 100644 index 0000000000..60c7f3b610 --- /dev/null +++ b/main_logic/voice_input/registrar.py @@ -0,0 +1,45 @@ +"""Namespace-bound registrar used by trusted host integration bridges.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from .contracts import ( + VoiceInputConsumer, + VoiceInputConsumerCapabilities, + VoiceInputConsumerIdentity, + VoiceInputRegistration, +) + +if TYPE_CHECKING: + from .registry import VoiceInputRegistry + + +class VoiceInputRegistrar: + """Register one consumer without exposing the underlying registry.""" + + def __init__( + self, + registry: VoiceInputRegistry, + identity: VoiceInputConsumerIdentity, + ) -> None: + self._registry = registry + self._identity = identity + self._registration: VoiceInputRegistration | None = None + + def register_consumer( + self, + consumer: VoiceInputConsumer, + *, + capabilities: VoiceInputConsumerCapabilities | None = None, + ) -> VoiceInputRegistration: + registration = self._registration + if registration is not None and not registration.closed: + raise RuntimeError("VOICE_INPUT_CONSUMER_ALREADY_REGISTERED") + registration = self._registry._register_plugin( + self._identity, + consumer, + capabilities or VoiceInputConsumerCapabilities(), + ) + self._registration = registration + return registration diff --git a/main_logic/voice_input/registry.py b/main_logic/voice_input/registry.py new file mode 100644 index 0000000000..d7153c132a --- /dev/null +++ b/main_logic/voice_input/registry.py @@ -0,0 +1,420 @@ +"""Controlled registration and utterance-scoped voice-input routing.""" + +from __future__ import annotations + +import asyncio +import re +from dataclasses import dataclass, field +from typing import TYPE_CHECKING + +from main_logic.voice_turn.contracts import ( + VoicePartialEvent, + VoiceTranscriptEvent, + VoiceTurnToken, +) + +from .contracts import ( + BuiltinVoiceInputConsumer, + VoiceInputConsumer, + VoiceInputConsumerCapabilities, + VoiceInputConsumerHandle, + VoiceInputConsumerIdentity, + VoiceInputDispatchResult, + VoiceInputRegistration, +) + +if TYPE_CHECKING: + from .plugin_api import PluginVoiceInputRegistrar + + +_PLUGIN_ID_PATTERN = re.compile(r"^[a-z0-9][a-z0-9._-]{0,63}$") +_RESERVED_PLUGIN_IDS = {consumer.value for consumer in BuiltinVoiceInputConsumer} + + +class VoiceInputHandleError(RuntimeError): + """Raised when an activation handle was not issued by this registry.""" + + +@dataclass(frozen=True, slots=True) +class _ConsumerRecord: + handle: VoiceInputConsumerHandle + consumer: VoiceInputConsumer + capabilities: VoiceInputConsumerCapabilities + + +@dataclass(slots=True) +class _PinnedUtterance: + record: _ConsumerRecord + token: VoiceTurnToken + activation_generation: int + prepare_callbacks_idle: asyncio.Event = field( + default_factory=asyncio.Event, + ) + pending_prepare_callbacks: int = 0 + + +class VoiceInputRegistry: + """Route each identified ASR utterance to one pinned live consumer.""" + + def __init__(self) -> None: + self._registry_token = object() + self._records: dict[VoiceInputConsumerIdentity, _ConsumerRecord] = {} + self._active: _ConsumerRecord | None = None + self._activation_generation = 0 + self._utterances: dict[VoiceTurnToken, _PinnedUtterance] = {} + self._background_tasks: set[asyncio.Task[None]] = set() + self._deferred_cancellations: list[ + tuple[_PinnedUtterance, str] + ] = [] + self._closed = False + + @property + def active_identity(self) -> VoiceInputConsumerIdentity | None: + active = self._active + return active.handle.identity if active is not None else None + + @property + def active_accepts_input(self) -> bool: + active = self._active + return bool( + not self._closed + and active is not None + and active.capabilities.accepts_final + and self._record_is_available(active) + ) + + def register_builtin( + self, + consumer_id: BuiltinVoiceInputConsumer, + consumer: VoiceInputConsumer, + *, + capabilities: VoiceInputConsumerCapabilities | None = None, + ) -> VoiceInputRegistration: + if not isinstance(consumer_id, BuiltinVoiceInputConsumer): + raise TypeError("VOICE_INPUT_BUILTIN_ID_REQUIRED") + return self._register( + VoiceInputConsumerIdentity("builtin", consumer_id.value), + consumer, + capabilities or VoiceInputConsumerCapabilities(), + ) + + def issue_plugin_registrar(self, plugin_id: str) -> PluginVoiceInputRegistrar: + """Issue one namespace-bound registrar for a host-validated plugin.""" + + normalized = str(plugin_id or "").strip().lower() + if ( + not _PLUGIN_ID_PATTERN.fullmatch(normalized) + or normalized in _RESERVED_PLUGIN_IDS + ): + raise ValueError("VOICE_INPUT_PLUGIN_ID_INVALID") + from .registrar import VoiceInputRegistrar + + return VoiceInputRegistrar( + self, + VoiceInputConsumerIdentity("plugin", normalized), + ) + + def activate(self, handle: VoiceInputConsumerHandle) -> None: + record = self._resolve_handle(handle) + if self._active is record: + return + self.invalidate_utterance(reason="consumer_switched") + self._activation_generation += 1 + self._active = record + + def begin_utterance(self, token: VoiceTurnToken) -> bool: + if ( + self._closed + or not isinstance(token, VoiceTurnToken) + or token in self._utterances + or not self.active_accepts_input + ): + return False + active = self._active + if active is None: + return False + route = _PinnedUtterance( + record=active, + token=token, + activation_generation=self._activation_generation, + ) + route.prepare_callbacks_idle.set() + self._utterances[token] = route + return True + + async def prepare_utterance(self, token: VoiceTurnToken) -> bool: + route = self._live_utterance(token) + if route is None: + return False + if not self._route_is_available(route): + consumed = self._consume_route(token) + if consumed is route: + await self._notify_cancelled( + consumed, + "consumer_unavailable", + ) + return False + route.pending_prepare_callbacks += 1 + route.prepare_callbacks_idle.clear() + try: + try: + accepted = bool( + await route.record.consumer.prepare_turn(token) + ) + except asyncio.CancelledError: + if self._utterances.get(token) is route: + self._invalidate_route(token, "prepare_cancelled") + raise + except Exception: + accepted = False + finally: + route.pending_prepare_callbacks -= 1 + if route.pending_prepare_callbacks == 0: + route.prepare_callbacks_idle.set() + if ( + not accepted + or self._live_utterance(token) is not route + or not self._route_is_available(route) + ): + if self._utterances.get(token) is route: + consumed = self._consume_route(token) + if consumed is not None: + # A lifecycle may retry the same token immediately after a + # transient prepare rejection. Finish this attempt's + # terminal callback before returning, otherwise its + # delayed cancellation could erase the retry's context. + await self._notify_cancelled( + consumed, + "prepare_rejected", + ) + return False + return True + + async def dispatch_partial( + self, + event: VoicePartialEvent, + ) -> VoiceInputDispatchResult: + if not isinstance(event, VoicePartialEvent): + return VoiceInputDispatchResult.REJECTED + route = self._live_utterance(event.turn_token) + if route is None or not route.record.capabilities.accepts_partial: + return VoiceInputDispatchResult.REJECTED + if not self._route_is_available(route): + self._invalidate_route(event.turn_token, "consumer_unavailable") + return VoiceInputDispatchResult.REJECTED + try: + await route.record.consumer.on_partial(event) + except Exception: + return VoiceInputDispatchResult.CALLBACK_FAILED + return VoiceInputDispatchResult.DELIVERED + + async def dispatch_final( + self, + event: VoiceTranscriptEvent, + ) -> VoiceInputDispatchResult: + if not isinstance(event, VoiceTranscriptEvent): + return VoiceInputDispatchResult.REJECTED + route = self._live_utterance(event.turn_token) + if route is None or not route.record.capabilities.accepts_final: + return VoiceInputDispatchResult.REJECTED + if not self._route_is_available(route): + self._invalidate_route(event.turn_token, "consumer_unavailable") + return VoiceInputDispatchResult.REJECTED + + # Consume before invoking external code. A duplicate final or callback + # failure can never restore this route or reach the next consumer. + self._consume_route(route.token) + if not event.text.strip(): + self._schedule_cancel(route, "empty_final") + return VoiceInputDispatchResult.EMPTY_CONSUMED + try: + await route.record.consumer.on_final(event) + except Exception: + return VoiceInputDispatchResult.CALLBACK_FAILED + return VoiceInputDispatchResult.DELIVERED + + def invalidate_utterance( + self, + token: VoiceTurnToken | None = None, + *, + reason: str, + ) -> bool: + if token is not None: + return self._invalidate_route(token, reason) + tokens = tuple(self._utterances) + for route_token in tokens: + self._invalidate_route(route_token, reason) + return bool(tokens) + + async def wait_idle(self) -> None: + while True: + deferred, self._deferred_cancellations = ( + self._deferred_cancellations, + [], + ) + for route, reason in deferred: + self._schedule_cancel(route, reason) + tasks = tuple(self._background_tasks) + if not tasks: + return + await asyncio.gather(*tasks, return_exceptions=True) + + async def close(self) -> None: + if not self._closed: + self.invalidate_utterance(reason="registry_closed") + self._activation_generation += 1 + self._active = None + self._records.clear() + self._closed = True + await self.wait_idle() + + def _register_plugin( + self, + identity: VoiceInputConsumerIdentity, + consumer: VoiceInputConsumer, + capabilities: VoiceInputConsumerCapabilities, + ) -> VoiceInputRegistration: + if identity.namespace != "plugin": + raise ValueError("VOICE_INPUT_PLUGIN_NAMESPACE_REQUIRED") + return self._register(identity, consumer, capabilities) + + def _register( + self, + identity: VoiceInputConsumerIdentity, + consumer: VoiceInputConsumer, + capabilities: VoiceInputConsumerCapabilities, + ) -> VoiceInputRegistration: + if self._closed: + raise RuntimeError("VOICE_INPUT_REGISTRY_CLOSED") + if identity in self._records: + raise RuntimeError("VOICE_INPUT_CONSUMER_ALREADY_REGISTERED") + required = ( + "is_available", + "prepare_turn", + "on_partial", + "on_final", + "on_cancelled", + ) + if any(not callable(getattr(consumer, name, None)) for name in required): + raise TypeError("VOICE_INPUT_CONSUMER_INVALID") + if not isinstance(capabilities, VoiceInputConsumerCapabilities): + raise TypeError("VOICE_INPUT_CAPABILITIES_REQUIRED") + handle = VoiceInputConsumerHandle( + identity=identity, + _registry_token=self._registry_token, + _registration_token=object(), + ) + record = _ConsumerRecord(handle, consumer, capabilities) + self._records[identity] = record + return VoiceInputRegistration( + handle, + lambda: self._close_registration(handle), + ) + + def _close_registration(self, handle: VoiceInputConsumerHandle) -> bool: + try: + record = self._resolve_handle(handle) + except VoiceInputHandleError: + return False + for token, route in tuple(self._utterances.items()): + if route.record is record: + self._invalidate_route(token, "consumer_unregistered") + if self._active is record: + self._activation_generation += 1 + self._active = None + del self._records[record.handle.identity] + return True + + def _resolve_handle( + self, + handle: VoiceInputConsumerHandle, + ) -> _ConsumerRecord: + if ( + not isinstance(handle, VoiceInputConsumerHandle) + or handle._registry_token is not self._registry_token + ): + raise VoiceInputHandleError("VOICE_INPUT_HANDLE_FOREIGN") + record = self._records.get(handle.identity) + if ( + record is None + or record.handle._registration_token + is not handle._registration_token + ): + raise VoiceInputHandleError("VOICE_INPUT_HANDLE_STALE") + return record + + def _live_utterance( + self, + token: VoiceTurnToken, + ) -> _PinnedUtterance | None: + route = self._utterances.get(token) + if route is None: + return None + record = self._records.get(route.record.handle.identity) + if ( + record is not route.record + or self._active is not route.record + or route.activation_generation != self._activation_generation + ): + return None + return route + + @staticmethod + def _record_is_available(record: _ConsumerRecord) -> bool: + try: + return bool(record.consumer.is_available()) + except Exception: + return False + + def _route_is_available(self, route: _PinnedUtterance) -> bool: + return self._record_is_available(route.record) + + def _consume_route( + self, + token: VoiceTurnToken, + ) -> _PinnedUtterance | None: + return self._utterances.pop(token, None) + + def _invalidate_route(self, token: VoiceTurnToken, reason: str) -> bool: + route = self._consume_route(token) + if route is None: + return False + self._schedule_cancel(route, str(reason or "cancelled")) + return True + + def _schedule_cancel( + self, + route: _PinnedUtterance, + reason: str, + ) -> None: + try: + loop = asyncio.get_running_loop() + except RuntimeError: + self._deferred_cancellations.append((route, reason)) + return + task = loop.create_task( + self._notify_cancelled(route, reason), + name="voice-input-consumer-cancel", + ) + self._background_tasks.add(task) + task.add_done_callback(self._background_tasks.discard) + + @staticmethod + async def _notify_cancelled( + route: _PinnedUtterance, + reason: str, + ) -> None: + try: + # A consumer may materialize its turn context only after an + # awaited prepare callback resumes. Terminal cancellation must + # therefore run after every prepare already in flight for this + # pinned route, or the late prepare could recreate state that the + # earlier cancellation had just cleared. + await route.prepare_callbacks_idle.wait() + await route.record.consumer.on_cancelled(route.token, reason) + except asyncio.CancelledError: + raise + except Exception: + # Cancellation is terminal and advisory; failure cannot reopen or + # redirect a consumed route. + return diff --git a/main_logic/voice_turn/contracts.py b/main_logic/voice_turn/contracts.py index 12bfe344c2..ed8fb936da 100644 --- a/main_logic/voice_turn/contracts.py +++ b/main_logic/voice_turn/contracts.py @@ -112,8 +112,14 @@ class AsrFailureEvent: class VoicePartialEvent: """Display-only partial transcript emitted by independent ASR.""" + turn_token: VoiceTurnToken text: str - session_epoch: int + + @property + def session_epoch(self) -> int: + """Compatibility view for existing read-only epoch checks.""" + + return self.turn_token.ingress.session_epoch @dataclass(frozen=True, slots=True) diff --git a/main_routers/websocket_router.py b/main_routers/websocket_router.py index 3e406c85fc..6e82611a0d 100644 --- a/main_routers/websocket_router.py +++ b/main_routers/websocket_router.py @@ -763,6 +763,20 @@ async def _dispatch_voice_message_while_superseded(message: dict) -> None: if action == "start_session": session_manager[lanlan_name].active_session_is_idle = False session_manager[lanlan_name].set_goodbye_silent(False, "start_session") + raw_handshake_override = message.get("independent_asr_enabled") + request_handshake_override = ( + raw_handshake_override + if isinstance(raw_handshake_override, bool) + else None + ) + raw_optimization_override = message.get( + "voice_input_resource_optimization_enabled" + ) + request_optimization_override = ( + raw_optimization_override + if isinstance(raw_optimization_override, bool) + else None + ) # Handshake: the frontend rides its authoritative independent-ASR # toggle along on every start_session so the route decision cannot # use a stale persisted value (settings POST failed or still in @@ -777,6 +791,15 @@ async def _dispatch_voice_message_while_superseded(message: dict) -> None: ) if callable(handshake_setter): handshake_setter(message.get("independent_asr_enabled")) + optimization_handshake_setter = getattr( + session_manager[lanlan_name], + "set_voice_input_resource_optimization_handshake", + None, + ) + if callable(optimization_handshake_setter): + optimization_handshake_setter( + message.get("voice_input_resource_optimization_enabled") + ) input_type = message.get("input_type", "audio") if input_type in _SESSION_INPUT_TYPES: if is_game_route_active(lanlan_name): @@ -790,7 +813,18 @@ async def _dispatch_voice_message_while_superseded(message: dict) -> None: if session_manager[lanlan_name]._starting_session_count == 0: session_manager[lanlan_name].reset_session_start_circuit() _fire_task(route_external_stream_message(lanlan_name, {"input_type": "audio", "stt_provider": "realtime"})) - _fire_task(session_manager[lanlan_name].start_session(websocket, message.get("new_session", False), "audio", user_initiated=True)) + _fire_task( + session_manager[lanlan_name].start_session( + websocket, + message.get("new_session", False), + "audio", + user_initiated=True, + handshake_override=request_handshake_override, + resource_optimization_override=( + request_optimization_override + ), + ) + ) continue # 传递input_mode参数，告知session manager使用何种模式 # 注意：音频模块由 main_server 后台预加载，Python import lock 会自动等待首次导入完成 @@ -828,7 +862,18 @@ async def _dispatch_voice_message_while_superseded(message: dict) -> None: # _starting_session_count > 0 的早退拦掉。 if session_manager[lanlan_name]._starting_session_count == 0: session_manager[lanlan_name].reset_session_start_circuit() - _fire_task(session_manager[lanlan_name].start_session(websocket, message.get("new_session", False), mode, user_initiated=True)) + _fire_task( + session_manager[lanlan_name].start_session( + websocket, + message.get("new_session", False), + mode, + user_initiated=True, + handshake_override=request_handshake_override, + resource_optimization_override=( + request_optimization_override + ), + ) + ) else: await session_manager[lanlan_name].send_status(json.dumps({"code": "INVALID_INPUT_TYPE", "details": {"input_type": input_type}})) diff --git a/scripts/check_core_contracts.py b/scripts/check_core_contracts.py index 4af2c0b861..0b8e3dd808 100644 --- a/scripts/check_core_contracts.py +++ b/scripts/check_core_contracts.py @@ -85,6 +85,13 @@ import Core, provider literals cannot leak into the bridge, and streaming can only enqueue audio into the bridge. +VOICE_INPUT_LAYERING + The controlled transcript Registry and its consumers may depend only on + their own package, provider-neutral voice-turn contracts, and the narrow + game-route facade. They cannot import Core, ASR/provider code, PCM + processing, routers, or arbitrary utility modules. ASR runtime code emits + neutral callbacks and cannot import the Core-owned Registry in reverse. + Every violation prints as ``path:line:col CODE message``. Exit 1 on any violation, 0 otherwise, 2 when the expected layout itself is missing (this gate hard-fails rather than silently skipping when paths move — see the @@ -97,6 +104,7 @@ import argparse import ast +import importlib.util import sys from pathlib import Path @@ -108,7 +116,6 @@ } MIXIN_SUPPORT_CLASSES = { "asr_runtime": { - "VoiceInputConsumerBinding", "_QueuedMicFrame", "_AudioDurationQueue", "_HotSwapAudioFrame", @@ -401,16 +408,20 @@ def _registry_provider_keys(path: Path) -> frozenset[str]: sys.exit(2) -def _dynamic_import_target(node: ast.AST, alias_paths: dict[str, str]) -> tuple[str | None, bool]: - """(module, is_dynamic) for ``importlib.import_module``/``__import__`` calls. +def _dynamic_import_target( + node: ast.AST, + alias_paths: dict[str, str], +) -> tuple[tuple[str, ...] | None, bool]: + """(modules, is_dynamic) for dynamic-import entry points. The static layering scans see only ``import``/``from`` forms, so ``importlib.import_module("main_logic.core")`` would sail through the gate. - ``module`` is the string-literal module argument when statically known and - None otherwise; ``is_dynamic`` is True whenever the call is one of the two - dynamic-import entry points, so guarded packages can also reject - non-literal targets the AST cannot verify. Recognizes ``importlib`` under - a top-level alias and ``from importlib import import_module [as x]`` via + ``modules`` contains the absolute module paths the call can load when they + are statically knowable, including relative ``import_module`` targets and + literal ``__import__`` fromlist entries. It is None when any required part + cannot be inferred, so guarded packages fail closed. ``is_dynamic`` is True + whenever the call is one of the two entry points. Recognizes ``importlib`` + under an alias and ``from importlib import import_module [as x]`` through ``alias_paths``. """ if not isinstance(node, ast.Call): @@ -421,11 +432,78 @@ def _dynamic_import_target(node: ast.AST, alias_paths: dict[str, str]) -> tuple[ resolved = resolve_chain(chain, alias_paths) or chain if resolved not in {"__import__", "importlib.import_module"}: return None, False - arg = node.args[0] if node.args else next( - (kw.value for kw in node.keywords if kw.arg == "name"), None) - if isinstance(arg, ast.Constant) and isinstance(arg.value, str): - return arg.value, True - return None, True + name_arg = node.args[0] if node.args else next( + (kw.value for kw in node.keywords if kw.arg == "name"), + None, + ) + if not ( + isinstance(name_arg, ast.Constant) + and isinstance(name_arg.value, str) + ): + return None, True + name = name_arg.value + + if resolved == "importlib.import_module": + if not name.startswith("."): + return (name,), True + package_arg = ( + node.args[1] + if len(node.args) > 1 + else next( + (kw.value for kw in node.keywords if kw.arg == "package"), + None, + ) + ) + if not ( + isinstance(package_arg, ast.Constant) + and isinstance(package_arg.value, str) + ): + return None, True + try: + return ( + importlib.util.resolve_name(name, package_arg.value), + ), True + except (ImportError, ValueError): + return None, True + + level_arg = ( + node.args[4] + if len(node.args) > 4 + else next( + (kw.value for kw in node.keywords if kw.arg == "level"), + None, + ) + ) + if level_arg is not None and not ( + isinstance(level_arg, ast.Constant) + and level_arg.value == 0 + ): + return None, True + fromlist_arg = ( + node.args[3] + if len(node.args) > 3 + else next( + (kw.value for kw in node.keywords if kw.arg == "fromlist"), + None, + ) + ) + if fromlist_arg is None: + return (name,), True + if not isinstance(fromlist_arg, (ast.List, ast.Tuple, ast.Set)): + return None, True + entries: list[str] = [] + for entry in fromlist_arg.elts: + if not ( + isinstance(entry, ast.Constant) + and isinstance(entry.value, str) + and entry.value + and entry.value != "*" + ): + return None, True + entries.append(entry.value) + targets = [name] + targets.extend(f"{name}.{entry}" for entry in entries) + return tuple(dict.fromkeys(targets)), True def _name_binding(node: ast.AST) -> tuple[str, ast.AST] | None: @@ -503,34 +581,57 @@ def _importlib_alias_paths(tree: ast.Module) -> dict[str, str]: return out -def _dynamic_import_violations(path: Path, tree: ast.Module, alias_paths: dict[str, str], - forbidden_prefix: str, where: str) -> list["Violation"]: +def _dynamic_import_violations( + path: Path, + tree: ast.Module, + alias_paths: dict[str, str], + forbidden_prefix: str | tuple[str, ...], + where: str, +) -> list["Violation"]: """ASR_LAYERING violations for dynamic imports in a guarded module. - Flags a literal target inside ``forbidden_prefix`` the same way the static - import ban does, and any non-literal target outright — the gate cannot + ``forbidden_prefix`` accepts one or multiple forbidden prefixes. A literal + target inside any of them is flagged the same way the static import ban + does, and any non-literal target is rejected outright — the gate cannot prove a computed module name stays on the right side of the boundary. """ + forbidden_prefixes = ( + (forbidden_prefix,) + if isinstance(forbidden_prefix, str) + else forbidden_prefix + ) # Function-local importlib aliases win over same-named module-level # bindings so nested ``import importlib as il`` cannot dodge the gate; # module-level importlib aliases resolve identically through either dict. alias_paths = {**alias_paths, **_importlib_alias_paths(tree)} out: list[Violation] = [] for node in ast.walk(tree): - target, dynamic = _dynamic_import_target(node, alias_paths) + targets, dynamic = _dynamic_import_target(node, alias_paths) if not dynamic: continue - if target is None: + if targets is None: out.append(Violation( path, node.lineno, node.col_offset, "ASR_LAYERING", f"dynamic import with a non-literal module name is not allowed in " f"{where} — the layering gate cannot verify its target; use a " f"static import or a string literal", )) - elif target == forbidden_prefix or target.startswith(f"{forbidden_prefix}."): + else: + matched_prefix = next( + ( + prefix + for target in targets + for prefix in forbidden_prefixes + if target == prefix + or target.startswith(f"{prefix}.") + ), + None, + ) + if matched_prefix is None: + continue out.append(Violation( path, node.lineno, node.col_offset, "ASR_LAYERING", - f"{where} must not import {forbidden_prefix} (dynamic import)", + f"{where} must not import {matched_prefix} (dynamic import)", )) return out @@ -967,23 +1068,25 @@ def run(root: Path) -> list[Violation]: asr_audio_path = asr_client_dir / "audio.py" asr_registry_path = asr_client_dir / "_registry_meta.py" voice_input_path = root / "main_logic" / "voice_turn" / "audio_input.py" - for required in ( - asr_bridge_path, - tts_path, - streaming_path, - asr_client_dir, - asr_component_path, - asr_audio_path, - asr_registry_path, - voice_input_path, + transcript_registry_dir = root / "main_logic" / "voice_input" + for required, violation_code in ( + (asr_bridge_path, "ASR_LAYERING"), + (tts_path, "ASR_LAYERING"), + (streaming_path, "ASR_LAYERING"), + (asr_client_dir, "ASR_LAYERING"), + (asr_component_path, "ASR_LAYERING"), + (asr_audio_path, "ASR_LAYERING"), + (asr_registry_path, "ASR_LAYERING"), + (voice_input_path, "ASR_LAYERING"), + (transcript_registry_dir, "VOICE_INPUT_LAYERING"), ): if not required.exists(): violations.append(Violation( required, 1, 0, - "ASR_LAYERING", - "required ASR layering path is missing", + violation_code, + f"required layering path is missing ({violation_code})", )) if tts_path.exists(): @@ -1031,22 +1134,104 @@ def run(root: Path) -> list[Violation]: pkg = ".".join(path.relative_to(root).parts[:-1]) alias_paths = module_alias_paths(tree, pkg) for node in ast.walk(tree): - if any( - module == "main_logic.core" - or module.startswith("main_logic.core.") - for module in _imported_paths(node, pkg, alias_paths) - ): + for module in _imported_paths(node, pkg, alias_paths): + forbidden = next( + ( + prefix + for prefix in ( + "main_logic.core", + "main_logic.voice_input", + ) + if module == prefix + or module.startswith(f"{prefix}.") + ), + None, + ) + if forbidden is not None: + violations.append(Violation( + path, + node.lineno, + node.col_offset, + "ASR_LAYERING", + f"asr_client must not import {forbidden}", + )) + violations.extend(_dynamic_import_violations( + path, tree, alias_paths, + ("main_logic.core", "main_logic.voice_input"), + "asr_client", + )) + + if transcript_registry_dir.exists(): + allowed_dependency_prefixes = ( + "main_logic.voice_input", + "main_logic.voice_turn.contracts", + "utils.game_route_state", + ) + # Resolve first-party roots from the repository instead of maintaining + # a narrow allowlist. Any importable sibling package/module (plugin, + # config, scripts, or a future root) must pass the same frozen + # dependency allowlist rather than silently bypassing the gate. + guarded_roots = tuple(sorted({ + entry.name if entry.is_dir() else entry.stem + for entry in root.iterdir() + if ( + entry.is_dir() + and entry.name.isidentifier() + ) or ( + entry.is_file() + and entry.suffix == ".py" + and entry.stem.isidentifier() + ) + })) + for path in sorted(transcript_registry_dir.rglob("*.py")): + tree = parse(path) + pkg = ".".join(path.relative_to(root).parts[:-1]) + alias_paths = module_alias_paths(tree, pkg) + dynamic_alias_paths = { + **alias_paths, + **_importlib_alias_paths(tree), + } + for node in ast.walk(tree): + for module in _imported_paths(node, pkg, alias_paths): + if not any( + module == root_name + or module.startswith(f"{root_name}.") + for root_name in guarded_roots + ): + continue + if any( + module == allowed + or module.startswith(f"{allowed}.") + for allowed in allowed_dependency_prefixes + ): + continue violations.append(Violation( path, node.lineno, node.col_offset, - "ASR_LAYERING", - "asr_client must not import main_logic.core", + "VOICE_INPUT_LAYERING", + "voice_input may depend only on its own package, " + "voice_turn.contracts, and utils.game_route_state " + f"(found {module})", + )) + targets, dynamic = _dynamic_import_target( + node, + dynamic_alias_paths, + ) + if dynamic: + detail = ( + ", ".join(targets) + if targets is not None + else "a non-literal module name" + ) + violations.append(Violation( + path, + node.lineno, + node.col_offset, + "VOICE_INPUT_LAYERING", + "dynamic imports are not allowed in voice_input " + f"(found {detail})", )) - violations.extend(_dynamic_import_violations( - path, tree, alias_paths, - "main_logic.core", "asr_client", - )) if asr_bridge_path.exists(): bridge_tree = parse(asr_bridge_path) diff --git a/static/app/app-audio-capture.js b/static/app/app-audio-capture.js index 09f0cb3385..9dd6134e5e 100644 --- a/static/app/app-audio-capture.js +++ b/static/app/app-audio-capture.js @@ -2386,6 +2386,8 @@ var micPermissionGranted = false; var cachedMicDevices = null; + var disposeVoiceRecognitionPopover = null; + var voiceRecognitionPopoverRenderGeneration = 0; function ensureMicPopupScrollbarStyle() { if (document.getElementById('neko-mic-popup-scrollbar-style')) return; @@ -2503,6 +2505,12 @@ window.renderFloatingMicList = async function (popupArg) { var micPopup = popupArg || document.getElementById('live2d-popup-mic') || document.getElementById('vrm-popup-mic') || document.getElementById('mmd-popup-mic'); if (!micPopup) return false; + var renderGeneration = ++voiceRecognitionPopoverRenderGeneration; + if (disposeVoiceRecognitionPopover) { + var previousDispose = disposeVoiceRecognitionPopover; + disposeVoiceRecognitionPopover = null; + previousDispose(); + } var popupId = micPopup.id; var isPopupAvailable = function () { if (!micPopup || !micPopup.isConnected) return false; @@ -2523,7 +2531,10 @@ if (!audioInputs || audioInputs.length === 0 || !micPermissionGranted) { audioInputs = await ensureMicrophonePermission(); } - if (!isPopupAvailable()) return false; + if ( + renderGeneration !== voiceRecognitionPopoverRenderGeneration + || !isPopupAvailable() + ) return false; if (typeof micPopup.__nekoMicScrollbarCleanup === 'function') { micPopup.__nekoMicScrollbarCleanup(); micPopup.__nekoMicScrollbarCleanup = null; @@ -2738,145 +2749,708 @@ if (typeof micPopup.__nekoMicScrollbarCleanup === 'function') { Object.assign(sep1.style, { height: '1px', backgroundColor: 'var(--neko-popup-separator)', margin: '8px 0' }); leftColumn.appendChild(sep1); - // ===== 左栏 1.5. 降噪开关 ===== - var nrContainer = document.createElement('div'); - nrContainer.style.padding = '8px 12px'; - - var nrRow = document.createElement('div'); - Object.assign(nrRow.style, { display: 'flex', justifyContent: 'space-between', alignItems: 'center' }); - - var nrLabel = document.createElement('span'); - nrLabel.textContent = window.t ? window.t('microphone.noiseReduction') : '降噪'; - nrLabel.setAttribute('data-i18n', 'microphone.noiseReduction'); - Object.assign(nrLabel.style, { fontSize: '13px', color: 'var(--neko-popup-text)', fontWeight: '500' }); - - var nrToggle = document.createElement('label'); - Object.assign(nrToggle.style, { position: 'relative', display: 'inline-block', width: '36px', height: '20px', flexShrink: '0' }); - var nrInput = document.createElement('input'); - nrInput.type = 'checkbox'; - nrInput.checked = S.noiseReductionEnabled; - Object.assign(nrInput.style, { opacity: '0', width: '0', height: '0' }); - var nrSlider = document.createElement('span'); - Object.assign(nrSlider.style, { position: 'absolute', cursor: 'pointer', top: '0', left: '0', right: '0', bottom: '0', backgroundColor: S.noiseReductionEnabled ? '#4f8cff' : '#ccc', borderRadius: '10px', transition: 'background-color 0.2s' }); - var nrKnob = document.createElement('span'); - Object.assign(nrKnob.style, { position: 'absolute', content: '""', height: '16px', width: '16px', left: S.noiseReductionEnabled ? '18px' : '2px', bottom: '2px', backgroundColor: 'white', borderRadius: '50%', transition: 'left 0.2s' }); - nrSlider.appendChild(nrKnob); - nrToggle.appendChild(nrInput); - nrToggle.appendChild(nrSlider); - - nrInput.addEventListener('change', function () { - S.noiseReductionEnabled = nrInput.checked; - nrSlider.style.backgroundColor = nrInput.checked ? '#4f8cff' : '#ccc'; - nrKnob.style.left = nrInput.checked ? '18px' : '2px'; - saveNoiseReductionSetting(); + // ===== 语音识别设置入口 + body portal popover ===== + var asrContainer = document.createElement('div'); + asrContainer.tabIndex = 0; + asrContainer.setAttribute('role', 'button'); + Object.assign(asrContainer.style, { + padding: '8px 12px', + cursor: 'pointer', + outline: 'none' }); - nrRow.appendChild(nrLabel); - nrRow.appendChild(nrToggle); - nrContainer.appendChild(nrRow); + var asrRow = document.createElement('div'); + Object.assign(asrRow.style, { + display: 'flex', + justifyContent: 'space-between', + alignItems: 'center', + gap: '12px' + }); + var asrCopy = document.createElement('div'); + Object.assign(asrCopy.style, { minWidth: '0', flex: '1' }); + var asrLabel = document.createElement('span'); + asrLabel.textContent = window.t + ? window.t('microphone.independentAsr') + : '语音识别'; + asrLabel.setAttribute('data-i18n', 'microphone.independentAsr'); + Object.assign(asrLabel.style, { + fontSize: '13px', + color: 'var(--neko-popup-text)', + fontWeight: '600' + }); + var asrSummary = document.createElement('div'); + Object.assign(asrSummary.style, { + fontSize: '11px', + color: 'var(--neko-popup-text-sub)', + marginTop: '4px', + overflow: 'hidden', + textOverflow: 'ellipsis', + whiteSpace: 'nowrap' + }); + asrCopy.appendChild(asrLabel); + asrCopy.appendChild(asrSummary); - var nrHint = document.createElement('div'); - nrHint.textContent = window.t ? window.t('microphone.noiseReductionHint') : 'RNNoise AI 降噪'; - nrHint.setAttribute('data-i18n', 'microphone.noiseReductionHint'); - Object.assign(nrHint.style, { fontSize: '11px', color: 'var(--neko-popup-text-sub)', marginTop: '6px' }); - nrContainer.appendChild(nrHint); - leftColumn.appendChild(nrContainer); + function createVoiceSettingToggle(checked, onChange) { + var focusStyle = document.getElementById( + 'neko-voice-setting-toggle-focus-style' + ); + if (!focusStyle) { + focusStyle = document.createElement('style'); + focusStyle.id = 'neko-voice-setting-toggle-focus-style'; + focusStyle.textContent = [ + '.neko-voice-setting-toggle-input:focus-visible', + '+ .neko-voice-setting-toggle-slider{', + 'box-shadow:0 0 0 2px #4f8cff;', + '}' + ].join(''); + document.head.appendChild(focusStyle); + } + var toggle = document.createElement('label'); + Object.assign(toggle.style, { + position: 'relative', + display: 'inline-block', + width: '36px', + height: '20px', + flexShrink: '0', + cursor: 'pointer' + }); + var input = document.createElement('input'); + input.className = 'neko-voice-setting-toggle-input'; + input.type = 'checkbox'; + input.checked = checked; + Object.assign(input.style, { + position: 'absolute', + inset: '0', + width: '100%', + height: '100%', + margin: '0', + opacity: '0', + cursor: 'pointer', + zIndex: '2' + }); + var slider = document.createElement('span'); + slider.className = 'neko-voice-setting-toggle-slider'; + Object.assign(slider.style, { + position: 'absolute', + inset: '0', + backgroundColor: checked ? '#4f8cff' : '#9aa0a6', + borderRadius: '10px', + transition: 'background-color 0.2s' + }); + var knob = document.createElement('span'); + Object.assign(knob.style, { + position: 'absolute', + height: '16px', + width: '16px', + left: checked ? '18px' : '2px', + bottom: '2px', + backgroundColor: 'white', + borderRadius: '50%', + transition: 'left 0.2s' + }); + slider.appendChild(knob); + toggle.appendChild(input); + toggle.appendChild(slider); + toggle.addEventListener('click', function (event) { + event.stopPropagation(); + }); + toggle.addEventListener('pointerup', function (event) { + event.stopPropagation(); + }); + input.addEventListener('change', function () { + slider.style.backgroundColor = input.checked + ? '#4f8cff' + : '#9aa0a6'; + knob.style.left = input.checked ? '18px' : '2px'; + onChange(input.checked); + }); + return { + element: toggle, + input: input, + setDisabled: function (disabled) { + input.disabled = disabled; + toggle.style.cursor = disabled ? 'not-allowed' : 'pointer'; + input.style.cursor = disabled ? 'not-allowed' : 'pointer'; + toggle.style.opacity = disabled ? '0.5' : '1'; + }, + setChecked: function (value) { + input.checked = value; + slider.style.backgroundColor = value + ? '#4f8cff' + : '#9aa0a6'; + knob.style.left = value ? '18px' : '2px'; + } + }; + } - // ===== 独立 ASR 开关（下次语音 session 生效） ===== - var asrContainer = document.createElement('div'); - asrContainer.style.padding = '8px 12px'; + function persistVoiceSettingChange() { + if ( + !window.appSettings + || typeof window.appSettings.saveSettings !== 'function' + ) return; + if (typeof window.appSettings.syncSettingsToServer !== 'function') { + window.appSettings.saveSettings(); + return; + } + // Preserve the existing session-start ownership fence: persist + // locally now, then expose the serialized server sync promise + // for ensureWebSocketOpen() to await before start_session. + window.appSettings.saveSettings({ skipServerSync: true }); + var syncPromise = Promise.resolve( + window.appSettings.syncSettingsToServer({ userInitiated: true }) + ) + .catch(function () { + // syncSettingsToServer owns failure reporting. + }) + .then(function () { + if (S.pendingSettingsSyncPromise === syncPromise) { + S.pendingSettingsSyncPromise = null; + } + }); + S.pendingSettingsSyncPromise = syncPromise; + } + + function markVoiceSettingsPending(activeRouteSnapshot) { + var targetEpoch = (Number(S.voiceSessionStartEpoch) || 0) + 1; + if ( + activeRouteSnapshot !== undefined + && ( + S.voiceSettingsPendingUntilEpoch !== targetEpoch + || S.pendingVoiceRouteIndependentAsr === null + ) + ) { + S.pendingVoiceRouteIndependentAsr = activeRouteSnapshot; + } + S.voiceSettingsPendingUntilEpoch = targetEpoch; + } + var asrToggle = createVoiceSettingToggle( + S.independentAsrEnabled === true, + function (enabled) { + var activeRouteSnapshot = S.voiceChatActive === true + ? ( + S.independentAsrActive === true + || ( + S.voiceInputLifecycleState === 'blocked' + && S.independentAsrEnabled === true + ) + ) + : null; + S.independentAsrEnabled = enabled; + markVoiceSettingsPending(activeRouteSnapshot); + updateVoiceRecognitionUi(); + persistVoiceSettingChange(); + } + ); + asrRow.appendChild(asrCopy); + asrRow.appendChild(asrToggle.element); + asrContainer.appendChild(asrRow); + leftColumn.appendChild(asrContainer); - var asrRow = document.createElement('div'); - Object.assign(asrRow.style, { display: 'flex', justifyContent: 'space-between', alignItems: 'center' }); + var voicePanelId = (popupId || 'neko-mic') + + '-voice-recognition-settings'; + asrContainer.setAttribute('aria-controls', voicePanelId); + asrContainer.setAttribute('aria-expanded', 'false'); + var voicePanel = null; + var voiceBridge = null; + var voicePanelOpen = false; + var voicePanelPinned = false; + var voiceOpenTimer = null; + var voiceCloseTimer = null; + var voicePopupObserver = null; + var noiseToggle = null; + var optimizationToggle = null; + var optimizationHint = null; + var voiceStatus = null; + + function providerDisplayName(provider) { + var value = String(provider || '').trim(); + if (!value) return ''; + var known = { + qwen: 'Qwen', + soniox: 'Soniox', + glm: 'GLM', + gemini: 'Gemini', + openai: 'OpenAI', + step: 'Step', + grok: 'Grok' + }; + return known[value.toLowerCase()] || value; + } + + function clearVoiceTimers() { + if (voiceOpenTimer !== null) clearTimeout(voiceOpenTimer); + if (voiceCloseTimer !== null) clearTimeout(voiceCloseTimer); + voiceOpenTimer = null; + voiceCloseTimer = null; + } + + function appendVoicePanelSetting( + labelKey, + fallbackLabel, + hintKey, + fallbackHint, + toggle + ) { + var block = document.createElement('div'); + block.style.marginBottom = '14px'; + var row = document.createElement('div'); + Object.assign(row.style, { + display: 'flex', + justifyContent: 'space-between', + alignItems: 'center', + gap: '12px' + }); + var label = document.createElement('span'); + var settingId = ( + voicePanelId + '-' + labelKey + ).replace(/[^a-z0-9_-]+/gi, '-'); + label.id = settingId + '-label'; + label.textContent = window.t ? window.t(labelKey) : fallbackLabel; + label.setAttribute('data-i18n', labelKey); + Object.assign(label.style, { + fontSize: '13px', + fontWeight: '500' + }); + row.appendChild(label); + row.appendChild(toggle.element); + var hint = document.createElement('div'); + hint.id = settingId + '-hint'; + hint.textContent = window.t ? window.t(hintKey) : fallbackHint; + hint.setAttribute('data-i18n', hintKey); + Object.assign(hint.style, { + fontSize: '11px', + color: 'var(--neko-popup-text-sub)', + marginTop: '5px', + lineHeight: '1.45' + }); + block.appendChild(row); + block.appendChild(hint); + toggle.input.setAttribute('aria-labelledby', label.id); + toggle.input.setAttribute('aria-describedby', hint.id); + voicePanel.appendChild(block); + return hint; + } + + function createVoicePanel() { + if (voicePanel) return voicePanel; + voicePanel = document.createElement('div'); + voicePanel.id = voicePanelId; + voicePanel.setAttribute('role', 'dialog'); + voicePanel.setAttribute( + 'aria-label', + window.t + ? window.t('microphone.voiceRecognitionSettings') + : '语音识别设置' + ); + Object.assign(voicePanel.style, { + display: 'none', + position: 'fixed', + zIndex: '2147483000', + width: '280px', + boxSizing: 'border-box', + padding: '14px', + borderRadius: '10px', + color: 'var(--neko-popup-text)', + background: 'var(--neko-popup-bg, rgba(30, 30, 34, 0.98))', + border: '1px solid var(--neko-popup-separator)', + boxShadow: '0 10px 32px rgba(0, 0, 0, 0.28)' + }); + document.body.appendChild(voicePanel); + + var voicePanelTitle = document.createElement('div'); + voicePanelTitle.textContent = window.t + ? window.t('microphone.voiceRecognitionSettings') + : '语音识别设置'; + voicePanelTitle.setAttribute( + 'data-i18n', + 'microphone.voiceRecognitionSettings' + ); + Object.assign(voicePanelTitle.style, { + fontSize: '14px', + fontWeight: '650', + marginBottom: '14px' + }); + voicePanel.appendChild(voicePanelTitle); - var asrLabel = document.createElement('span'); - asrLabel.textContent = window.t ? window.t('microphone.independentAsr') : 'Independent ASR'; - asrLabel.setAttribute('data-i18n', 'microphone.independentAsr'); - Object.assign(asrLabel.style, { fontSize: '13px', color: 'var(--neko-popup-text)', fontWeight: '500' }); - - var asrToggle = document.createElement('label'); - Object.assign(asrToggle.style, { position: 'relative', display: 'inline-block', width: '36px', height: '20px', flexShrink: '0' }); - var asrInput = document.createElement('input'); - asrInput.type = 'checkbox'; - asrInput.checked = S.independentAsrEnabled === true; - Object.assign(asrInput.style, { opacity: '0', width: '0', height: '0' }); - var asrSlider = document.createElement('span'); - Object.assign(asrSlider.style, { position: 'absolute', cursor: 'pointer', top: '0', left: '0', right: '0', bottom: '0', backgroundColor: asrInput.checked ? '#4f8cff' : '#ccc', borderRadius: '10px', transition: 'background-color 0.2s' }); - var asrKnob = document.createElement('span'); - Object.assign(asrKnob.style, { position: 'absolute', content: '""', height: '16px', width: '16px', left: asrInput.checked ? '18px' : '2px', bottom: '2px', backgroundColor: 'white', borderRadius: '50%', transition: 'left 0.2s' }); - asrSlider.appendChild(asrKnob); - asrToggle.appendChild(asrInput); - asrToggle.appendChild(asrSlider); - - function renderAsrHint() { - if (!asrHint) return; - var hintKey = S.independentAsrActive - ? 'microphone.independentAsrActive' - : (S.independentAsrEnabled ? 'microphone.independentAsrNextSession' : 'microphone.independentAsrNative'); - var hintParams = { providerKey: S.independentAsrProvider || 'unknown' }; - asrHint.setAttribute('data-i18n', hintKey); - asrHint.setAttribute('data-i18n-params', JSON.stringify(hintParams)); - asrHint.textContent = window.t - ? window.t(hintKey, hintParams) - : (S.independentAsrActive ? 'Independent ASR active' : (S.independentAsrEnabled ? 'Takes effect next voice session' : 'Using Omni native recognition')); - } - - asrInput.addEventListener('change', function () { - S.independentAsrEnabled = asrInput.checked; - asrSlider.style.backgroundColor = asrInput.checked ? '#4f8cff' : '#ccc'; - asrKnob.style.left = asrInput.checked ? '18px' : '2px'; - // The confirmation text has to follow the switch it confirms. - renderAsrHint(); - if (window.appSettings && typeof window.appSettings.saveSettings === 'function') { - if (typeof window.appSettings.syncSettingsToServer === 'function') { - // Session start reads the SERVER-persisted value (asr_runtime.py - // _start_independent_asr_if_enabled -> aload_global_conversation_settings), - // so the fire-and-forget POST inside saveSettings() can race a - // mic start and silently keep the previous route. Persist - // locally first, then run the POST ourselves and publish it as - // S.pendingSettingsSyncPromise; ensureWebSocketOpen() - // (app-websocket.js) awaits it before any start_session send. - // userInitiated: true marks settings hydrated so the - // start_session handshake stamps this explicit choice - // even while the settings GET is failing. - // syncSettingsToServer serializes its POSTs internally - // and snapshots settings at send time, so flipping the - // toggle twice quickly cannot let the older request - // finish last and persist the stale value; the newer - // promise published below also resolves only after any - // predecessor POST completed. - window.appSettings.saveSettings({ skipServerSync: true }); - var syncPromise = Promise.resolve(window.appSettings.syncSettingsToServer({ userInitiated: true })) - .catch(function () { /* syncSettingsToServer already logs failures */ }) - .then(function () { - if (S.pendingSettingsSyncPromise === syncPromise) { - S.pendingSettingsSyncPromise = null; - } - }); - S.pendingSettingsSyncPromise = syncPromise; - } else { - window.appSettings.saveSettings(); + noiseToggle = createVoiceSettingToggle( + S.noiseReductionEnabled === true, + function (enabled) { + S.noiseReductionEnabled = enabled; + saveNoiseReductionSetting(); + } + ); + appendVoicePanelSetting( + 'microphone.noiseReduction', + '降噪', + 'microphone.noiseReductionHint', + '让输入语音更加清晰', + noiseToggle + ); + + optimizationToggle = createVoiceSettingToggle( + S.voiceInputResourceOptimizationEnabled !== false, + function (enabled) { + S.voiceInputResourceOptimizationEnabled = enabled; + markVoiceSettingsPending(); + updateVoiceRecognitionUi(); + persistVoiceSettingChange(); } + ); + optimizationHint = appendVoicePanelSetting( + 'microphone.voiceResourceOptimization', + '智能资源优化', + 'microphone.voiceResourceOptimizationHintOn', + '空闲时减少连接和音频上传', + optimizationToggle + ); + + voiceStatus = document.createElement('div'); + Object.assign(voiceStatus.style, { + borderTop: '1px solid var(--neko-popup-separator)', + paddingTop: '11px', + fontSize: '11px', + lineHeight: '1.45', + color: 'var(--neko-popup-text-sub)' + }); + voicePanel.appendChild(voiceStatus); + + voiceBridge = document.createElement('div'); + Object.assign(voiceBridge.style, { + display: 'none', + position: 'fixed', + zIndex: '2147482999' + }); + document.body.appendChild(voiceBridge); + + voicePanel.addEventListener('mouseenter', clearVoiceTimers); + voicePanel.addEventListener('mouseleave', scheduleVoiceClose); + voicePanel.addEventListener('focusin', clearVoiceTimers); + voicePanel.addEventListener('focusout', scheduleVoiceClose); + voiceBridge.addEventListener('mouseenter', clearVoiceTimers); + voiceBridge.addEventListener('mouseleave', scheduleVoiceClose); + + document.addEventListener('pointerdown', onVoiceDocumentPointerDown, true); + document.addEventListener( + 'keydown', + onVoiceDocumentKeyDown, + true + ); + window.addEventListener('resize', positionVoicePanel); + window.addEventListener('scroll', positionVoicePanel, true); + window.addEventListener( + 'voice-input-lifecycle-changed', + onVoiceLifecycleChanged + ); + window.addEventListener( + 'neko:voice-session-started', + onVoiceSessionStarted + ); + window.addEventListener( + 'neko:voice-settings-pending-changed', + onVoiceSettingsPendingChanged + ); + voicePopupObserver = new MutationObserver(function () { + if (!isPopupAvailable()) destroyVoicePanel(); + }); + var popupAncestor = micPopup.parentNode; + while (popupAncestor) { + voicePopupObserver.observe(popupAncestor, { + childList: true + }); + popupAncestor = popupAncestor.parentNode; } - }); + voicePopupObserver.observe(micPopup, { + attributes: true, + attributeFilter: ['style', 'class'] + }); + updateVoiceRecognitionUi(); + return voicePanel; + } + + function updateVoiceRecognitionUi() { + var enabled = S.independentAsrEnabled === true; + if ( + S.voiceSettingsPendingUntilEpoch !== null + && (Number(S.voiceSessionStartEpoch) || 0) + >= S.voiceSettingsPendingUntilEpoch + ) { + S.voiceSettingsPendingUntilEpoch = null; + S.pendingVoiceRouteIndependentAsr = null; + } + var summaryUsesIndependentAsr = + S.voiceSettingsPendingUntilEpoch !== null + && S.pendingVoiceRouteIndependentAsr !== null + ? S.pendingVoiceRouteIndependentAsr + : enabled; + var provider = providerDisplayName(S.independentAsrProvider); + var blocked = S.voiceInputLifecycleState === 'blocked'; + asrToggle.setChecked(enabled); + asrSummary.textContent = summaryUsesIndependentAsr + ? ( + window.t + ? window.t( + provider + ? 'microphone.independentAsrSummary' + : 'microphone.independentAsrSummaryGeneric', + { provider: provider } + ) + : ('独立 ASR' + (provider ? ' · ' + provider : '')) + ) + : ( + window.t + ? window.t('microphone.voiceRecognitionDisabled') + : '当前使用 Omni 原生语音识别' + ); + if (!voicePanel) return; + // RNNoise is local PCM preprocessing shared by both the + // independent-ASR and Omni-native routes. + noiseToggle.setDisabled(false); + optimizationToggle.setDisabled(!enabled); + if (S.voiceSettingsPendingUntilEpoch !== null) { + voiceStatus.textContent = window.t + ? window.t('microphone.voiceRecognitionSettingsPending') + : '◐ 设置将在下次语音会话生效'; + } else if (!enabled) { + voiceStatus.textContent = window.t + ? window.t('microphone.voiceRecognitionDisabledHint') + : '独立 ASR 已关闭；语音输入使用 Omni 原生语音识别'; + } else if (blocked) { + voiceStatus.textContent = window.t + ? window.t('microphone.voiceRecognitionUnavailable') + : '本次独立语音识别已停止，不会切换到其他 Provider 或 Omni'; + } else if (S.independentAsrActive) { + voiceStatus.textContent = window.t + ? window.t('microphone.voiceRecognitionStatusReady') + : '● 当前运行正常'; + } else { + voiceStatus.textContent = window.t + ? window.t('microphone.voiceRecognitionSettingsPending') + : '◐ 设置将在下次语音会话生效'; + } + var optimizationEnabled = + S.voiceInputResourceOptimizationEnabled !== false; + optimizationToggle.setChecked(optimizationEnabled); + optimizationHint.textContent = window.t + ? window.t( + optimizationEnabled + ? 'microphone.voiceResourceOptimizationHintOn' + : 'microphone.voiceResourceOptimizationHintOff' + ) + : ( + optimizationEnabled + ? '空闲时减少连接和音频上传' + : '持续保持语音识别，可能增加网络和资源占用' + ); + } + + function positionVoicePanel() { + if (!voicePanelOpen || !voicePanel || !asrContainer.isConnected) { + return; + } + var rect = asrContainer.getBoundingClientRect(); + var gap = 10; + var viewportWidth = + window.innerWidth || document.documentElement.clientWidth; + var viewportHeight = + window.innerHeight || document.documentElement.clientHeight; + var panelRect = voicePanel.getBoundingClientRect(); + var panelWidth = panelRect.width || 280; + var panelHeight = panelRect.height || 240; + var placeRight = + rect.right + gap + panelWidth <= viewportWidth - 12; + var placeLeft = rect.left - gap - panelWidth >= 12; + var placeBelow = + rect.bottom + gap + panelHeight <= viewportHeight - 12; + var left; + var top; + var bridgeLeft; + var bridgeTop; + var bridgeWidth; + var bridgeHeight; + if (placeRight || placeLeft) { + left = placeRight + ? rect.right + gap + : rect.left - panelWidth - gap; + top = Math.max( + 12, + Math.min(rect.top, viewportHeight - panelHeight - 12) + ); + bridgeLeft = placeRight ? rect.right : left + panelWidth; + bridgeTop = Math.min(rect.top, top); + bridgeWidth = gap; + bridgeHeight = + Math.max(rect.bottom, top + panelHeight) - bridgeTop; + } else { + left = Math.max( + 12, + Math.min(rect.left, viewportWidth - panelWidth - 12) + ); + top = placeBelow + ? rect.bottom + gap + : rect.top - panelHeight - gap; + top = Math.max( + 12, + Math.min(top, viewportHeight - panelHeight - 12) + ); + bridgeLeft = Math.min(rect.left, left); + bridgeTop = placeBelow ? rect.bottom : top + panelHeight; + bridgeWidth = + Math.max(rect.right, left + panelWidth) - bridgeLeft; + bridgeHeight = gap; + } + voicePanel.style.left = Math.round(left) + 'px'; + voicePanel.style.top = Math.round(top) + 'px'; + voiceBridge.style.display = 'block'; + voiceBridge.style.left = Math.round(bridgeLeft) + 'px'; + voiceBridge.style.top = Math.round(bridgeTop) + 'px'; + voiceBridge.style.width = Math.round(bridgeWidth) + 'px'; + voiceBridge.style.height = Math.round(bridgeHeight) + 'px'; + } + + function openVoicePanel(pinned) { + clearVoiceTimers(); + createVoicePanel(); + if (pinned === true) voicePanelPinned = true; + voicePanelOpen = true; + voicePanel.style.display = 'block'; + asrContainer.setAttribute('aria-expanded', 'true'); + updateVoiceRecognitionUi(); + positionVoicePanel(); + } + + function closeVoicePanel(force) { + if (voicePanelPinned && force !== true) return; + clearVoiceTimers(); + voicePanelPinned = false; + voicePanelOpen = false; + if (voicePanel) voicePanel.style.display = 'none'; + if (voiceBridge) voiceBridge.style.display = 'none'; + asrContainer.setAttribute('aria-expanded', 'false'); + } + + function destroyVoicePanel() { + clearVoiceTimers(); + closeVoicePanel(true); + if (voicePopupObserver) { + voicePopupObserver.disconnect(); + voicePopupObserver = null; + } + document.removeEventListener('pointerdown', onVoiceDocumentPointerDown, true); + document.removeEventListener( + 'keydown', + onVoiceDocumentKeyDown, + true + ); + window.removeEventListener('resize', positionVoicePanel); + window.removeEventListener('scroll', positionVoicePanel, true); + window.removeEventListener( + 'voice-input-lifecycle-changed', + onVoiceLifecycleChanged + ); + window.removeEventListener( + 'neko:voice-session-started', + onVoiceSessionStarted + ); + window.removeEventListener( + 'neko:voice-settings-pending-changed', + onVoiceSettingsPendingChanged + ); + if (voicePanel) voicePanel.remove(); + if (voiceBridge) voiceBridge.remove(); + voicePanel = null; + voiceBridge = null; + noiseToggle = null; + optimizationToggle = null; + optimizationHint = null; + voiceStatus = null; + if (disposeVoiceRecognitionPopover === destroyVoicePanel) { + disposeVoiceRecognitionPopover = null; + } + } + + function scheduleVoiceOpen() { + if (voicePanelOpen) return; + if (voiceOpenTimer !== null) clearTimeout(voiceOpenTimer); + voiceOpenTimer = setTimeout(function () { + openVoicePanel(false); + }, 150); + } + + function scheduleVoiceClose() { + if (voicePanelPinned) return; + if (voiceOpenTimer !== null) clearTimeout(voiceOpenTimer); + if (voiceCloseTimer !== null) clearTimeout(voiceCloseTimer); + voiceCloseTimer = setTimeout(function () { + closeVoicePanel(false); + }, 300); + } + + function togglePinnedVoicePanel(focusPanel) { + if (voicePanelOpen && voicePanelPinned) closeVoicePanel(true); + else { + openVoicePanel(true); + if (focusPanel === true && voicePanel) { + var firstControl = voicePanel.querySelector( + 'input:not([disabled])' + ); + if (firstControl) firstControl.focus(); + } + } + } - asrRow.appendChild(asrLabel); - asrRow.appendChild(asrToggle); - asrContainer.appendChild(asrRow); + function onVoiceDocumentPointerDown(event) { + if (!voicePanelOpen || !voicePanel) return; + if ( + !asrContainer.contains(event.target) + && !voicePanel.contains(event.target) + ) closeVoicePanel(true); + } - var asrHint = document.createElement('div'); - // Single renderer, called at build time AND from the toggle's change - // handler above (function declarations hoist, so it is reachable - // there). The hint used to be computed once here, so flipping the - // switch with the popup open left "Using Omni native speech - // recognition" on screen after enabling -- and the inverse stale - // text after disabling -- until the popup was rebuilt: the - // confirmation contradicted the choice the user had just made - // (Codex P2). - renderAsrHint(); - Object.assign(asrHint.style, { fontSize: '11px', color: 'var(--neko-popup-text-sub)', marginTop: '6px' }); - asrContainer.appendChild(asrHint); - leftColumn.appendChild(asrContainer); + function onVoiceDocumentKeyDown(event) { + if (event.key === 'Escape' && voicePanelOpen) { + closeVoicePanel(true); + asrContainer.focus(); + } + } + + function onVoiceLifecycleChanged() { + updateVoiceRecognitionUi(); + } + + function onVoiceSessionStarted() { + if ( + S.voiceSettingsPendingUntilEpoch === null + || (Number(S.voiceSessionStartEpoch) || 0) + < S.voiceSettingsPendingUntilEpoch + ) return; + S.voiceSettingsPendingUntilEpoch = null; + S.pendingVoiceRouteIndependentAsr = null; + updateVoiceRecognitionUi(); + } + + function onVoiceSettingsPendingChanged() { + updateVoiceRecognitionUi(); + } + + asrContainer.addEventListener('mouseenter', scheduleVoiceOpen); + asrContainer.addEventListener('mouseleave', scheduleVoiceClose); + asrContainer.addEventListener('focusin', function () { + openVoicePanel(false); + }); + asrContainer.addEventListener('focusout', scheduleVoiceClose); + asrContainer.addEventListener('pointerup', function (event) { + if (event.button !== undefined && event.button !== 0) return; + togglePinnedVoicePanel(); + }); + asrContainer.addEventListener('keydown', function (event) { + if (event.target !== asrContainer) return; + if (event.key === 'Enter' || event.key === ' ') { + event.preventDefault(); + togglePinnedVoicePanel(true); + } + }); + disposeVoiceRecognitionPopover = destroyVoicePanel; + createVoicePanel(); var sep1b = document.createElement('div'); Object.assign(sep1b.style, { height: '1px', backgroundColor: 'var(--neko-popup-separator)', margin: '8px 0' }); @@ -3402,8 +3976,14 @@ if (typeof micPopup.__nekoMicScrollbarCleanup === 'function') { startMicVolumeVisualization(); return true; } catch (error) { - if (!isPopupAvailable()) return false; + if ( + renderGeneration !== voiceRecognitionPopoverRenderGeneration + || !isPopupAvailable() + ) return false; console.error('渲染麦克风列表失败:', error); + if (disposeVoiceRecognitionPopover) { + disposeVoiceRecognitionPopover(); + } micPopup.innerHTML = ''; var errorItem = document.createElement('div'); errorItem.textContent = window.t ? window.t('microphone.loadFailed') : '获取麦克风列表失败'; diff --git a/static/app/app-settings.js b/static/app/app-settings.js index a61033ce4a..cb04842d2a 100644 --- a/static/app/app-settings.js +++ b/static/app/app-settings.js @@ -83,42 +83,109 @@ ); return value <= maxAccepted; } - function _noteAsrDecision(writeId, writerId, value) { + function _noteSettingDecision( + current, + writeId, + writerId, + value, + isFreshChoice + ) { // A write that merely re-asserts the value already decided is not a new // choice: _dirtySettingsKeys is monotone, so every later save from a // window that once toggled declares the key explicit, and treating those // as fresh intent would shield this window from a genuinely newer toggle. - if (_lastAsrDecision && _lastAsrDecision.value === value) return; - if (_lastAsrDecision - && !(writeId > _lastAsrDecision.writeId - || (writeId === _lastAsrDecision.writeId && writerId > _lastAsrDecision.writerId))) { - return; + if ( + current + && current.value === value + && isFreshChoice !== true + ) return current; + if (current + && !(writeId > current.writeId + || (writeId === current.writeId && writerId > current.writerId))) { + return current; } - _lastAsrDecision = { writeId: writeId, writerId: writerId || '', value: value }; - _asrDecisionWriteIdFloor = Math.max(_asrDecisionWriteIdFloor, writeId); + return { writeId: writeId, writerId: writerId || '', value: value }; } - function _asrWriteOutranksLocalChoice(meta) { - if (!_lastAsrDecision) return true; + let _lastOptimizationDecision = null; + // Durable write-ahead bit for the optimization handshake. A localStorage + // decision can outlive the page that issued its POST; keep it authoritative + // across reloads until the exact decision is acknowledged by the server. + let _optimizationDecisionPendingSync = false; + function _noteAsrDecision( + writeId, + writerId, + value, + isFreshChoice + ) { + const nextDecision = _noteSettingDecision( + _lastAsrDecision, + writeId, + writerId, + value, + isFreshChoice + ); + _lastAsrDecision = nextDecision; + if (nextDecision) { + _asrDecisionWriteIdFloor = Math.max( + _asrDecisionWriteIdFloor, + nextDecision.writeId + ); + } + } + function _noteOptimizationDecision( + writeId, + writerId, + value, + isFreshChoice + ) { + _lastOptimizationDecision = _noteSettingDecision( + _lastOptimizationDecision, + writeId, + writerId, + value, + isFreshChoice + ); + } + function _settingWriteOutranksLocalChoice( + meta, + settingKey, + decisionKey, + localDecision + ) { + if (!localDecision) return true; // Order on the DECISION that produced this value, not on the id of the // write that happens to carry it. _dirtySettingsKeys is monotone and - // every save copies independentAsrEnabled, so once a window has toggled - // once, each later UNRELATED save re-declares the key explicit with a - // fresh id -- and would outrank a genuinely newer toggle in another - // window. That needs no race at all, which makes it strictly more - // reachable than the same-millisecond tie this ordering was added for. - const decision = meta.asrDecision - || (meta.changedKeys.indexOf('independentAsrEnabled') !== -1 ? meta : null); + // every save copies shared settings, so once a window has toggled once, + // each later unrelated save re-declares the dirty key with a fresh id. + const decision = meta[decisionKey] + || (meta.changedKeys.indexOf(settingKey) !== -1 ? meta : null); // No tuple AND not declared explicit: an incidental copy of whatever // this writer happened to hold. It must never outrank an explicit local // choice. Previous-build snapshots that DO declare the key keep today's // writeId ordering through the fallback above. if (!decision) return false; - if (decision.writeId > _lastAsrDecision.writeId) return true; - if (decision.writeId < _lastAsrDecision.writeId) return false; + if (decision.writeId > localDecision.writeId) return true; + if (decision.writeId < localDecision.writeId) return false; // Equal ids are unordered in time; break on the window-unique key so // both windows pick the SAME winner. An absent writerId (previous // build) reads as '' and loses, keeping this window's own choice. - return (decision.writerId || '') > _lastAsrDecision.writerId; + return (decision.writerId || '') > localDecision.writerId; + } + function _asrWriteOutranksLocalChoice(meta) { + return _settingWriteOutranksLocalChoice( + meta, + 'independentAsrEnabled', + 'asrDecision', + _lastAsrDecision + ); + } + function _optimizationWriteOutranksLocalChoice(meta) { + return _settingWriteOutranksLocalChoice( + meta, + 'voiceInputResourceOptimizationEnabled', + 'optimizationDecision', + _lastOptimizationDecision + ); } function _normalizeServerAsrDecision(value, serverAuthoritative) { if (!value || typeof value !== 'object') return null; @@ -484,6 +551,7 @@ 'focusCognitionEnabled', 'noiseReductionEnabled', 'independentAsrEnabled', + 'voiceInputResourceOptimizationEnabled', 'avatarReactionBubbleEnabled', 'slopFilterEnabled', 'proactiveChatInterval', @@ -510,7 +578,8 @@ focusModeEnabled: false, focusCognitionEnabled: true, noiseReductionEnabled: true, - independentAsrEnabled: false, + independentAsrEnabled: true, + voiceInputResourceOptimizationEnabled: true, avatarReactionBubbleEnabled: true, slopFilterEnabled: true, proactiveChatInterval: Number.isFinite(C.DEFAULT_PROACTIVE_CHAT_INTERVAL) @@ -569,6 +638,7 @@ focusCognitionEnabled: S.focusCognitionEnabled, noiseReductionEnabled: S.noiseReductionEnabled, independentAsrEnabled: S.independentAsrEnabled, + voiceInputResourceOptimizationEnabled: S.voiceInputResourceOptimizationEnabled, avatarReactionBubbleEnabled: S.avatarReactionBubbleEnabled, slopFilterEnabled: S.slopFilterEnabled, proactiveChatInterval: S.proactiveChatInterval, @@ -678,6 +748,12 @@ return keys; } + function _settingDivergedFromBaseline(snapshot, key) { + return !!_settingsBaseline + && Object.prototype.hasOwnProperty.call(_settingsBaseline, key) + && _settingsBaseline[key] !== snapshot[key]; + } + /** * Persist the shared settings snapshot with its write metadata. * `hydrated` records whether this window held ANY authoritative settings @@ -741,8 +817,27 @@ _noteAsrDecision( _nextAsrDecisionWriteId(ownMeta.writeId), ownMeta.writerId, - snapshot.independentAsrEnabled + snapshot.independentAsrEnabled, + _settingDivergedFromBaseline( + snapshot, + 'independentAsrEnabled' + ) + ); + } + if (ownMeta.changedKeys.indexOf('voiceInputResourceOptimizationEnabled') !== -1) { + const isFreshOptimizationChoice = _settingDivergedFromBaseline( + snapshot, + 'voiceInputResourceOptimizationEnabled' + ); + _noteOptimizationDecision( + ownMeta.writeId, + ownMeta.writerId, + snapshot.voiceInputResourceOptimizationEnabled, + isFreshOptimizationChoice ); + if (isFreshOptimizationChoice) { + _optimizationDecisionPendingSync = true; + } } // Stamp the ASR key with the id of the decision that PRODUCED this // value. _noteAsrDecision already refuses to advance the LOCAL decision @@ -759,6 +854,17 @@ value: _lastAsrDecision.value }; } + if (_lastOptimizationDecision + && _lastOptimizationDecision.value + === snapshot.voiceInputResourceOptimizationEnabled) { + ownMeta.optimizationDecision = { + writeId: _lastOptimizationDecision.writeId, + writerId: _lastOptimizationDecision.writerId, + value: _lastOptimizationDecision.value + }; + ownMeta.optimizationDecisionPendingSync = + _optimizationDecisionPendingSync === true; + } try { localStorage.setItem('project_neko_settings', JSON.stringify(payload)); } catch (error) { @@ -879,7 +985,25 @@ : '', value: meta.asrDecision.value } - : null + : null, + optimizationDecision: (meta.optimizationDecision + && _isValidAsrWriteId( + meta.optimizationDecision.writeId, + Number.isInteger(meta.serverRevision) + )) + ? { + writeId: meta.optimizationDecision.writeId, + writerId: typeof meta.optimizationDecision.writerId === 'string' + ? meta.optimizationDecision.writerId + : '', + value: meta.optimizationDecision.value + } + : null, + // Snapshots from the previous PR head already carry the decision + // tuple but not this bit. Treat those as pending so an interrupted + // POST cannot be forgotten during rollout. + optimizationDecisionPendingSync: !!meta.optimizationDecision + && meta.optimizationDecisionPendingSync !== false }; } @@ -1177,6 +1301,9 @@ // touched the ASR key, synchronously before any await so the very // next start_session already carries it. if (_dirtySettingsKeys.has('independentAsrEnabled')) S.independentAsrAuthoritative = true; + if (_dirtySettingsKeys.has('voiceInputResourceOptimizationEnabled')) { + S.voiceInputResourceOptimizationAuthoritative = true; + } } // Serialize the POST behind any in-flight sync (Codex P2): the // settings snapshot is built inside runSync, at SEND time — after the @@ -1229,6 +1356,17 @@ writerId: _lastAsrDecision.writerId, value: _lastAsrDecision.value } : null; + const optimizationDecisionAtSend = ( + Object.prototype.hasOwnProperty.call( + payload, + 'voiceInputResourceOptimizationEnabled' + ) + && _lastOptimizationDecision + && _lastOptimizationDecision.value + === payload.voiceInputResourceOptimizationEnabled + ) + ? Object.assign({}, _lastOptimizationDecision) + : null; const headers = { 'Content-Type': 'application/json' }; if (_conversationSettingsEtag) { headers['If-Match'] = _conversationSettingsEtag; @@ -1278,6 +1416,21 @@ console.error('[app-settings] 同步设置到服务器失败:', data.error || '未知错误'); return; } + if ( + optimizationDecisionAtSend + && _lastOptimizationDecision + && optimizationDecisionAtSend.writeId + === _lastOptimizationDecision.writeId + && optimizationDecisionAtSend.writerId + === _lastOptimizationDecision.writerId + && optimizationDecisionAtSend.value + === _lastOptimizationDecision.value + && S.voiceInputResourceOptimizationEnabled + === optimizationDecisionAtSend.value + ) { + _optimizationDecisionPendingSync = false; + _writeSharedSettings(getConversationSettings(), []); + } _confirmSharedKeyWrites( payload, data.settings, @@ -1447,6 +1600,8 @@ ? window.focusCognitionEnabled : S.focusCognitionEnabled; const currentIndependentAsr = S.independentAsrEnabled === true; + const currentVoiceResourceOptimization = + S.voiceInputResourceOptimizationEnabled !== false; const currentProactiveChatInterval = typeof window.proactiveChatInterval !== 'undefined' ? window.proactiveChatInterval : S.proactiveChatInterval; @@ -1520,6 +1675,7 @@ focusCognitionEnabled: currentFocusCognition, noiseReductionEnabled: S.noiseReductionEnabled, independentAsrEnabled: currentIndependentAsr, + voiceInputResourceOptimizationEnabled: currentVoiceResourceOptimization, avatarReactionBubbleEnabled: currentAvatarReactionBubble, slopFilterEnabled: currentSlopFilter, proactiveChatInterval: currentProactiveChatInterval, @@ -1561,6 +1717,7 @@ S.focusModeEnabled = currentFocus; S.focusCognitionEnabled = currentFocusCognition; S.independentAsrEnabled = currentIndependentAsr; + S.voiceInputResourceOptimizationEnabled = currentVoiceResourceOptimization; S.avatarReactionBubbleEnabled = currentAvatarReactionBubble; S.slopFilterEnabled = currentSlopFilter; S.proactiveChatInterval = currentProactiveChatInterval; @@ -1651,6 +1808,38 @@ const bootDecision = bootMeta.asrDecision || bootMeta; _noteAsrDecision(bootDecision.writeId, bootDecision.writerId, settings.independentAsrEnabled); } + if ( + bootMeta + && ( + bootMeta.optimizationDecision + || bootMeta.changedKeys.indexOf( + 'voiceInputResourceOptimizationEnabled' + ) !== -1 + ) + ) { + const optimizationDecision = + bootMeta.optimizationDecision || bootMeta; + _noteOptimizationDecision( + optimizationDecision.writeId, + optimizationDecision.writerId, + settings.voiceInputResourceOptimizationEnabled + ); + if ( + bootMeta.optimizationDecisionPendingSync + && typeof settings.voiceInputResourceOptimizationEnabled + === 'boolean' + ) { + _optimizationDecisionPendingSync = true; + _dirtySettingsKeys.add( + 'voiceInputResourceOptimizationEnabled' + ); + _pendingSettingsKeys.add( + 'voiceInputResourceOptimizationEnabled' + ); + S.settingsHydrated = true; + S.voiceInputResourceOptimizationAuthoritative = true; + } + } // 迁移逻辑：检测旧版设置并迁移到新字段 // 如果旧版 proactiveChatEnabled=true 但新字段未定义，则迁移 @@ -1707,7 +1896,9 @@ S.mergeMessagesEnabled = settings.mergeMessagesEnabled ?? false; S.focusModeEnabled = settings.focusModeEnabled ?? false; S.focusCognitionEnabled = settings.focusCognitionEnabled ?? true; - S.independentAsrEnabled = settings.independentAsrEnabled ?? false; + S.independentAsrEnabled = settings.independentAsrEnabled ?? true; + S.voiceInputResourceOptimizationEnabled = + settings.voiceInputResourceOptimizationEnabled ?? true; S.avatarReactionBubbleEnabled = settings.avatarReactionBubbleEnabled ?? true; S.slopFilterEnabled = settings.slopFilterEnabled ?? true; S.proactiveChatInterval = settings.proactiveChatInterval ?? C.DEFAULT_PROACTIVE_CHAT_INTERVAL; @@ -1871,6 +2062,7 @@ // now holds either server truth or a user change the field-level // merge preserved — authoritative for the handshake either way. S.independentAsrAuthoritative = true; + S.voiceInputResourceOptimizationAuthoritative = true; // Distinct from the hydration mark above (which a user action // also sets, because a user choice is authoritative for the // handshake even before any GET): THIS flag means server values @@ -2152,6 +2344,54 @@ ? (asrValueDiffers && asrMarkedExplicit && asrWriteIsNewer && asrOutranksLocalChoice) : asrValueDiffers; + const optimizationKey = 'voiceInputResourceOptimizationEnabled'; + const optimizationValueDiffers = + Object.prototype.hasOwnProperty.call(settings, optimizationKey) + && S[optimizationKey] !== settings[optimizationKey]; + const optimizationMarkedExplicit = !!meta + && meta.changedKeys.indexOf(optimizationKey) !== -1; + const optimizationWriteIsNewer = !meta + || meta.writeId > _lastAppliedSharedWriteId + || ( + meta.writeId === _lastAppliedSharedWriteId + && optimizationMarkedExplicit + ); + const optimizationOutranksLocalChoice = + !meta || _optimizationWriteOutranksLocalChoice(meta); + // Like independentAsrEnabled, this key is copied into every shared + // snapshot. With metadata, only an explicit user change may alter + // another window; otherwise an unhydrated writer's boot default + // could overwrite a server-merged preference incidentally. + const optimizationChangedByOtherWindow = meta + ? ( + optimizationValueDiffers + && optimizationMarkedExplicit + && optimizationWriteIsNewer + && optimizationOutranksLocalChoice + ) + : optimizationValueDiffers; + const optimizationSyncAcknowledgesLocalDecision = !!meta + && !!meta.optimizationDecision + && meta.optimizationDecisionPendingSync === false + && !!_lastOptimizationDecision + && meta.optimizationDecision.writeId + === _lastOptimizationDecision.writeId + && meta.optimizationDecision.writerId + === _lastOptimizationDecision.writerId + && meta.optimizationDecision.value + === _lastOptimizationDecision.value; + if (optimizationSyncAcknowledgesLocalDecision) { + _optimizationDecisionPendingSync = false; + } + const activeRouteBeforeSharedVoiceChange = S.voiceChatActive === true + ? ( + S.independentAsrActive === true + || ( + S.voiceInputLifecycleState === 'blocked' + && S.independentAsrEnabled === true + ) + ) + : null; // Drop the key from the apply set when the snapshot's ASR value // carries neither user intent nor trustworthy server truth: an // already-superseded write, or one made before its own window @@ -2163,13 +2403,20 @@ && !asrChangedByOtherWindow && (!asrWriteIsNewer || !asrOutranksLocalChoice || (!meta.asrAuthoritative && S.settingsHydrated === true)); + const optimizationValueIsStale = !!meta + && optimizationValueDiffers + && ( + !optimizationChangedByOtherWindow + || !optimizationOutranksLocalChoice + ); if (meta && meta.writeId > _lastAppliedSharedWriteId) { _lastAppliedSharedWriteId = meta.writeId; } let incoming = settings; - if (asrValueIsStale) { + if (asrValueIsStale || optimizationValueIsStale) { incoming = Object.assign({}, settings); - delete incoming.independentAsrEnabled; + if (asrValueIsStale) delete incoming.independentAsrEnabled; + if (optimizationValueIsStale) delete incoming[optimizationKey]; } if (meta) { for (const key of meta.changedKeys) { @@ -2399,6 +2646,41 @@ _noteAsrDecision(adopted.writeId, adopted.writerId, settings.independentAsrEnabled); } } + if (optimizationChangedByOtherWindow) { + // Preserve a genuine cross-window toggle across this window's + // still-pending server merge without granting unrelated fields + // handshake authority or emitting a duplicate POST. + S.settingsHydrated = true; + S.voiceInputResourceOptimizationAuthoritative = true; + _dirtySettingsKeys.add(optimizationKey); + _optimizationDecisionPendingSync = !meta + || meta.optimizationDecisionPendingSync; + if (meta) { + const adopted = meta.optimizationDecision || meta; + _noteOptimizationDecision( + adopted.writeId, + adopted.writerId, + settings[optimizationKey] + ); + } + } + if (asrChangedByOtherWindow || optimizationChangedByOtherWindow) { + const targetEpoch = (Number(S.voiceSessionStartEpoch) || 0) + 1; + if ( + asrChangedByOtherWindow + && ( + S.voiceSettingsPendingUntilEpoch !== targetEpoch + || S.pendingVoiceRouteIndependentAsr === null + ) + ) { + S.pendingVoiceRouteIndependentAsr = + activeRouteBeforeSharedVoiceChange; + } + S.voiceSettingsPendingUntilEpoch = targetEpoch; + window.dispatchEvent(new CustomEvent( + 'neko:voice-settings-pending-changed' + )); + } stopVisionAfterPrivacyEnabled(); if (changed && typeof window.scheduleProactiveChat === 'function') { window.scheduleProactiveChat(); diff --git a/static/app/app-state.js b/static/app/app-state.js index 0fe26c6fa6..8c32acc12d 100644 --- a/static/app/app-state.js +++ b/static/app/app-state.js @@ -88,17 +88,26 @@ selectedMicrophoneId: null, microphoneGainDb: 0, noiseReductionEnabled: true, - independentAsrEnabled: false, + independentAsrEnabled: true, + voiceInputResourceOptimizationEnabled: true, // 设置是否已"水合"：server GET 合并成功或用户显式改过设置后才为 true。 - // 在此之前 S.independentAsrEnabled 只是启动默认值（false），不代表权威偏好， + // 在此之前两个 true 都只是启动默认值，不代表服务器权威偏好； + // independentAsrEnabled 尤其不能提前进入会话握手， // start_session 握手（app-websocket.js attachStartSessionHandshake）不得携带它， - // 否则新浏览器 profile 首个会话会用默认 false 覆盖后端持久化的 true。 + // 否则新浏览器 profile 首个会话会覆盖后端持久化的显式 false。 settingsHydrated: false, // independentAsrEnabled 的按键权威位：settingsHydrated 在任何一次用户改 // 设置时都会翻真，而那与 ASR 的值毫无关系。只有「server GET 合并成功」 // 「用户显式改过 ASR 开关」「跨窗口 ASR 翻转」这三种事件才让它变权威； // 在此之前 start_session 握手必须省略该字段，由后端持久化值兜底。 independentAsrAuthoritative: false, + // 资源优化同样参与会话路由启动，必须独立证明该键来自 server merge、 + // 本窗口显式修改或可信的跨窗口修改，不能用启动默认值覆盖持久化选择。 + voiceInputResourceOptimizationAuthoritative: false, + // 跨 popup generation 保存「下次会话生效」状态与当前会话的实际 ASR + // route。否则跨窗口设置事件更新偏好后，重渲染会把偏好误报成当前 route。 + voiceSettingsPendingUntilEpoch: null, + pendingVoiceRouteIndependentAsr: null, // 独立 ASR 已 fail-closed 的粘性标记：blocked 生命周期事件只发一次， // 而游戏 STT 网关持有麦克风时会跳过停麦，退出游戏的恢复路径必须据此 // 拒绝把麦克风重新开到一条仍然关闭的路由上。 diff --git a/static/app/app-websocket.js b/static/app/app-websocket.js index 6b5f090865..2ced7c9abf 100644 --- a/static/app/app-websocket.js +++ b/static/app/app-websocket.js @@ -1583,8 +1583,16 @@ if (typeof data === 'string' && data.indexOf('start_session') !== -1) { try { var msg = JSON.parse(data); + var handshakeStamped = false; if (msg && msg.action === 'start_session' && S.settingsHydrated === true && S.independentAsrAuthoritative === true) { msg.independent_asr_enabled = S.independentAsrEnabled === true; + handshakeStamped = true; + } + if (msg && msg.action === 'start_session' && S.settingsHydrated === true && S.voiceInputResourceOptimizationAuthoritative === true) { + msg.voice_input_resource_optimization_enabled = S.voiceInputResourceOptimizationEnabled !== false; + handshakeStamped = true; + } + if (handshakeStamped) { data = JSON.stringify(msg); } } catch (e) { diff --git a/static/locales/en.json b/static/locales/en.json index 3bac3312f3..724b40fd38 100644 --- a/static/locales/en.json +++ b/static/locales/en.json @@ -929,14 +929,25 @@ "volumeWaiting": "Waiting", "volumeHintWaiting": "Mic is listening, go ahead and speak", "noiseReduction": "Noise Reduction", - "noiseReductionHint": "RNNoise AI noise reduction for clearer voice", - "independentAsr": "Independent Speech Recognition", - "independentAsrActive": "Independent ASR active: {{provider}}", + "noiseReductionHint": "Makes your voice input clearer", + "independentAsr": "Speech Recognition", + "independentAsrSummary": "Independent ASR · {{provider}}", + "independentAsrSummaryGeneric": "Independent ASR", + "independentAsrActive": "Independent ASR active: {{providerKey}}", "independentAsrFallback": "Independent ASR unavailable. Voice input has stopped for this session. Check the independent ASR configuration, then start a new voice session.", "audioPreprocessingFailed": "Microphone audio processing failed. Voice input has stopped for this session. Please start a new voice session.", - "independentAsrProviderUnavailable": "{{provider}} is temporarily unavailable. Voice input has stopped for this session. It did not switch to another speech recognition service. Please start a new voice session later.", + "independentAsrProviderUnavailable": "{{providerKey}} is temporarily unavailable. Voice input has stopped for this session. It did not switch to another speech recognition service. Please start a new voice session later.", "independentAsrNextSession": "Enabled for the next voice session; it will not automatically switch to Omni if unavailable.", - "independentAsrNative": "Using Omni native speech recognition" + "independentAsrNative": "Using Omni native speech recognition", + "voiceRecognitionSettings": "Speech Recognition Settings", + "voiceRecognitionDisabled": "Using Omni native speech recognition", + "voiceRecognitionDisabledHint": "Independent ASR is off; voice input uses Omni native speech recognition", + "voiceRecognitionUnavailable": "Independent speech recognition is unavailable. Voice input has stopped for this session", + "voiceRecognitionStatusReady": "● Running normally", + "voiceRecognitionSettingsPending": "◐ Settings take effect in the next voice session", + "voiceResourceOptimization": "Smart Resource Optimization", + "voiceResourceOptimizationHintOn": "Reduces connections and audio uploads while idle", + "voiceResourceOptimizationHintOff": "Keeps speech recognition active and may use more network and system resources" }, "speaker": { "volumeLabel": "Speaker Volume", diff --git a/static/locales/es.json b/static/locales/es.json index da2a4a629d..f192b8ef1a 100644 --- a/static/locales/es.json +++ b/static/locales/es.json @@ -929,14 +929,25 @@ "volumeWaiting": "En espera", "volumeHintWaiting": "El micrófono está escuchando, puede hablar", "noiseReduction": "Reducción de ruido", - "noiseReductionHint": "Reducción de ruido RNNoise AI para una voz más clara", - "independentAsr": "Reconocimiento de voz independiente", - "independentAsrActive": "ASR independiente activo: {{provider}}", + "noiseReductionHint": "Hace que la entrada de voz sea más clara", + "independentAsr": "Reconocimiento de voz", + "independentAsrSummary": "ASR independiente · {{provider}}", + "independentAsrSummaryGeneric": "ASR independiente", + "independentAsrActive": "ASR independiente activo: {{providerKey}}", "independentAsrFallback": "El ASR independiente no está disponible. La entrada de voz se ha detenido para esta sesión. Revisa la configuración del ASR independiente y después inicia una nueva sesión de voz.", "audioPreprocessingFailed": "Error al procesar el audio del micrófono. La entrada de voz se ha detenido en esta sesión. Inicia una nueva sesión de voz.", - "independentAsrProviderUnavailable": "{{provider}} no está disponible temporalmente. La entrada de voz se ha detenido para esta sesión. No se cambió a otro servicio de reconocimiento de voz. Inicia una nueva sesión de voz más tarde.", + "independentAsrProviderUnavailable": "{{providerKey}} no está disponible temporalmente. La entrada de voz se ha detenido para esta sesión. No se cambió a otro servicio de reconocimiento de voz. Inicia una nueva sesión de voz más tarde.", "independentAsrNextSession": "Se activará en la próxima sesión de voz; no cambiará automáticamente a Omni si no está disponible.", - "independentAsrNative": "Usando el reconocimiento de voz nativo de Omni" + "independentAsrNative": "Usando el reconocimiento de voz nativo de Omni", + "voiceRecognitionSettings": "Configuración de reconocimiento de voz", + "voiceRecognitionDisabled": "Usando el reconocimiento de voz nativo de Omni", + "voiceRecognitionDisabledHint": "El ASR independiente está desactivado; la entrada de voz usa el reconocimiento de voz nativo de Omni", + "voiceRecognitionUnavailable": "El reconocimiento de voz independiente no está disponible. La entrada de voz de esta sesión se ha detenido", + "voiceRecognitionStatusReady": "● Funciona con normalidad", + "voiceRecognitionSettingsPending": "◐ La configuración se aplicará en la próxima sesión de voz", + "voiceResourceOptimization": "Optimización inteligente de recursos", + "voiceResourceOptimizationHintOn": "Reduce las conexiones y la carga de audio cuando está inactivo", + "voiceResourceOptimizationHintOff": "Mantiene activo el reconocimiento de voz y puede aumentar el uso de red y recursos" }, "speaker": { "volumeLabel": "Volumen del altavoz", diff --git a/static/locales/ja.json b/static/locales/ja.json index 31706ee843..32fe2aa8b4 100644 --- a/static/locales/ja.json +++ b/static/locales/ja.json @@ -929,14 +929,25 @@ "volumeWaiting": "待機中", "volumeHintWaiting": "マイクが入力を待っています、話してください", "noiseReduction": "ノイズリダクション", - "noiseReductionHint": "RNNoise AI ノイズリダクションで音声をクリアに", - "independentAsr": "独立音声認識", - "independentAsrActive": "独立 ASR を使用中：{{provider}}", + "noiseReductionHint": "入力音声をより明瞭にします", + "independentAsr": "音声認識", + "independentAsrSummary": "独立 ASR · {{provider}}", + "independentAsrSummaryGeneric": "独立 ASR", + "independentAsrActive": "独立 ASR を使用中：{{providerKey}}", "independentAsrFallback": "独立 ASR を利用できないため、この音声セッションの入力を停止しました。独立 ASR の設定を確認してから、新しい音声セッションを開始してください。", "audioPreprocessingFailed": "マイク音声の処理に失敗しました。このセッションの音声入力は停止しました。新しい音声セッションを開始してください。", - "independentAsrProviderUnavailable": "{{provider}} は一時的に利用できません。この音声セッションの入力を停止しました。別の音声認識サービスには切り替えていません。後でもう一度音声セッションを開始してください。", + "independentAsrProviderUnavailable": "{{providerKey}} は一時的に利用できません。この音声セッションの入力を停止しました。別の音声認識サービスには切り替えていません。後でもう一度音声セッションを開始してください。", "independentAsrNextSession": "次の音声セッションから有効になります。利用できない場合も Omni へ自動的に切り替わりません。", - "independentAsrNative": "Omni 標準音声認識を使用中" + "independentAsrNative": "Omni ネイティブ音声認識を使用中", + "voiceRecognitionSettings": "音声認識設定", + "voiceRecognitionDisabled": "Omni ネイティブ音声認識を使用中", + "voiceRecognitionDisabledHint": "独立 ASR はオフです。音声入力には Omni ネイティブ音声認識を使用します", + "voiceRecognitionUnavailable": "独立音声認識を利用できないため、このセッションの音声入力を停止しました", + "voiceRecognitionStatusReady": "● 正常に動作中", + "voiceRecognitionSettingsPending": "◐ 設定は次の音声セッションから反映されます", + "voiceResourceOptimization": "スマートリソース最適化", + "voiceResourceOptimizationHintOn": "待機中の接続と音声アップロードを減らします", + "voiceResourceOptimizationHintOff": "音声認識を常時維持するため、通信量とリソース使用量が増える場合があります" }, "speaker": { "volumeLabel": "スピーカー音量", diff --git a/static/locales/ko.json b/static/locales/ko.json index a3c2de5542..dfa46f29f8 100644 --- a/static/locales/ko.json +++ b/static/locales/ko.json @@ -929,14 +929,25 @@ "volumeWaiting": "대기 중", "volumeHintWaiting": "마이크가 입력을 대기 중입니다. 말씀해 주세요.", "noiseReduction": "노이즈 리덕션", - "noiseReductionHint": "RNNoise AI 노이즈 리덕션으로 더 선명한 음성", - "independentAsr": "독립 음성 인식", - "independentAsrActive": "독립 ASR 사용 중: {{provider}}", + "noiseReductionHint": "입력 음성을 더 선명하게 만듭니다", + "independentAsr": "음성 인식", + "independentAsrSummary": "독립 ASR · {{provider}}", + "independentAsrSummaryGeneric": "독립 ASR", + "independentAsrActive": "독립 ASR 사용 중: {{providerKey}}", "independentAsrFallback": "독립 ASR을 사용할 수 없어 이번 음성 세션의 입력을 중지했습니다. 독립 ASR 설정을 확인한 다음 새 음성 세션을 시작하세요.", "audioPreprocessingFailed": "마이크 오디오 처리에 실패했습니다. 이번 세션의 음성 입력이 중지되었습니다. 새 음성 세션을 시작해 주세요.", - "independentAsrProviderUnavailable": "{{provider}}을(를) 일시적으로 사용할 수 없어 이번 음성 세션의 입력을 중지했습니다. 다른 음성 인식 서비스로 전환하지 않았습니다. 나중에 새 음성 세션을 시작하세요.", + "independentAsrProviderUnavailable": "{{providerKey}}을(를) 일시적으로 사용할 수 없어 이번 음성 세션의 입력을 중지했습니다. 다른 음성 인식 서비스로 전환하지 않았습니다. 나중에 새 음성 세션을 시작하세요.", "independentAsrNextSession": "다음 음성 세션부터 활성화되며, 사용할 수 없어도 Omni로 자동 전환되지 않습니다.", - "independentAsrNative": "Omni 기본 음성 인식 사용 중" + "independentAsrNative": "Omni 네이티브 음성 인식 사용 중", + "voiceRecognitionSettings": "음성 인식 설정", + "voiceRecognitionDisabled": "Omni 네이티브 음성 인식 사용 중", + "voiceRecognitionDisabledHint": "독립 ASR이 꺼져 있습니다. 음성 입력에는 Omni 네이티브 음성 인식을 사용합니다", + "voiceRecognitionUnavailable": "독립 음성 인식을 사용할 수 없어 이번 세션의 음성 입력이 중지되었습니다", + "voiceRecognitionStatusReady": "● 정상 작동 중", + "voiceRecognitionSettingsPending": "◐ 설정은 다음 음성 세션부터 적용됩니다", + "voiceResourceOptimization": "스마트 리소스 최적화", + "voiceResourceOptimizationHintOn": "유휴 상태에서 연결과 오디오 업로드를 줄입니다", + "voiceResourceOptimizationHintOff": "음성 인식을 계속 유지하므로 네트워크 및 리소스 사용량이 늘 수 있습니다" }, "speaker": { "volumeLabel": "스피커 볼륨", diff --git a/static/locales/pt.json b/static/locales/pt.json index f22343b8d1..523dcd1089 100644 --- a/static/locales/pt.json +++ b/static/locales/pt.json @@ -929,14 +929,25 @@ "volumeWaiting": "Aguardando", "volumeHintWaiting": "O microfone está ouvindo, pode falar", "noiseReduction": "Redução de ruído", - "noiseReductionHint": "Redução de ruído RNNoise AI para voz mais clara", - "independentAsr": "Reconhecimento de voz independente", - "independentAsrActive": "ASR independente ativo: {{provider}}", + "noiseReductionHint": "Deixa a entrada de voz mais clara", + "independentAsr": "Reconhecimento de voz", + "independentAsrSummary": "ASR independente · {{provider}}", + "independentAsrSummaryGeneric": "ASR independente", + "independentAsrActive": "ASR independente ativo: {{providerKey}}", "independentAsrFallback": "O ASR independente não está disponível. A entrada de voz foi interrompida nesta sessão. Verifique a configuração do ASR independente e depois inicie uma nova sessão de voz.", "audioPreprocessingFailed": "Falha ao processar o áudio do microfone. A entrada de voz foi interrompida nesta sessão. Inicie uma nova sessão de voz.", - "independentAsrProviderUnavailable": "{{provider}} está temporariamente indisponível. A entrada de voz foi interrompida nesta sessão. O sistema não mudou para outro serviço de reconhecimento de voz. Inicie uma nova sessão de voz mais tarde.", + "independentAsrProviderUnavailable": "{{providerKey}} está temporariamente indisponível. A entrada de voz foi interrompida nesta sessão. O sistema não mudou para outro serviço de reconhecimento de voz. Inicie uma nova sessão de voz mais tarde.", "independentAsrNextSession": "Será ativado na próxima sessão de voz; não mudará automaticamente para o Omni se estiver indisponível.", - "independentAsrNative": "Usando o reconhecimento de voz nativo do Omni" + "independentAsrNative": "Usando o reconhecimento de voz nativo do Omni", + "voiceRecognitionSettings": "Configurações de reconhecimento de voz", + "voiceRecognitionDisabled": "Usando o reconhecimento de voz nativo do Omni", + "voiceRecognitionDisabledHint": "O ASR independente está desativado; a entrada de voz usa o reconhecimento de voz nativo do Omni", + "voiceRecognitionUnavailable": "O reconhecimento de voz independente está indisponível. A entrada de voz desta sessão foi interrompida", + "voiceRecognitionStatusReady": "● Funcionando normalmente", + "voiceRecognitionSettingsPending": "◐ As configurações terão efeito na próxima sessão de voz", + "voiceResourceOptimization": "Otimização inteligente de recursos", + "voiceResourceOptimizationHintOn": "Reduz conexões e envio de áudio durante a inatividade", + "voiceResourceOptimizationHintOff": "Mantém o reconhecimento de voz ativo e pode aumentar o uso de rede e recursos" }, "speaker": { "volumeLabel": "Volume do alto-falante", diff --git a/static/locales/ru.json b/static/locales/ru.json index a33abbb5d6..110ef94f21 100644 --- a/static/locales/ru.json +++ b/static/locales/ru.json @@ -929,14 +929,25 @@ "volumeWaiting": "Ожидание", "volumeHintWaiting": "Микрофон слушает, можете говорить", "noiseReduction": "Шумоподавление", - "noiseReductionHint": "Шумоподавление RNNoise AI для более чистого голоса", - "independentAsr": "Независимое распознавание речи", - "independentAsrActive": "Активен независимый ASR: {{provider}}", + "noiseReductionHint": "Делает входящий голос более чистым", + "independentAsr": "Распознавание речи", + "independentAsrSummary": "Независимый ASR · {{provider}}", + "independentAsrSummaryGeneric": "Независимый ASR", + "independentAsrActive": "Активен независимый ASR: {{providerKey}}", "independentAsrFallback": "Независимый ASR недоступен. Голосовой ввод в этом сеансе остановлен. Проверьте настройки независимого ASR, затем начните новый голосовой сеанс.", "audioPreprocessingFailed": "Не удалось обработать звук микрофона. Голосовой ввод в этой сессии остановлен. Начните новую голосовую сессию.", - "independentAsrProviderUnavailable": "{{provider}} временно недоступен. Голосовой ввод в этом сеансе остановлен. Переключения на другую службу распознавания речи не произошло. Начните новый голосовой сеанс позже.", + "independentAsrProviderUnavailable": "{{providerKey}} временно недоступен. Голосовой ввод в этом сеансе остановлен. Переключения на другую службу распознавания речи не произошло. Начните новый голосовой сеанс позже.", "independentAsrNextSession": "Будет включён в следующем голосовом сеансе; при недоступности автоматического переключения на Omni не произойдёт.", - "independentAsrNative": "Используется встроенное распознавание речи Omni" + "independentAsrNative": "Используется встроенное распознавание речи Omni", + "voiceRecognitionSettings": "Настройки распознавания речи", + "voiceRecognitionDisabled": "Используется встроенное распознавание речи Omni", + "voiceRecognitionDisabledHint": "Независимый ASR выключен; для голосового ввода используется встроенное распознавание речи Omni", + "voiceRecognitionUnavailable": "Независимое распознавание речи недоступно. Голосовой ввод в этом сеансе остановлен", + "voiceRecognitionStatusReady": "● Работает нормально", + "voiceRecognitionSettingsPending": "◐ Настройки вступят в силу в следующем голосовом сеансе", + "voiceResourceOptimization": "Умная оптимизация ресурсов", + "voiceResourceOptimizationHintOn": "Сокращает подключения и отправку аудио во время простоя", + "voiceResourceOptimizationHintOff": "Поддерживает распознавание речи постоянно и может увеличить расход сети и ресурсов" }, "speaker": { "volumeLabel": "Громкость динамика", diff --git a/static/locales/zh-CN.json b/static/locales/zh-CN.json index 9198d41c65..3485a87246 100644 --- a/static/locales/zh-CN.json +++ b/static/locales/zh-CN.json @@ -929,14 +929,25 @@ "volumeWaiting": "等待声音", "volumeHintWaiting": "麦克风正在监听，请说话", "noiseReduction": "降噪", - "noiseReductionHint": "RNNoise AI 降噪，让语音更清晰", - "independentAsr": "独立语音识别", - "independentAsrActive": "当前使用独立 ASR：{{provider}}", + "noiseReductionHint": "让输入语音更加清晰", + "independentAsr": "语音识别", + "independentAsrSummary": "独立 ASR · {{provider}}", + "independentAsrSummaryGeneric": "独立 ASR", + "independentAsrActive": "当前使用独立 ASR：{{providerKey}}", "independentAsrFallback": "独立 ASR 不可用，本次语音输入已停止。请检查独立 ASR 配置，然后重新开始语音会话。", "audioPreprocessingFailed": "麦克风音频处理失败，本次会话的语音输入已停止。请重新开始一次语音会话。", - "independentAsrProviderUnavailable": "{{provider}} 暂时不可用，本次语音输入已停止。未切换到其他语音识别服务，请稍后重新开始语音会话。", + "independentAsrProviderUnavailable": "{{providerKey}} 暂时不可用，本次语音输入已停止。未切换到其他语音识别服务，请稍后重新开始语音会话。", "independentAsrNextSession": "将在下次语音会话启用；不可用时不会自动切换到 Omni。", - "independentAsrNative": "当前使用 Omni 原生语音识别" + "independentAsrNative": "当前使用 Omni 原生语音识别", + "voiceRecognitionSettings": "语音识别设置", + "voiceRecognitionDisabled": "当前使用 Omni 原生语音识别", + "voiceRecognitionDisabledHint": "独立 ASR 已关闭；语音输入使用 Omni 原生语音识别", + "voiceRecognitionUnavailable": "独立语音识别不可用，本次语音输入已停止", + "voiceRecognitionStatusReady": "● 当前运行正常", + "voiceRecognitionSettingsPending": "◐ 设置将在下次语音会话生效", + "voiceResourceOptimization": "智能资源优化", + "voiceResourceOptimizationHintOn": "空闲时减少连接和音频上传", + "voiceResourceOptimizationHintOff": "持续保持语音识别，可能增加网络和资源占用" }, "speaker": { "volumeLabel": "扬声器音量", diff --git a/static/locales/zh-TW.json b/static/locales/zh-TW.json index 671bbc8bd8..8f6fd0a584 100644 --- a/static/locales/zh-TW.json +++ b/static/locales/zh-TW.json @@ -929,14 +929,25 @@ "volumeWaiting": "等待聲音", "volumeHintWaiting": "麥克風正在監聽，請說話", "noiseReduction": "降噪", - "noiseReductionHint": "RNNoise AI 降噪，讓語音更清晰", - "independentAsr": "獨立語音辨識", - "independentAsrActive": "目前使用獨立 ASR：{{provider}}", + "noiseReductionHint": "讓輸入語音更加清晰", + "independentAsr": "語音辨識", + "independentAsrSummary": "獨立 ASR · {{provider}}", + "independentAsrSummaryGeneric": "獨立 ASR", + "independentAsrActive": "目前使用獨立 ASR：{{providerKey}}", "independentAsrFallback": "獨立 ASR 無法使用，本次語音輸入已停止。請檢查獨立 ASR 設定，然後重新開始語音會話。", "audioPreprocessingFailed": "麥克風音訊處理失敗，本次工作階段的語音輸入已停止。請重新開始一次語音工作階段。", - "independentAsrProviderUnavailable": "{{provider}} 暫時無法使用，本次語音輸入已停止。未切換到其他語音辨識服務，請稍後重新開始語音會話。", + "independentAsrProviderUnavailable": "{{providerKey}} 暫時無法使用，本次語音輸入已停止。未切換到其他語音辨識服務，請稍後重新開始語音會話。", "independentAsrNextSession": "將於下次語音會話啟用；無法使用時不會自動切換到 Omni。", - "independentAsrNative": "目前使用 Omni 原生語音辨識" + "independentAsrNative": "目前使用 Omni 原生語音辨識", + "voiceRecognitionSettings": "語音辨識設定", + "voiceRecognitionDisabled": "目前使用 Omni 原生語音辨識", + "voiceRecognitionDisabledHint": "獨立 ASR 已關閉；語音輸入使用 Omni 原生語音辨識", + "voiceRecognitionUnavailable": "獨立語音辨識無法使用，本次語音輸入已停止", + "voiceRecognitionStatusReady": "● 目前運作正常", + "voiceRecognitionSettingsPending": "◐ 設定將於下次語音會話生效", + "voiceResourceOptimization": "智慧資源最佳化", + "voiceResourceOptimizationHintOn": "閒置時減少連線和音訊上傳", + "voiceResourceOptimizationHintOff": "持續保持語音辨識，可能增加網路和資源用量" }, "speaker": { "volumeLabel": "揚聲器音量", diff --git a/tests/frontend/test_voice_recognition_popover.py b/tests/frontend/test_voice_recognition_popover.py new file mode 100644 index 0000000000..39295d65f5 --- /dev/null +++ b/tests/frontend/test_voice_recognition_popover.py @@ -0,0 +1,703 @@ +from pathlib import Path + +import pytest +from playwright.sync_api import Page + + +ROOT = Path(__file__).resolve().parents[2] +APP_AUDIO_CAPTURE = ROOT / "static" / "app" / "app-audio-capture.js" +VOICE_POPOVER_GLOBAL_LISTENERS = ( + "document:pointerdown", + "document:keydown", + "window:resize", + "window:scroll", + "window:voice-input-lifecycle-changed", + "window:neko:voice-session-started", + "window:neko:voice-settings-pending-changed", +) + + +def _voice_popover_sources() -> tuple[str, str]: + source = APP_AUDIO_CAPTURE.read_text(encoding="utf-8") + + permission_start = source.index("async function ensureMicrophonePermission()") + permission_end = source.index("// 监听设备变化", permission_start) + permission_source = source[permission_start:permission_end].strip() + + render_marker = "window.renderFloatingMicList = async function" + render_start = source.index(render_marker) + render_end = source.index( + "/** 轻量级更新：仅更新选中状态 */", render_start + ) + render_assignment = source[render_start:render_end].strip() + render_expression = render_assignment.split("=", 1)[1].strip() + if not render_expression.endswith(";"): + raise AssertionError("renderFloatingMicList assignment is not terminated") + return permission_source, render_expression[:-1] + + +def _install_voice_popover_harness( + page: Page, *, deferred_permission: bool +) -> None: + permission_source, render_expression = _voice_popover_sources() + page.set_content( + '
' + ) + + harness = r""" +(() => { + const listenerBalance = Object.create(null); + let failWindowListenerType = null; + function trackListeners(target, prefix) { + const originalAdd = target.addEventListener.bind(target); + const originalRemove = target.removeEventListener.bind(target); + target.addEventListener = function (type, listener, options) { + const key = prefix + ':' + type; + listenerBalance[key] = (listenerBalance[key] || 0) + 1; + const result = originalAdd(type, listener, options); + if (prefix === 'window' && type === failWindowListenerType) { + failWindowListenerType = null; + throw new Error('forced voice panel setup failure'); + } + return result; + }; + target.removeEventListener = function (type, listener, options) { + const key = prefix + ':' + type; + listenerBalance[key] = (listenerBalance[key] || 0) - 1; + return originalRemove(type, listener, options); + }; + } + trackListeners(document, 'document'); + trackListeners(window, 'window'); + const capturedErrors = []; + const originalConsoleError = console.error.bind(console); + console.error = (...args) => { + capturedErrors.push(args.map((value) => String(value)).join(' ')); + originalConsoleError(...args); + }; + + const mediaResolvers = []; + const stream = { getTracks: () => [{ stop() {} }] }; + Object.defineProperty(navigator, 'mediaDevices', { + configurable: true, + value: { + getUserMedia() { + if (!__DEFERRED_PERMISSION__) return Promise.resolve(stream); + return new Promise((resolve, reject) => { + mediaResolvers.push({ resolve, reject }); + }); + }, + enumerateDevices() { + return Promise.resolve([ + { kind: 'audioinput', deviceId: 'test-mic' }, + ]); + }, + addEventListener() {}, + }, + }); + + const S = { + speakerVolume: 100, + speakerGainNode: null, + spatialAudioEnabled: true, + independentAsrEnabled: true, + independentAsrActive: true, + independentAsrProvider: 'qwen', + voiceInputResourceOptimizationEnabled: true, + voiceInputLifecycleState: 'active', + voiceSessionStartEpoch: 10, + voiceSettingsPendingUntilEpoch: null, + pendingVoiceRouteIndependentAsr: null, + voiceChatActive: false, + noiseReductionEnabled: true, + microphoneGainDb: 0, + micGainNode: null, + selectedMicrophoneId: null, + }; + const C = { + DEFAULT_SPEAKER_VOLUME: 100, + MIN_MIC_GAIN_DB: -5, + MAX_MIC_GAIN_DB: 25, + }; + window.appState = S; + window.appConst = C; + window.appUtils = { + dbToLinear: (value) => value, + valueToKneeTrack: (value) => value, + kneeTrackToValue: (value) => value, + }; + window.appSpatialAudio = { + getEnabled: () => S.spatialAudioEnabled, + setEnabled: (enabled) => { S.spatialAudioEnabled = enabled; }, + }; + window.appSettings = { saveSettings: () => { window.__saveCalls += 1; } }; + window.__saveCalls = 0; + window.t = (key) => key; + + function formatGainDisplay(value) { return String(value); } + function saveSpeakerVolumeSetting() {} + function saveNoiseReductionSetting() {} + function saveMicGainSetting() {} + async function selectMicrophone() {} + let failMicVolumeVisualization = false; + function startMicVolumeVisualization() { + if (failMicVolumeVisualization) { + throw new Error('forced mic visualization failure'); + } + } + function ensureMicPopupScrollbarStyle() {} + function attachTransientMicPopupScrollbar() { return () => {}; } + function createScreenShareToggleButton() { + return document.createElement('button'); + } + + let micPermissionGranted = false; + let cachedMicDevices = null; + let disposeVoiceRecognitionPopover = null; + let voiceRecognitionPopoverRenderGeneration = 0; + + __PERMISSION_SOURCE__ + window.renderFloatingMicList = __RENDER_EXPRESSION__; + + window.__voicePopoverTest = { + state: S, + capturedErrors, + listenerBalance, + resolvePermissions() { + while (mediaResolvers.length) { + mediaResolvers.shift().resolve(stream); + } + }, + resolvePermission(index) { + mediaResolvers.splice(index, 1)[0].resolve(stream); + }, + rejectPermission(index) { + mediaResolvers.splice(index, 1)[0].reject( + new Error('permission rejected') + ); + }, + failMicVolumeVisualization() { + failMicVolumeVisualization = true; + }, + failVoicePanelSetupOn(type) { + failWindowListenerType = type; + }, + pendingPermissions: () => mediaResolvers.length, + popup: () => document.getElementById('live2d-popup-mic'), + panel: () => document.querySelector('[role="dialog"]'), + panels: () => document.querySelectorAll('[role="dialog"]').length, + }; +})(); +""" + harness = harness.replace( + "__DEFERRED_PERMISSION__", "true" if deferred_permission else "false" + ) + harness = harness.replace("__PERMISSION_SOURCE__", permission_source) + harness = harness.replace("__RENDER_EXPRESSION__", render_expression) + page.add_script_tag(content=harness) + + +@pytest.mark.frontend +def test_overlapping_voice_popover_renders_keep_one_owned_instance( + page: Page, +) -> None: + _install_voice_popover_harness(page, deferred_permission=True) + + result = page.evaluate( + """async () => { + const popup = window.__voicePopoverTest.popup(); + const first = window.renderFloatingMicList(popup); + const second = window.renderFloatingMicList(popup); + if (window.__voicePopoverTest.pendingPermissions() !== 2) { + throw new Error('expected two pending permission requests'); + } + window.__voicePopoverTest.resolvePermissions(); + const renderResults = await Promise.all([first, second]); + const afterOverlap = { + renderResults, + panels: window.__voicePopoverTest.panels(), + capturedErrors: [...window.__voicePopoverTest.capturedErrors], + listenerBalance: { ...window.__voicePopoverTest.listenerBalance }, + }; + const third = await window.renderFloatingMicList(popup); + return { + afterOverlap, + third, + panelsAfterRerender: window.__voicePopoverTest.panels(), + listenerBalanceAfterRerender: { + ...window.__voicePopoverTest.listenerBalance, + }, + }; + }""" + ) + + assert result["afterOverlap"]["renderResults"] == [False, True] + assert not result["afterOverlap"]["capturedErrors"] + assert result["afterOverlap"]["panels"] == 1 + assert result["third"] is True + assert result["panelsAfterRerender"] == 1 + + expected_global_listeners = { + "document:pointerdown": 1, + "document:keydown": 1, + "window:resize": 1, + "window:scroll": 1, + "window:voice-input-lifecycle-changed": 1, + "window:neko:voice-session-started": 1, + "window:neko:voice-settings-pending-changed": 1, + } + for key, expected in expected_global_listeners.items(): + assert result["afterOverlap"]["listenerBalance"].get(key) == expected + assert result["listenerBalanceAfterRerender"].get(key) == expected + + +@pytest.mark.frontend +def test_stale_voice_popover_failure_cannot_clear_new_render(page: Page) -> None: + _install_voice_popover_harness(page, deferred_permission=True) + + result = page.evaluate( + """async () => { + const popup = window.__voicePopoverTest.popup(); + const first = window.renderFloatingMicList(popup); + const second = window.renderFloatingMicList(popup); + window.__voicePopoverTest.resolvePermission(1); + const secondResult = await second; + const currentMarkup = popup.innerHTML; + console.warn = () => { + throw new Error('forced permission failure'); + }; + window.__voicePopoverTest.rejectPermission(0); + const firstResult = await first; + return { + firstResult, + secondResult, + markupPreserved: popup.innerHTML === currentMarkup, + errors: [...window.__voicePopoverTest.capturedErrors], + }; + }""" + ) + + assert result == { + "firstResult": False, + "secondResult": True, + "markupPreserved": True, + "errors": [], + } + + +@pytest.mark.frontend +def test_current_voice_popover_failure_disposes_owned_portal(page: Page) -> None: + _install_voice_popover_harness(page, deferred_permission=False) + + result = page.evaluate( + """async () => { + const popup = window.__voicePopoverTest.popup(); + window.__voicePopoverTest.failMicVolumeVisualization(); + const rendered = await window.renderFloatingMicList(popup); + return { + rendered, + panels: window.__voicePopoverTest.panels(), + errorText: popup.textContent, + listenerBalance: { + ...window.__voicePopoverTest.listenerBalance, + }, + }; + }""" + ) + + assert result["rendered"] is True + assert result["panels"] == 0 + assert result["errorText"] == "microphone.loadFailed" + for key in VOICE_POPOVER_GLOBAL_LISTENERS: + assert result["listenerBalance"].get(key) == 0 + + +@pytest.mark.frontend +def test_voice_popover_setup_failure_disposes_registered_listeners( + page: Page, +) -> None: + _install_voice_popover_harness(page, deferred_permission=False) + + result = page.evaluate( + """async () => { + const popup = window.__voicePopoverTest.popup(); + window.__voicePopoverTest.failVoicePanelSetupOn( + 'neko:voice-settings-pending-changed' + ); + const rendered = await window.renderFloatingMicList(popup); + return { + rendered, + panels: window.__voicePopoverTest.panels(), + errorText: popup.textContent, + listenerBalance: { + ...window.__voicePopoverTest.listenerBalance, + }, + }; + }""" + ) + + assert result["rendered"] is True + assert result["panels"] == 0 + assert result["errorText"] == "microphone.loadFailed" + for key in VOICE_POPOVER_GLOBAL_LISTENERS: + assert result["listenerBalance"].get(key) == 0 + + +@pytest.mark.frontend +def test_voice_popover_disposes_when_popup_host_is_removed(page: Page) -> None: + _install_voice_popover_harness(page, deferred_permission=False) + + result = page.evaluate( + """async () => { + const popup = window.__voicePopoverTest.popup(); + await window.renderFloatingMicList(popup); + popup.remove(); + await Promise.resolve(); + return { + panels: window.__voicePopoverTest.panels(), + listenerBalance: { ...window.__voicePopoverTest.listenerBalance }, + }; + }""" + ) + + assert result["panels"] == 0 + for key in VOICE_POPOVER_GLOBAL_LISTENERS: + assert result["listenerBalance"].get(key) == 0 + + +@pytest.mark.frontend +def test_voice_popover_toggles_have_accessible_names_and_hints( + page: Page, +) -> None: + _install_voice_popover_harness(page, deferred_permission=False) + + result = page.evaluate( + """async () => { + const popup = window.__voicePopoverTest.popup(); + await window.renderFloatingMicList(popup); + return Array.from( + window.__voicePopoverTest.panel().querySelectorAll( + 'input[type="checkbox"]' + ) + ).map((input) => { + const labelId = input.getAttribute('aria-labelledby'); + const hintId = input.getAttribute('aria-describedby'); + return { + labelId, + hintId, + labelText: labelId + ? document.getElementById(labelId)?.textContent + : null, + hintText: hintId + ? document.getElementById(hintId)?.textContent + : null, + }; + }); + }""" + ) + + assert len(result) == 2 + assert all(item["labelId"] and item["labelText"] for item in result) + assert all(item["hintId"] and item["hintText"] for item in result) + + +@pytest.mark.frontend +def test_voice_settings_pending_clears_only_after_target_session( + page: Page, +) -> None: + _install_voice_popover_harness(page, deferred_permission=False) + + result = page.evaluate( + """async () => { + const popup = window.__voicePopoverTest.popup(); + await window.renderFloatingMicList(popup); + const firstPanel = window.__voicePopoverTest.panel(); + const firstStatus = firstPanel.lastElementChild; + const optimizationInput = firstPanel.querySelectorAll( + 'input[type="checkbox"]' + )[1]; + optimizationInput.checked = false; + optimizationInput.dispatchEvent(new Event('change', { bubbles: true })); + const pending = firstStatus.textContent; + + window.dispatchEvent(new CustomEvent('voice-input-lifecycle-changed', { + detail: { state: 'warm_idle' }, + })); + const afterLifecycleOnly = firstStatus.textContent; + + window.dispatchEvent(new CustomEvent('neko:voice-session-started')); + const afterCurrentEpochStart = firstStatus.textContent; + + window.__voicePopoverTest.state.voiceSessionStartEpoch = 11; + window.dispatchEvent(new CustomEvent('neko:voice-session-started')); + const afterReadySession = firstStatus.textContent; + + optimizationInput.checked = true; + optimizationInput.dispatchEvent(new Event('change', { bubbles: true })); + window.__voicePopoverTest.state.voiceInputLifecycleState = 'blocked'; + window.dispatchEvent(new CustomEvent('voice-input-lifecycle-changed', { + detail: { state: 'blocked' }, + })); + const afterFailedStart = firstStatus.textContent; + + window.__voicePopoverTest.state.voiceSessionStartEpoch = 12; + window.dispatchEvent(new CustomEvent('neko:voice-session-started')); + const afterBlockedSession = firstStatus.textContent; + + const asrInput = document.querySelector( + '[aria-controls="' + firstPanel.id + '"] input[type="checkbox"]' + ); + asrInput.checked = false; + asrInput.dispatchEvent(new Event('change', { bubbles: true })); + window.__voicePopoverTest.state.voiceSessionStartEpoch = 13; + window.dispatchEvent(new CustomEvent('neko:voice-session-started')); + const afterNativeSession = firstStatus.textContent; + + asrInput.checked = true; + asrInput.dispatchEvent(new Event('change', { bubbles: true })); + const beforeDispose = firstStatus.textContent; + await window.renderFloatingMicList(popup); + const oldStatusAfterDispose = firstStatus.textContent; + window.__voicePopoverTest.state.voiceSessionStartEpoch = 14; + window.dispatchEvent(new CustomEvent('neko:voice-session-started')); + + return { + pending, + afterLifecycleOnly, + afterCurrentEpochStart, + afterReadySession, + afterFailedStart, + afterBlockedSession, + afterNativeSession, + beforeDispose, + oldStatusAfterDispose, + oldStatusAfterEvent: firstStatus.textContent, + oldPanelConnected: firstPanel.isConnected, + panels: window.__voicePopoverTest.panels(), + listenerBalance: { ...window.__voicePopoverTest.listenerBalance }, + }; + }""" + ) + + pending_key = "microphone.voiceRecognitionSettingsPending" + assert result["pending"] == pending_key + assert result["afterLifecycleOnly"] == pending_key + assert result["afterCurrentEpochStart"] == pending_key + assert result["afterReadySession"] == "microphone.voiceRecognitionStatusReady" + assert result["afterFailedStart"] == pending_key + assert result["afterBlockedSession"] == "microphone.voiceRecognitionUnavailable" + assert result["afterNativeSession"] == "microphone.voiceRecognitionDisabledHint" + assert result["beforeDispose"] == pending_key + assert result["oldStatusAfterDispose"] == pending_key + assert result["oldStatusAfterEvent"] == pending_key + assert result["oldPanelConnected"] is False + assert result["panels"] == 1 + assert result["listenerBalance"]["window:neko:voice-session-started"] == 1 + assert ( + result["listenerBalance"]["window:neko:voice-settings-pending-changed"] + == 1 + ) + + +@pytest.mark.frontend +def test_voice_popover_keeps_active_route_and_keyboard_access( + page: Page, +) -> None: + _install_voice_popover_harness(page, deferred_permission=False) + + result = page.evaluate( + """async () => { + const popup = window.__voicePopoverTest.popup(); + await window.renderFloatingMicList(popup); + const panel = window.__voicePopoverTest.panel(); + const container = document.querySelector( + '[aria-controls="' + panel.id + '"]' + ); + const asrInput = container.querySelector('input[type="checkbox"]'); + const panelInputs = panel.querySelectorAll('input[type="checkbox"]'); + const noiseInput = panelInputs[0]; + const optimizationInput = panelInputs[1]; + + window.__voicePopoverTest.state.voiceChatActive = true; + window.__voicePopoverTest.state.independentAsrActive = true; + asrInput.checked = false; + asrInput.dispatchEvent(new Event('change', { bubbles: true })); + + const summary = container.firstElementChild + .firstElementChild.lastElementChild.textContent; + container.focus(); + container.dispatchEvent(new KeyboardEvent('keydown', { + key: 'Enter', + bubbles: true, + })); + + return { + summary, + noiseDisabled: noiseInput.disabled, + optimizationDisabled: optimizationInput.disabled, + panelOpen: container.getAttribute('aria-expanded'), + focusedNoise: document.activeElement === noiseInput, + }; + }""" + ) + + assert result == { + "summary": "microphone.independentAsrSummary", + "noiseDisabled": False, + "optimizationDisabled": True, + "panelOpen": "true", + "focusedNoise": True, + } + + +@pytest.mark.frontend +def test_voice_popover_preserves_cross_window_active_route_across_rerender( + page: Page, +) -> None: + _install_voice_popover_harness(page, deferred_permission=False) + + result = page.evaluate( + """async () => { + const popup = window.__voicePopoverTest.popup(); + const state = window.__voicePopoverTest.state; + await window.renderFloatingMicList(popup); + + // app-settings applies the other window's new preference to S, but + // the current session remains on the route captured before that + // preference changed. The shared pending snapshot must survive the + // popup's owned-disposer rerender. + state.voiceChatActive = true; + state.independentAsrActive = true; + state.pendingVoiceRouteIndependentAsr = true; + state.voiceSettingsPendingUntilEpoch = 11; + state.independentAsrEnabled = false; + await window.renderFloatingMicList(popup); + + const panel = window.__voicePopoverTest.panel(); + const container = document.querySelector( + '[aria-controls="' + panel.id + '"]' + ); + return { + summary: container.firstElementChild + .firstElementChild.lastElementChild.textContent, + status: panel.lastElementChild.textContent, + }; + }""" + ) + + assert result == { + "summary": "microphone.independentAsrSummary", + "status": "microphone.voiceRecognitionSettingsPending", + } + + +@pytest.mark.frontend +def test_voice_popover_keyboard_focus_ring_is_visible(page: Page) -> None: + _install_voice_popover_harness(page, deferred_permission=False) + page.evaluate( + """async () => { + const popup = window.__voicePopoverTest.popup(); + await window.renderFloatingMicList(popup); + const panel = window.__voicePopoverTest.panel(); + const container = document.querySelector( + '[aria-controls="' + panel.id + '"]' + ); + container.focus(); + }""" + ) + + page.keyboard.press("Enter") + + result = page.evaluate( + """() => { + const panel = window.__voicePopoverTest.panel(); + const input = panel.querySelector('input[type="checkbox"]'); + const slider = input.nextElementSibling; + return { + focused: document.activeElement === input, + boxShadow: getComputedStyle(slider).boxShadow, + }; + }""" + ) + assert result["focused"] is True + assert result["boxShadow"] != "none" + + +@pytest.mark.frontend +def test_shared_audio_capture_script_is_safe_on_web_routes( + page: Page, running_server: str +) -> None: + audio_capture_console_errors: list[str] = [] + page_errors: list[str] = [] + script_responses: list[tuple[str, int]] = [] + + page.on( + "console", + lambda message: audio_capture_console_errors.append( + f"{message.text} @ {message.location}" + ) + if ( + message.type == "error" + and "/static/app/app-audio-capture.js" + in message.location.get("url", "") + ) + else None, + ) + page.on("pageerror", lambda error: page_errors.append(str(error))) + page.on( + "response", + lambda response: script_responses.append((response.url, response.status)) + if "/static/app/app-audio-capture.js" in response.url + else None, + ) + + root_page = page.context.new_page() + root_audio_capture_console_errors: list[str] = [] + root_page_errors: list[str] = [] + root_script_responses: list[tuple[str, int]] = [] + root_page.on( + "console", + lambda message: root_audio_capture_console_errors.append( + f"{message.text} @ {message.location}" + ) + if ( + message.type == "error" + and "/static/app/app-audio-capture.js" + in message.location.get("url", "") + ) + else None, + ) + root_page.on( + "pageerror", + lambda error: root_page_errors.append(str(error)), + ) + root_page.on( + "response", + lambda response: root_script_responses.append( + (response.url, response.status) + ) + if "/static/app/app-audio-capture.js" in response.url + else None, + ) + root_page.goto(f"{running_server}/", wait_until="domcontentloaded") + root_page.wait_for_function("typeof window.renderFloatingMicList === 'function'") + assert any(status == 200 for _, status in root_script_responses) + assert not root_page_errors, root_page_errors + assert not root_audio_capture_console_errors, "\n".join( + root_audio_capture_console_errors + ) + root_page.close() + + page.goto(f"{running_server}/chat", wait_until="domcontentloaded") + page.wait_for_function("typeof window.renderFloatingMicList === 'function'") + page.wait_for_timeout(500) + + assert any(status == 200 for _, status in script_responses) + assert page.locator( + "#live2d-popup-mic, #vrm-popup-mic, #mmd-popup-mic" + ).count() == 0 + assert page.locator('[id$="-voice-recognition-settings"]').count() == 0 + assert not page_errors, page_errors + assert not audio_capture_console_errors, "\n".join( + audio_capture_console_errors + ) diff --git a/tests/unit/test_app_websocket_static.py b/tests/unit/test_app_websocket_static.py index 2a026650c2..1b7f4ef780 100644 --- a/tests/unit/test_app_websocket_static.py +++ b/tests/unit/test_app_websocket_static.py @@ -256,28 +256,21 @@ def test_independent_asr_provider_copy_resolves_via_provider_names(): assert "window.t('microphone.independentAsrActive', { providerKey: asrProvider || 'unknown' })" in ready_branch assert "window.t('microphone.independentAsrProviderUnavailable', { providerKey: asrProvider || 'unknown' })" in source - # The hint is rendered by one function now, called both at build time and - # from the toggle's change handler -- it used to be computed once, so - # flipping the switch with the popup open left the previous text standing - # and the confirmation contradicted the choice just made. - hint_block = capture_source.split("function renderAsrHint() {", 1)[1].split( - "asrInput.addEventListener('change'", - 1, - )[0] - assert "{ providerKey: S.independentAsrProvider || 'unknown' }" in hint_block - assert "asrHint.setAttribute('data-i18n-params', JSON.stringify(hintParams));" in hint_block - assert hint_block.index("asrHint.setAttribute('data-i18n-params', JSON.stringify(hintParams));") < hint_block.index( - "window.t(hintKey, hintParams)" - ) - assert "provider: S.independentAsrProvider" not in hint_block + # The shared popover now owns the summary. It resolves the registry key to a + # display name, renders that value through the locale template, and refreshes + # from the toggle handler so the visible route never lags the user's choice. + summary_block = capture_source.split( + "function updateVoiceRecognitionUi() {", 1 + )[1].split("function positionVoicePanel()", 1)[0] + assert "'microphone.independentAsrSummary'" in summary_block + assert "{ provider: provider }" in summary_block + assert "'microphone.voiceRecognitionDisabled'" in summary_block + assert "provider: S.independentAsrProvider" not in summary_block - # ...and the change handler actually re-renders it. change_handler = capture_source.split( - "asrInput.addEventListener('change', function () {", 1 - )[1].split("window.appSettings.saveSettings", 1)[0] - assert "renderAsrHint();" in change_handler, ( - "flipping the toggle must refresh the hint it confirms" - ) + "var asrToggle = createVoiceSettingToggle(", 1 + )[1].split("asrRow.appendChild(asrCopy);", 1)[0] + assert "updateVoiceRecognitionUi();" in change_handler def test_provider_names_cover_asr_registry_keys_in_all_locales(): @@ -312,42 +305,42 @@ def test_independent_asr_failure_copy_matches_hard_route_in_all_locales(): "en.json": ( "Independent ASR unavailable. Voice input has stopped for this session. Check the independent ASR configuration, then start a new voice session.", "Enabled for the next voice session; it will not automatically switch to Omni if unavailable.", - "{{provider}} is temporarily unavailable. Voice input has stopped for this session. It did not switch to another speech recognition service. Please start a new voice session later.", + "{{providerKey}} is temporarily unavailable. Voice input has stopped for this session. It did not switch to another speech recognition service. Please start a new voice session later.", ), "es.json": ( "El ASR independiente no está disponible. La entrada de voz se ha detenido para esta sesión. Revisa la configuración del ASR independiente y después inicia una nueva sesión de voz.", "Se activará en la próxima sesión de voz; no cambiará automáticamente a Omni si no está disponible.", - "{{provider}} no está disponible temporalmente. La entrada de voz se ha detenido para esta sesión. No se cambió a otro servicio de reconocimiento de voz. Inicia una nueva sesión de voz más tarde.", + "{{providerKey}} no está disponible temporalmente. La entrada de voz se ha detenido para esta sesión. No se cambió a otro servicio de reconocimiento de voz. Inicia una nueva sesión de voz más tarde.", ), "ja.json": ( "独立 ASR を利用できないため、この音声セッションの入力を停止しました。独立 ASR の設定を確認してから、新しい音声セッションを開始してください。", "次の音声セッションから有効になります。利用できない場合も Omni へ自動的に切り替わりません。", - "{{provider}} は一時的に利用できません。この音声セッションの入力を停止しました。別の音声認識サービスには切り替えていません。後でもう一度音声セッションを開始してください。", + "{{providerKey}} は一時的に利用できません。この音声セッションの入力を停止しました。別の音声認識サービスには切り替えていません。後でもう一度音声セッションを開始してください。", ), "ko.json": ( "독립 ASR을 사용할 수 없어 이번 음성 세션의 입력을 중지했습니다. 독립 ASR 설정을 확인한 다음 새 음성 세션을 시작하세요.", "다음 음성 세션부터 활성화되며, 사용할 수 없어도 Omni로 자동 전환되지 않습니다.", - "{{provider}}을(를) 일시적으로 사용할 수 없어 이번 음성 세션의 입력을 중지했습니다. 다른 음성 인식 서비스로 전환하지 않았습니다. 나중에 새 음성 세션을 시작하세요.", + "{{providerKey}}을(를) 일시적으로 사용할 수 없어 이번 음성 세션의 입력을 중지했습니다. 다른 음성 인식 서비스로 전환하지 않았습니다. 나중에 새 음성 세션을 시작하세요.", ), "pt.json": ( "O ASR independente não está disponível. A entrada de voz foi interrompida nesta sessão. Verifique a configuração do ASR independente e depois inicie uma nova sessão de voz.", "Será ativado na próxima sessão de voz; não mudará automaticamente para o Omni se estiver indisponível.", - "{{provider}} está temporariamente indisponível. A entrada de voz foi interrompida nesta sessão. O sistema não mudou para outro serviço de reconhecimento de voz. Inicie uma nova sessão de voz mais tarde.", + "{{providerKey}} está temporariamente indisponível. A entrada de voz foi interrompida nesta sessão. O sistema não mudou para outro serviço de reconhecimento de voz. Inicie uma nova sessão de voz mais tarde.", ), "ru.json": ( "Независимый ASR недоступен. Голосовой ввод в этом сеансе остановлен. Проверьте настройки независимого ASR, затем начните новый голосовой сеанс.", "Будет включён в следующем голосовом сеансе; при недоступности автоматического переключения на Omni не произойдёт.", - "{{provider}} временно недоступен. Голосовой ввод в этом сеансе остановлен. Переключения на другую службу распознавания речи не произошло. Начните новый голосовой сеанс позже.", + "{{providerKey}} временно недоступен. Голосовой ввод в этом сеансе остановлен. Переключения на другую службу распознавания речи не произошло. Начните новый голосовой сеанс позже.", ), "zh-CN.json": ( "独立 ASR 不可用，本次语音输入已停止。请检查独立 ASR 配置，然后重新开始语音会话。", "将在下次语音会话启用；不可用时不会自动切换到 Omni。", - "{{provider}} 暂时不可用，本次语音输入已停止。未切换到其他语音识别服务，请稍后重新开始语音会话。", + "{{providerKey}} 暂时不可用，本次语音输入已停止。未切换到其他语音识别服务，请稍后重新开始语音会话。", ), "zh-TW.json": ( "獨立 ASR 無法使用，本次語音輸入已停止。請檢查獨立 ASR 設定，然後重新開始語音會話。", "將於下次語音會話啟用；無法使用時不會自動切換到 Omni。", - "{{provider}} 暫時無法使用，本次語音輸入已停止。未切換到其他語音辨識服務，請稍後重新開始語音會話。", + "{{providerKey}} 暫時無法使用，本次語音輸入已停止。未切換到其他語音辨識服務，請稍後重新開始語音會話。", ), } @@ -493,19 +486,22 @@ def test_independent_asr_toggle_awaits_server_sync_before_next_session(): capture_source = APP_AUDIO_CAPTURE_PATH.read_text(encoding="utf-8") websocket_source = APP_WEBSOCKET_PATH.read_text(encoding="utf-8") + persist_block = capture_source.split( + "function persistVoiceSettingChange() {", 1 + )[1].split("function markVoiceSettingsPending", 1)[0] toggle_block = capture_source.split( - "asrInput.addEventListener('change', function () {", - 1, - )[1].split("asrRow.appendChild(asrLabel);", 1)[0] - assert "window.appSettings.saveSettings({ skipServerSync: true });" in toggle_block - assert "window.appSettings.syncSettingsToServer({ userInitiated: true })" in toggle_block - assert "S.pendingSettingsSyncPromise = syncPromise;" in toggle_block + "var asrToggle = createVoiceSettingToggle(", 1 + )[1].split("asrRow.appendChild(asrCopy);", 1)[0] + assert "persistVoiceSettingChange();" in toggle_block + assert "window.appSettings.saveSettings({ skipServerSync: true });" in persist_block + assert "window.appSettings.syncSettingsToServer({ userInitiated: true })" in persist_block + assert "S.pendingSettingsSyncPromise = syncPromise;" in persist_block # Completion clears the gate only when it still owns it (a newer toggle # may have replaced the pending promise meanwhile). - assert "if (S.pendingSettingsSyncPromise === syncPromise)" in toggle_block - assert "S.pendingSettingsSyncPromise = null;" in toggle_block + assert "if (S.pendingSettingsSyncPromise === syncPromise)" in persist_block + assert "S.pendingSettingsSyncPromise = null;" in persist_block # Fallback when the settings module does not expose syncSettingsToServer. - assert "window.appSettings.saveSettings();" in toggle_block + assert "window.appSettings.saveSettings();" in persist_block gate_block = websocket_source.split( "function ensureWebSocketOpen(timeoutMs = 5000)", @@ -588,6 +584,28 @@ def test_start_session_payload_carries_independent_asr_handshake(): assert 0 < attach_index - creation_index < 200 +def test_start_session_payload_carries_resource_optimization_handshake(): + websocket_source = APP_WEBSOCKET_PATH.read_text(encoding="utf-8") + state_source = APP_STATE_PATH.read_text(encoding="utf-8") + settings_source = APP_SETTINGS_PATH.read_text(encoding="utf-8") + wrapper = websocket_source.split( + "function attachStartSessionHandshake(ws)", + 1, + )[1].split("function connectWebSocket()", 1)[0] + + assert "voiceInputResourceOptimizationAuthoritative: false," in state_source + assert "S.voiceInputResourceOptimizationAuthoritative === true" in wrapper + assert ( + "msg.voice_input_resource_optimization_enabled = " + "S.voiceInputResourceOptimizationEnabled !== false;" + ) in wrapper + assert ( + "_dirtySettingsKeys.has('voiceInputResourceOptimizationEnabled')" + in settings_source + ) + assert "S.voiceInputResourceOptimizationAuthoritative = true;" in settings_source + + def test_start_session_handshake_omitted_until_settings_hydrated(): # On a fresh browser profile — or while the async conversation-settings # GET is still pending — S.independentAsrEnabled is only the boot default @@ -626,7 +644,7 @@ def test_start_session_handshake_omitted_until_settings_hydrated(): def test_settings_hydration_marked_on_server_merge_and_user_change(): - # S.settingsHydrated must flip true on exactly the three authoritative + # S.settingsHydrated must flip true on authoritative settings evidence: # events, and never merely at boot: # (1) the conversation-settings GET succeeded (server values merged); # (2) the user explicitly changed a setting — the independent-ASR toggle @@ -637,6 +655,8 @@ def test_settings_hydration_marked_on_server_merge_and_user_change(): # (3) a cross-window independent-ASR flip arrived via the 'storage' # listener — the originating window's user action, pinned by # test_cross_window_asr_flip_marks_hydration_and_asr_dirty. + # (4) a durable, explicit optimization decision survived a reload while + # its server synchronization is still pending. settings_source = APP_SETTINGS_PATH.read_text(encoding="utf-8") capture_source = APP_AUDIO_CAPTURE_PATH.read_text(encoding="utf-8") @@ -677,11 +697,14 @@ def test_settings_hydration_marked_on_server_merge_and_user_change(): # syncSettingsToServer({ userInitiated: true }); the saveSettings call it # makes skips the internal server sync, so the direct call is the seam. toggle_handler = capture_source.split( - "asrInput.addEventListener('change', function () {", - 1, - )[1].split("asrRow.appendChild(", 1)[0] - assert "S.independentAsrEnabled = asrInput.checked;" in toggle_handler - assert "window.appSettings.syncSettingsToServer({ userInitiated: true })" in toggle_handler + "var asrToggle = createVoiceSettingToggle(", 1 + )[1].split("asrRow.appendChild(asrCopy);", 1)[0] + persist_block = capture_source.split( + "function persistVoiceSettingChange() {", 1 + )[1].split("function markVoiceSettingsPending", 1)[0] + assert "S.independentAsrEnabled = enabled;" in toggle_handler + assert "persistVoiceSettingChange();" in toggle_handler + assert "window.appSettings.syncSettingsToServer({ userInitiated: true })" in persist_block # Boot must NOT mark hydration: the first-launch initialization save goes # through saveSettings({ skipServerSync: true }) which bypasses @@ -693,13 +716,18 @@ def test_settings_hydration_marked_on_server_merge_and_user_change(): )[1].split("} catch (error) {", 1)[0] assert "saveSettings({ skipServerSync: true });" in first_launch_block assert "S.settingsHydrated" not in first_launch_block - # And nothing else in loadSettings' synchronous body marks hydration - # before the async GET callback runs. + # The only synchronous load-time hydration is guarded by a durable pending + # optimization decision; ordinary boot defaults still cannot gain authority. sync_load_body = settings_source.split("function loadSettings()", 1)[1].split( "loadSettingsFromServer().then(serverResult => {", 1, )[0] - assert "S.settingsHydrated" not in sync_load_body + assert sync_load_body.count("S.settingsHydrated = true;") == 1 + assert "bootMeta.optimizationDecisionPendingSync" in sync_load_body + assert ( + sync_load_body.index("bootMeta.optimizationDecisionPendingSync") + < sync_load_body.index("S.settingsHydrated = true;") + ) def test_periodic_sync_skips_post_and_never_marks_hydration_while_unhydrated(): @@ -759,10 +787,13 @@ def test_user_toggle_during_get_failure_marks_hydration_posts_and_stamps(): # The toggle's direct sync call is user-initiated and still POSTs. toggle_block = capture_source.split( - "asrInput.addEventListener('change', function () {", - 1, - )[1].split("asrRow.appendChild(asrLabel);", 1)[0] - assert "window.appSettings.syncSettingsToServer({ userInitiated: true })" in toggle_block + "var asrToggle = createVoiceSettingToggle(", 1 + )[1].split("asrRow.appendChild(asrCopy);", 1)[0] + persist_block = capture_source.split( + "function persistVoiceSettingChange() {", 1 + )[1].split("function markVoiceSettingsPending", 1)[0] + assert "persistVoiceSettingChange();" in toggle_block + assert "window.appSettings.syncSettingsToServer({ userInitiated: true })" in persist_block # saveSettings' full (non-skipServerSync) path is the other user seam — # the settings popup, subtitle toggles and chat-window toggles all route @@ -1051,7 +1082,12 @@ def test_cross_window_asr_flip_marks_hydration_and_asr_dirty(): )[0] assert "S.settingsHydrated = true;" in flip_gate assert "_dirtySettingsKeys.add('independentAsrEnabled');" in flip_gate - assert listener_block.count("S.settingsHydrated = true;") == 1 + optimization_gate = listener_block.split( + "if (optimizationChangedByOtherWindow) {", + 1, + )[1].split("}", 1)[0] + assert "S.settingsHydrated = true;" in optimization_gate + assert listener_block.count("S.settingsHydrated = true;") == 2 assert listener_block.count("_dirtySettingsKeys.add('independentAsrEnabled');") == 1 # No POST from the receiving window: the originating window owns @@ -1503,8 +1539,12 @@ def test_settings_cas_conflict_rebuilds_body_from_winning_asr_decision_harness() const resetPriorDecision = { writeId: Date.now() + 1000, writerId: 'server-before-reset', - value: true, + value: false, }; + assert( + resetPriorDecision.value !== true, + 'the pre-reset ASR decision must differ from the enabled reset default' + ); const reset = makeContext(false, { success: true, settings: {}, @@ -1519,13 +1559,15 @@ def test_settings_cas_conflict_rebuilds_body_from_winning_asr_decision_harness() assert( reset.S.slopFilterEnabled === true && reset.S.proactiveVisionEnabled === resetVisionDefault - && reset.S.independentAsrEnabled === false, + && reset.S.independentAsrEnabled === true + && reset.S.voiceInputResourceOptimizationEnabled === true, 'an empty authoritative restore must reset stale local values to defaults: ' + JSON.stringify({ slop: reset.S.slopFilterEnabled, vision: reset.S.proactiveVisionEnabled, visionDefault: resetVisionDefault, asr: reset.S.independentAsrEnabled, + optimization: reset.S.voiceInputResourceOptimizationEnabled, }) ); assert(reset.postCalls.length === 1, 'the reset defaults must be written back once'); @@ -1535,7 +1577,7 @@ def test_settings_cas_conflict_rebuilds_body_from_winning_asr_decision_harness() ] ); assert( - resetWritebackDecision.value === false + resetWritebackDecision.value === true && resetWritebackDecision.writeId > resetPriorDecision.writeId, 'a reset writeback must rebase the stale tuple onto the reset default' @@ -1548,7 +1590,8 @@ def test_settings_cas_conflict_rebuilds_body_from_winning_asr_decision_harness() assert( resetBody.slopFilterEnabled === true && resetBody.proactiveVisionEnabled === resetVisionDefault - && resetBody.independentAsrEnabled === false, + && resetBody.independentAsrEnabled === true + && resetBody.voiceInputResourceOptimizationEnabled === true, 'the reset writeback must not repopulate the server with stale localStorage' ); reset.postCalls[0].resolve(response( @@ -1570,7 +1613,7 @@ def test_settings_cas_conflict_rebuilds_body_from_winning_asr_decision_harness() reset.store.get('project_neko_settings') ); assert( - resetPersisted._sharedWriteMeta.asrDecision.value === false + resetPersisted._sharedWriteMeta.asrDecision.value === true && resetPersisted._sharedWriteMeta.serverRevision === 11, 'a full-write success must adopt and rebroadcast the generated server ASR tuple' ); @@ -1594,7 +1637,7 @@ def test_settings_cas_conflict_rebuilds_body_from_winning_asr_decision_harness() 'X-Conversation-Settings-ASR-Decision' ] ); - resetRace.S.independentAsrEnabled = true; + resetRace.S.independentAsrEnabled = false; resetRace.mod.saveSettings({ skipServerSync: true, explicitSharedKeys: ['independentAsrEnabled'], @@ -1611,7 +1654,7 @@ def test_settings_cas_conflict_rebuilds_body_from_winning_asr_decision_harness() resetRace.store.get('project_neko_settings') )._sharedWriteMeta.asrDecision; assert( - resetToggleDecision.value === true + resetToggleDecision.value === false && resetToggleDecision.writeId > resetRaceBaselineDecision.writeId, 'a toggle during reset must mint above the rebased reset decision' @@ -1623,7 +1666,7 @@ def test_settings_cas_conflict_rebuilds_body_from_winning_asr_decision_harness() { success: true, settings: { - independentAsrEnabled: false, + independentAsrEnabled: true, slopFilterEnabled: true, }, revision: 11, @@ -1636,7 +1679,7 @@ def test_settings_cas_conflict_rebuilds_body_from_winning_asr_decision_harness() await tick(); await tick(); assert( - resetRace.S.independentAsrEnabled === true + resetRace.S.independentAsrEnabled === false && resetRace.postCalls.length === 2, 'the older reset response must not overwrite the queued toggle' ); @@ -1647,7 +1690,7 @@ def test_settings_cas_conflict_rebuilds_body_from_winning_asr_decision_harness() ] ); assert( - resetToggleBody.independentAsrEnabled === true + resetToggleBody.independentAsrEnabled === false && JSON.stringify(resetToggleHeader) === JSON.stringify(resetToggleDecision), 'the queued sync must persist the newer toggle tuple' @@ -2642,7 +2685,7 @@ def test_settings_cas_conflict_rebuilds_body_from_winning_asr_decision_harness() assert( ctx.S.focusModeEnabled === true && ctx.S.slopFilterEnabled === true - && ctx.S.independentAsrEnabled === false, + && ctx.S.independentAsrEnabled === true, 'a partial reset response must materialize defaults without losing the edit' ); const restoredLocal = JSON.parse(ctx.store.get('project_neko_settings')); @@ -2805,8 +2848,12 @@ def test_cross_window_asr_flip_authoritative_over_pending_get_harness(): const postCalls = []; const getCalls = []; const listeners = []; + const dispatchedEvents = []; const sandbox = { console: { log() {}, warn() {}, error() {} }, + CustomEvent: class { + constructor(type) { this.type = type; } + }, setInterval() { return 0; }, clearInterval() {}, setTimeout(fn, ms) { @@ -2830,11 +2877,21 @@ def test_cross_window_asr_flip_authoritative_over_pending_get_harness(): }, }; sandbox.window = { - appState: { independentAsrEnabled: false, settingsHydrated: false }, + appState: { + independentAsrEnabled: false, + independentAsrActive: true, + voiceChatActive: true, + voiceInputLifecycleState: 'active', + voiceSessionStartEpoch: 10, + voiceSettingsPendingUntilEpoch: null, + pendingVoiceRouteIndependentAsr: null, + settingsHydrated: false, + }, appConst: {}, appUtils: { mapRenderQualityToFollowPerf() { return 'medium'; } }, addEventListener(type, fn) { listeners.push({ type, fn }); }, removeEventListener() {}, + dispatchEvent(event) { dispatchedEvents.push(event.type); }, }; vm.createContext(sandbox); vm.runInContext(source, sandbox); @@ -2843,6 +2900,7 @@ def test_cross_window_asr_flip_authoritative_over_pending_get_harness(): return { postCalls, getCalls, + dispatchedEvents, S: sandbox.window.appState, fireStorage(newValue) { storage[0].fn({ key: 'project_neko_settings', newValue }); @@ -2862,6 +2920,12 @@ def test_cross_window_asr_flip_authoritative_over_pending_get_harness(): ctx.fireStorage(JSON.stringify({ independentAsrEnabled: true })); assert(ctx.S.independentAsrEnabled === true, 'the flip must be applied to S'); assert(ctx.S.settingsHydrated === true, 'the flip must arm the start_session handshake stamp'); + assert(ctx.S.voiceSettingsPendingUntilEpoch === 11, 'the flip must target the next voice-session epoch'); + assert(ctx.S.pendingVoiceRouteIndependentAsr === true, 'the pending summary must preserve the active route'); + assert( + ctx.dispatchedEvents.includes('neko:voice-settings-pending-changed'), + 'the flip must notify an already-open microphone popover' + ); assert(ctx.postCalls.length === 0, 'the receiving window must not POST (originating window owns persistence)'); // The GET now resolves with the server value read BEFORE the other @@ -2883,6 +2947,8 @@ def test_cross_window_asr_flip_authoritative_over_pending_get_harness(): ctx2.fireStorage(JSON.stringify({ independentAsrEnabled: false, mergeMessagesEnabled: true })); assert(ctx2.S.mergeMessagesEnabled === true, 'other shared keys must still sync across windows'); assert(ctx2.S.settingsHydrated === false, 'no ASR flip means no hydration mark'); + assert(ctx2.S.voiceSettingsPendingUntilEpoch === null, 'no flip means no pending voice-session marker'); + assert(ctx2.dispatchedEvents.length === 0, 'no flip means no popover notification'); assert(ctx2.postCalls.length === 0, 'a non-flip storage event must not POST either'); ctx2.getCalls[0].resolve({ @@ -2932,7 +2998,7 @@ def test_shared_settings_writes_carry_explicit_change_metadata(): "localStorage.setItem('project_neko_settings', JSON.stringify(settings))" not in settings_source ) - assert settings_source.count("_writeSharedSettings(") == 3 # 1 def + 2 writes + assert settings_source.count("_writeSharedSettings(") == 4 # 1 def + 3 writes save_fn = _block_after(settings_source, "function saveSettings(options) {") assert "serverMerged ? [] : _collectExplicitSharedKeys(settings)" in save_fn assert "const serverMerged = !!(options && options.serverMerged);" in save_fn @@ -3001,6 +3067,13 @@ def test_shared_settings_writes_carry_explicit_change_metadata(): assert "if (serverAuthoritative === true) return true;" in id_validator assert "Array.isArray(meta.changedKeys) ? meta.changedKeys : []" in read_fn assert "knownKeyWritesPresent" in read_fn + optimization_reader = read_fn.split( + "optimizationDecision: (meta.optimizationDecision", 1 + )[1].split("optimizationDecisionPendingSync:", 1)[0] + assert "_isValidAsrWriteId(" in optimization_reader + assert "meta.optimizationDecision.writeId," in optimization_reader + assert "Number.isInteger(meta.serverRevision)" in optimization_reader + assert "isFinite(meta.optimizationDecision.writeId)" not in optimization_reader listener_block = settings_source.split( "window.addEventListener('storage', function (event) {", 1 @@ -3062,7 +3135,7 @@ def test_unrelated_save_from_unhydrated_window_is_not_an_asr_toggle_harness(): if (!cond) throw new Error('ASSERT: ' + msg); } - function makeContext(initialSettings) { + function makeContext(initialSettings = null) { const postCalls = []; const getCalls = []; const listeners = []; @@ -3139,11 +3212,16 @@ def test_unrelated_save_from_unhydrated_window_is_not_an_asr_toggle_harness(): const okPost = { ok: true, json: async () => ({ success: true }) }; const tick = () => new Promise((resolve) => setImmediate(resolve)); - async function hydrateFromServer(ctx, settings) { + async function hydrateFromServer(ctx, settings, decisions = null) { assert(ctx.getCalls.length === 1, 'boot must issue the settings GET'); ctx.getCalls[0].resolve({ ok: true, - json: async () => ({ success: true, settings, telemetryBranch: null }), + json: async () => ({ + success: true, + settings, + decisions: decisions || {}, + telemetryBranch: null, + }), }); await tick(); await tick(); @@ -3157,8 +3235,15 @@ def test_unrelated_save_from_unhydrated_window_is_not_an_asr_toggle_harness(): async function main() { // ---- Scenario 1: unrelated save from an UNHYDRATED window ---- const receiver = makeContext(); - await hydrateFromServer(receiver, { independentAsrEnabled: true }); + await hydrateFromServer(receiver, { + independentAsrEnabled: true, + voiceInputResourceOptimizationEnabled: false, + }); assert(receiver.S.independentAsrEnabled === true, 'receiver merged the server ASR value'); + assert( + receiver.S.voiceInputResourceOptimizationEnabled === false, + 'receiver merged the server optimization value' + ); const writer = makeContext(); // boot GET left pending -> unhydrated writer.win.mergeMessagesEnabled = true; @@ -3169,12 +3254,20 @@ def test_unrelated_save_from_unhydrated_window_is_not_an_asr_toggle_harness(): staleParsed.independentAsrEnabled === false, 'saveSettings still copies the ASR key into every snapshot (that is the trap)' ); + assert( + staleParsed.voiceInputResourceOptimizationEnabled === true, + 'saveSettings still copies the optimization key into every snapshot (that is the trap)' + ); const receiverPostsBefore = receiver.postCalls.length; receiver.fireStorage(stalePayload); assert( receiver.S.independentAsrEnabled === true, 'the hydrated ASR value must survive an unrelated save from an unhydrated window' ); + assert( + receiver.S.voiceInputResourceOptimizationEnabled === false, + 'the hydrated optimization value must survive an unrelated save from an unhydrated window' + ); assert( receiver.S.mergeMessagesEnabled === true, 'every other shared key must still sync across windows' @@ -3195,6 +3288,10 @@ def test_unrelated_save_from_unhydrated_window_is_not_an_asr_toggle_harness(): staleMeta.changedKeys.indexOf('independentAsrEnabled') === -1, 'an unrelated save must NOT declare the ASR key as user-changed' ); + assert( + staleMeta.changedKeys.indexOf('voiceInputResourceOptimizationEnabled') === -1, + 'an unrelated save must NOT declare the optimization key as user-changed' + ); assert(staleMeta.hydrated === false, 'the writer had not merged the server settings yet'); // ---- Scenario 2 (negative): the ASR key must not be dirtied ---- @@ -3484,6 +3581,163 @@ def test_unrelated_save_from_unhydrated_window_is_not_an_asr_toggle_harness(): 'a fresh recovery envelope must not outrank its older per-key provenance' ); + // ---- Scenario 11: a genuine optimization toggle still propagates ---- + const optimizationWriter = makeContext(); + await hydrateFromServer(optimizationWriter, { + independentAsrEnabled: false, + voiceInputResourceOptimizationEnabled: true, + }); + optimizationWriter.S.voiceInputResourceOptimizationEnabled = false; + optimizationWriter.mod.saveSettings({ skipServerSync: true }); + const optimizationPayload = optimizationWriter.lastSharedWrite(); + const optimizationMeta = JSON.parse(optimizationPayload)._sharedWriteMeta; + assert( + optimizationMeta.changedKeys.indexOf( + 'voiceInputResourceOptimizationEnabled' + ) !== -1, + 'a real optimization toggle must be declared explicitly' + ); + + const optimizationReceiver = makeContext(); + await hydrateFromServer(optimizationReceiver, { + independentAsrEnabled: false, + voiceInputResourceOptimizationEnabled: true, + }); + optimizationReceiver.fireStorage(optimizationPayload); + assert( + optimizationReceiver.S.voiceInputResourceOptimizationEnabled === false, + 'a real optimization toggle must apply across windows' + ); + + // ---- Scenario 7: concurrent optimization toggles converge ---- + // Each window writes before observing the other. Freshness against + // received writes cannot order either window's own pending choice; + // the per-key decision tuple must select the same winner on both. + const optimizationA = makeContext(); + await hydrateFromServer(optimizationA, { + independentAsrEnabled: false, + voiceInputResourceOptimizationEnabled: true, + }); + optimizationA.S.voiceInputResourceOptimizationEnabled = false; + optimizationA.mod.saveSettings({ skipServerSync: true }); + const optimizationPayloadA = optimizationA.lastSharedWrite(); + + const optimizationB = makeContext(); + await hydrateFromServer(optimizationB, { + independentAsrEnabled: false, + voiceInputResourceOptimizationEnabled: false, + }); + optimizationB.S.voiceInputResourceOptimizationEnabled = true; + optimizationB.mod.saveSettings({ skipServerSync: true }); + const optimizationPayloadB = optimizationB.lastSharedWrite(); + + const parsedA = JSON.parse(optimizationPayloadA); + const parsedB = JSON.parse(optimizationPayloadB); + const decisionA = parsedA._sharedWriteMeta.optimizationDecision; + const decisionB = parsedB._sharedWriteMeta.optimizationDecision; + assert(decisionA && decisionB, 'each explicit optimization write must carry its decision tuple'); + const aWins = decisionA.writeId > decisionB.writeId + || ( + decisionA.writeId === decisionB.writeId + && decisionA.writerId > decisionB.writerId + ); + const winningValue = aWins + ? parsedA.voiceInputResourceOptimizationEnabled + : parsedB.voiceInputResourceOptimizationEnabled; + + optimizationA.fireStorage(optimizationPayloadB); + optimizationB.fireStorage(optimizationPayloadA); + assert( + optimizationA.S.voiceInputResourceOptimizationEnabled === winningValue, + 'window A must converge on the winning optimization choice' + ); + assert( + optimizationB.S.voiceInputResourceOptimizationEnabled === winningValue, + 'window B must converge on the winning optimization choice' + ); + + // ---- Scenario 8: a real return to the restored value is fresh ---- + const restoredDecision = { + writeId: 1, + writerId: 'restored-writer', + value: false, + }; + const rebound = makeContext({ + independentAsrEnabled: false, + voiceInputResourceOptimizationEnabled: false, + _sharedWriteMeta: { + writeId: 1, + writerId: 'restored-writer', + changedKeys: [ + 'independentAsrEnabled', + 'voiceInputResourceOptimizationEnabled', + ], + asrDecision: restoredDecision, + optimizationDecision: restoredDecision, + optimizationDecisionPendingSync: false, + }, + }); + await hydrateFromServer(rebound, { + independentAsrEnabled: true, + voiceInputResourceOptimizationEnabled: true, + }, { + independentAsrEnabled: { + writeId: 2, + writerId: 'server-winner', + value: true, + }, + }); + assert( + rebound.S.independentAsrEnabled === true + && rebound.S.voiceInputResourceOptimizationEnabled === true, + 'server decisions must apply before testing a return to the restored value' + ); + rebound.S.independentAsrEnabled = false; + rebound.S.voiceInputResourceOptimizationEnabled = false; + rebound.mod.saveSettings({ skipServerSync: true }); + const reboundMeta = JSON.parse( + rebound.lastSharedWrite() + )._sharedWriteMeta; + assert( + reboundMeta.asrDecision.writeId === reboundMeta.writeId + && reboundMeta.asrDecision.writerId === reboundMeta.writerId, + 'returning to a restored ASR value is a fresh user decision' + ); + assert( + reboundMeta.optimizationDecision.writeId === reboundMeta.writeId + && reboundMeta.optimizationDecision.writerId === reboundMeta.writerId, + 'returning to a restored optimization value is a fresh user decision' + ); + + // ---- Scenario 9: an invalid optimization decision id cannot poison + // later local choices by permanently outranking the browser clock. + const poisonedOptimization = makeContext({ + voiceInputResourceOptimizationEnabled: false, + _sharedWriteMeta: { + writeId: 1, + writerId: 'poisoned-writer', + changedKeys: ['voiceInputResourceOptimizationEnabled'], + optimizationDecision: { + writeId: Number.MAX_SAFE_INTEGER, + writerId: 'poisoned-writer', + value: false, + }, + optimizationDecisionPendingSync: false, + }, + }); + poisonedOptimization.S.voiceInputResourceOptimizationEnabled = true; + poisonedOptimization.mod.saveSettings({ skipServerSync: true }); + const recoveredOptimizationMeta = JSON.parse( + poisonedOptimization.lastSharedWrite() + )._sharedWriteMeta; + assert( + recoveredOptimizationMeta.optimizationDecision + && recoveredOptimizationMeta.optimizationDecision.value === true + && recoveredOptimizationMeta.optimizationDecision.writeId + === recoveredOptimizationMeta.writeId, + 'invalid optimization decision ids must not block a fresh local choice' + ); + console.log('HARNESS_OK'); // Every sandbox timer is harness-controlled, so the process exits // naturally once main() returns and piped stdout is fully flushed. @@ -3973,6 +4227,197 @@ def test_never_settling_get_posts_only_dirty_keys_harness(): assert "HARNESS_OK" in result.stdout +def test_unsynced_optimization_decision_survives_reload_until_posted_harness(): + """A persisted explicit choice stays authoritative until its POST succeeds.""" + harness = textwrap.dedent( + """ + const fs = require('node:fs'); + const vm = require('node:vm'); + + const source = fs.readFileSync(__APP_SETTINGS_PATH__, 'utf8'); + const optimizationKey = 'voiceInputResourceOptimizationEnabled'; + + function assert(cond, msg) { + if (!cond) throw new Error('ASSERT: ' + msg); + } + + function makeContext(initialSnapshot) { + let stored = initialSnapshot ? JSON.stringify(initialSnapshot) : null; + const postCalls = []; + const getCalls = []; + const sandbox = { + console: { log() {}, warn() {}, error() {} }, + setInterval() { return 0; }, + clearInterval() {}, + setTimeout(fn, ms) { + const t = setTimeout(fn, ms); + if (t && typeof t.unref === 'function') t.unref(); + return t; + }, + clearTimeout, + localStorage: { + getItem(key) { + return key === 'project_neko_settings' ? stored : null; + }, + setItem(key, value) { + if (key === 'project_neko_settings') stored = value; + }, + removeItem() {}, + }, + document: { getElementById() { return null; } }, + fetch(url, opts) { + return new Promise((resolve, reject) => { + if (opts && opts.method === 'POST') { + postCalls.push({ body: opts.body, resolve, reject }); + } else { + getCalls.push({ resolve, reject }); + } + }); + }, + }; + sandbox.window = { + appState: { + independentAsrEnabled: true, + settingsHydrated: false, + independentAsrAuthoritative: false, + voiceInputResourceOptimizationEnabled: true, + voiceInputResourceOptimizationAuthoritative: false, + }, + appConst: {}, + appUtils: { mapRenderQualityToFollowPerf() { return 'medium'; } }, + addEventListener() {}, + removeEventListener() {}, + dispatchEvent() {}, + }; + vm.createContext(sandbox); + vm.runInContext(source, sandbox); + return { + getCalls, + postCalls, + S: sandbox.window.appState, + mod: sandbox.window.appSettings, + snapshot() { return JSON.parse(stored); }, + }; + } + + const tick = () => new Promise((resolve) => setImmediate(resolve)); + const okPost = { ok: true, json: async () => ({ success: true }) }; + + async function main() { + // This is a snapshot written by the previous PR head: it records the + // explicit decision but predates the durable pending-sync marker. + const legacyPendingSnapshot = { + [optimizationKey]: false, + _sharedWriteMeta: { + writeId: 41, + writerId: 'window-a', + changedKeys: [optimizationKey], + hydrated: true, + asrAuthoritative: false, + optimizationDecision: { + writeId: 41, + writerId: 'window-a', + value: false, + }, + }, + }; + + // If the boot GET also fails, pre-merge sync must still retry the + // durable decision. `_pickDirtySettings()` reads only the pending + // set, so restoring just dirty membership would produce no POST. + const offline = makeContext(legacyPendingSnapshot); + offline.getCalls[0].resolve({ ok: false }); + await tick(); + await tick(); + const offlineSync = offline.mod.syncSettingsToServer(); + await tick(); + assert( + offline.postCalls.length === 1, + 'failed boot GET must not forget the pending optimization POST' + ); + assert( + JSON.parse(offline.postCalls[0].body)[optimizationKey] === false, + 'dirty-only retry must carry the durable optimization choice' + ); + offline.postCalls[0].resolve(okPost); + await offlineSync; + + const ctx = makeContext(legacyPendingSnapshot); + assert(ctx.S[optimizationKey] === false, 'boot must load the local choice'); + assert(ctx.S.settingsHydrated === true, 'pending choice must hydrate the handshake'); + assert( + ctx.S.voiceInputResourceOptimizationAuthoritative === true, + 'pending choice must be authoritative for the next start handshake' + ); + + ctx.getCalls[0].resolve({ + ok: true, + json: async () => ({ + success: true, + settings: { [optimizationKey]: true }, + telemetryBranch: null, + }), + }); + await tick(); + await tick(); + assert( + ctx.S[optimizationKey] === false, + 'stale server GET must not overwrite the unsynced local choice' + ); + + const sync = ctx.mod.syncSettingsToServer(); + await tick(); + assert(ctx.postCalls.length === 1, 'pending choice must be POSTed after reload'); + assert( + JSON.parse(ctx.postCalls[0].body)[optimizationKey] === false, + 'POST must carry the pending local choice' + ); + ctx.postCalls[0].resolve(okPost); + await sync; + const syncedSnapshot = ctx.snapshot(); + assert( + syncedSnapshot._sharedWriteMeta.optimizationDecisionPendingSync === false, + 'successful POST must durably clear the pending marker' + ); + + // Once synchronization is durable, a later reload may accept newer + // server truth instead of pinning the old local choice forever. + const reloaded = makeContext(syncedSnapshot); + reloaded.getCalls[0].resolve({ + ok: true, + json: async () => ({ + success: true, + settings: { [optimizationKey]: true }, + telemetryBranch: null, + }), + }); + await tick(); + await tick(); + assert( + reloaded.S[optimizationKey] === true, + 'synced decision must no longer block server truth on a later reload' + ); + + console.log('HARNESS_OK'); + process.exitCode = 0; + } + + main().catch((err) => { + console.error(err && err.stack ? err.stack : String(err)); + process.exitCode = 1; + }); + """ + ).replace("__APP_SETTINGS_PATH__", json.dumps(str(APP_SETTINGS_PATH))) + + result = _run_settings_node_harness(harness) + assert result.returncode == 0, ( + "unsynced-optimization-reload harness failed\n" + f"stdout:\n{result.stdout}\n" + f"stderr:\n{result.stderr}" + ) + assert "HARNESS_OK" in result.stdout + + def test_failed_boot_get_keeps_posts_dirty_only_harness(): # Codex P2 (round 16): the round-15 flag was released in the merge chain's # `finally`, which also runs when the GET resolved to null (HTTP error, @@ -5167,21 +5612,21 @@ def test_concurrent_asr_toggles_are_totally_ordered_not_swapped(): assert "typeof meta.writerId === 'string' ? meta.writerId : ''" in read_fn # The comparison is (writeId, writerId) against the local decision. - outranks = settings_source.split("function _asrWriteOutranksLocalChoice(", 1)[ + outranks = settings_source.split("function _settingWriteOutranksLocalChoice(", 1)[ 1 ].split("\n }", 1)[0] # Ordering is on the DECISION that produced the value, not on the id of the # write carrying it: a monotone dirty key makes every later unrelated save # re-declare the ASR key explicit with a fresh id, which would outrank a # genuinely newer toggle elsewhere (no race required). - assert "decision.writeId > _lastAsrDecision.writeId" in outranks - assert "(decision.writerId || '') > _lastAsrDecision.writerId" in outranks + assert "decision.writeId > localDecision.writeId" in outranks + assert "(decision.writerId || '') > localDecision.writerId" in outranks # A write with neither a decision tuple nor an explicit declaration is an # incidental copy and must never outrank a local choice. assert "if (!decision) return false;" in outranks # The decision must be DERIVED (tuple, else an explicit declaration), never # taken as the incoming write itself -- that is the bug being fixed. - assert "const decision = meta.asrDecision" in outranks + assert "const decision = meta[decisionKey]" in outranks assert "const decision = meta;" not in outranks # A window's OWN explicit write must be recorded, or it has nothing to @@ -5189,8 +5634,10 @@ def test_concurrent_asr_toggles_are_totally_ordered_not_swapped(): write_fn = settings_source.split("function _writeSharedSettings(", 1)[1].split( "\n }", 1 )[0] - assert "_nextAsrDecisionWriteId(ownMeta.writeId)" in write_fn - assert "_noteAsrDecision(" in write_fn + asr_note = write_fn.split("_noteAsrDecision(", 1)[1].split(");", 1)[0] + assert "_nextAsrDecisionWriteId(ownMeta.writeId)" in asr_note + assert "ownMeta.writerId" in asr_note + assert "snapshot.independentAsrEnabled" in asr_note # Refusing authority alone is not enough: applySharedRuntimeSettings copies # independentAsrEnabled unconditionally, so the losing write must also be diff --git a/tests/unit/test_asr_detector_runtime.py b/tests/unit/test_asr_detector_runtime.py index cc2aecd643..1597d78c08 100644 --- a/tests/unit/test_asr_detector_runtime.py +++ b/tests/unit/test_asr_detector_runtime.py @@ -216,6 +216,25 @@ async def test_rnnoise_soft_gate_skips_silero_until_probable_voice() -> None: assert speech.events == (SpeechActivityEvent.SPEECH_STARTED,) +async def test_disabled_resource_optimization_never_skips_quiet_silero_pcm() -> None: + gate = _Gate((SpeechActivityEvent.SPEECH_STARTED,)) + detector = DetectorRuntime( + vad=_Vad(), + gate=gate, + rnnoise_onset_probability=0.4, + resource_optimization_enabled=False, + ) + + result = await detector.feed( + b"\x01\x00", + speech_probability=0.1, + rnnoise_available=True, + ) + + assert gate.inputs == [b"\x01\x00"] + assert result.events == (SpeechActivityEvent.SPEECH_STARTED,) + + async def test_rnnoise_unavailable_does_not_look_like_zero_probability() -> None: gate = _Gate((SpeechActivityEvent.SPEECH_STARTED,)) detector = DetectorRuntime(vad=_Vad(), gate=gate, rnnoise_onset_probability=0.4) @@ -350,6 +369,28 @@ async def test_candidate_open_prevents_rnnoise_from_skipping_followup_pcm() -> N await detector.close() +async def test_disabled_resource_optimization_never_skips_quiet_smart_turn_pcm() -> None: + detector = DetectorRuntime( + vad=_Vad(), + gate=_Gate(), + provider_policy=_smart_turn_policy(), + coordinator=_SemanticCoordinator(), + on_turn_complete=AsyncMock(), + resource_optimization_enabled=False, + ) + + result = await detector.submit_audio( + b"\x01\x00" * 160, + ingress_token=_ingress_token(), + sample_rate_hz=16_000, + speech_probability=0.1, + rnnoise_available=True, + ) + + assert result.status is DetectorSubmitStatus.ACCEPTED + await detector.close() + + async def test_smart_turn_loading_does_not_hold_detector_audio_submission() -> None: coordinator = _BlockingSemanticCoordinator(block_prepare=True) detector = DetectorRuntime( diff --git a/tests/unit/test_audio_stream_queue.py b/tests/unit/test_audio_stream_queue.py index b2c73520b1..819093c0fe 100644 --- a/tests/unit/test_audio_stream_queue.py +++ b/tests/unit/test_audio_stream_queue.py @@ -834,7 +834,6 @@ def test_hot_swap_never_rebinds_lease_mute_or_game_identity( mgr._voice_lease_focus_suppressed = True else: mgr._voice_lease_owner = "game" - mgr._voice_input_consumer_bindings["game"] = object() assert ( mgr._rebind_hot_swap_ingress_token( @@ -1425,27 +1424,63 @@ async def _reject_and_swap_session(*args, **kwargs): assert expected_clear # payload helper stays usable for the positive twin -async def test_bound_game_consumer_empty_final_sends_preview_clear(): - # A preview created before a game takeover would otherwise survive an - # empty final silently consumed by the binding branch. +async def test_game_takeover_clears_core_preview_and_empty_final_stays_terminal( + monkeypatch, +): mgr = _make_transcript_dispatch_manager() - on_final = AsyncMock() - mgr._voice_lease_owner = "none" - mgr.bind_voice_input_consumer("game", on_final) - mgr._voice_lease_owner = "game" - event = _transcript_event(mgr, " ") + mgr._set_microphone_route("independent") + route_transcript = AsyncMock(return_value=True) + monkeypatch.setattr( + "main_logic.voice_input.consumers.game.is_game_route_active", + lambda _name: True, + ) + monkeypatch.setattr( + "main_logic.voice_input.consumers.game.get_active_game_route_identity", + lambda _name: ("game", "session-a"), + ) + monkeypatch.setattr( + "main_logic.voice_input.consumers.game.route_external_voice_transcript", + route_transcript, + ) + core_turn = VoiceTurnToken(ingress=mgr._capture_ingress_token(), turn_id=7) + assert await mgr._prepare_voice_input_turn(core_turn) is True + await mgr._dispatch_voice_input_partial( + VoicePartialEvent(turn_token=core_turn, text="go"), + ) + mgr.websocket.send_json.reset_mock() - await mgr._dispatch_core_asr_transcript(event) + await mgr._apply_voice_lease_state( + owner="game", + hard_muted=False, + focus_suppressed=False, + reason="game_takeover", + force_abort=True, + ) mgr.websocket.send_json.assert_awaited_once_with(_preview_clear_payload(mgr)) - on_final.assert_not_awaited() + route_transcript.assert_not_awaited() - # Negative validation: a non-empty game final reaches the consumer and - # sends nothing on the core websocket. mgr.websocket.send_json.reset_mock() - non_empty = _transcript_event(mgr, "go left", turn_id=9) - await mgr._dispatch_core_asr_transcript(non_empty) - on_final.assert_awaited_once_with(non_empty) + empty = _transcript_event(mgr, " ", turn_id=9) + assert await mgr._prepare_voice_input_turn(empty.turn_token) is True + await mgr._dispatch_voice_input_final(empty) + await mgr._voice_input_registry.wait_idle() + route_transcript.assert_not_awaited() + mgr.websocket.send_json.assert_not_awaited() + + non_empty = _transcript_event(mgr, "go left", turn_id=10) + assert await mgr._prepare_voice_input_turn(non_empty.turn_token) is True + await mgr._dispatch_voice_input_final(non_empty) + route_transcript.assert_awaited_once_with( + "Test", + "go left", + request_id=( + f"asr-{non_empty.turn_token.ingress.session_epoch}-" + f"{non_empty.turn_token.turn_id}" + ), + game_type="game", + session_id="session-a", + ) mgr.websocket.send_json.assert_not_awaited() @@ -1492,10 +1527,16 @@ async def _prepare_preview_turn(mgr: LLMSessionManager, turn_id: int) -> str: async def _send_preview_partial(mgr: LLMSessionManager, text: str) -> dict: + turn_token = getattr(mgr, "_core_asr_preview_turn_token", None) + if turn_token is None: + turn_token = VoiceTurnToken( + ingress=mgr._capture_ingress_token(), + turn_id=0, + ) await mgr._send_core_asr_preview( VoicePartialEvent( + turn_token=turn_token, text=text, - session_epoch=mgr._capture_ingress_token().session_epoch, ) ) return mgr.websocket.send_json.await_args.args[0] @@ -2077,7 +2118,7 @@ async def test_startup_failure_revokes_the_lease_except_for_the_game_owner( # Codex P2. A startup failure (provider connect, credentials, config) pins # the route blocked but can never emit a BLOCKED lifecycle event, so the # backstop is the only server-side stop. The galgame route holds the lease - # through its own consumer binding and must not be collaterally revoked. + # through its built-in Registry consumer and must not be collaterally revoked. mgr = _make_routable_audio_manager(True) _authorize_core_lease(mgr) mgr._begin_voice_input_connection("socket-a") @@ -2445,6 +2486,52 @@ def passes_handshake(fn, keyword_value): ) +def test_start_session_snapshots_resource_optimization_handshake_before_await(): + import ast + + source = ( + Path(__file__).resolve().parents[2] / "main_logic" / "core" / "lifecycle.py" + ).read_text(encoding="utf-8") + tree = ast.parse(source) + functions = { + node.name: node + for node in ast.walk(tree) + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + } + start_session = functions["start_session"] + snapshot_name = "session_resource_optimization_handshake_override" + snapshot_lines = [ + node.lineno + for node in ast.walk(start_session) + if isinstance(node, ast.Name) + and node.id == snapshot_name + and isinstance(node.ctx, ast.Store) + ] + await_lines = [ + node.lineno for node in ast.walk(start_session) if isinstance(node, ast.Await) + ] + + assert snapshot_lines + assert await_lines + assert max(snapshot_lines) < min(await_lines) + + def passes_override(fn, keyword_value): + return any( + kw.arg == "resource_optimization_override" + and isinstance(kw.value, ast.Name) + and kw.value.id == keyword_value + for node in ast.walk(fn) + if isinstance(node, ast.Call) + for kw in node.keywords + ) + + assert passes_override(start_session, snapshot_name) + assert passes_override( + functions["_start_session_activate"], + "resource_optimization_override", + ) + + async def test_start_without_a_snapshot_still_reads_the_shared_handshake(): # Non-vacuity, and the contract for the internal re-entry paths (hot swap, # device change): with no snapshot supplied the shared field still decides. @@ -2716,7 +2803,7 @@ async def _unreadable(*, strict: bool = False, **_kwargs): async def test_absent_settings_still_default_without_failing_closed(): # The other half of the strict read: an ABSENT file is not a failure. A - # first run has no settings yet and must keep defaulting normally rather + # first run has no settings yet and must use the enabled default rather # than blocking the route. mgr = _make_routable_audio_manager(True) mgr._begin_voice_input_connection("socket-a") @@ -2726,6 +2813,15 @@ async def test_absent_settings_still_default_without_failing_closed(): mgr.send_status = AsyncMock() mgr._asr_runtime.abort = AsyncMock() + async def _ready_start(**_kwargs): + return AsrStartResult( + AsrStartStatus.READY, + provider="qwen", + session_epoch=mgr._capture_ingress_token().session_epoch, + ) + + mgr._asr_runtime.start = AsyncMock(side_effect=_ready_start) + async def _empty(**_kwargs): return {} @@ -2736,7 +2832,7 @@ async def _empty(**_kwargs): ): await LLMSessionManager._start_independent_asr_if_enabled(mgr, "audio") - assert mgr._asr_route_mode == "native" + assert mgr._asr_route_mode == "independent" async def test_fail_closed_chokepoint_honours_the_callers_own_predicate(): @@ -2760,7 +2856,7 @@ async def test_fail_closed_chokepoint_honours_the_callers_own_predicate(): async def test_fail_closed_chokepoint_exempts_the_game_owner(): - # The galgame gate owns the mic through its own consumer binding and tears + # The galgame gate owns the mic through its built-in consumer route and tears # down via GAME_ROUTE_ENDED, so it must be neither notified nor revoked. mgr, recorder = _blocked_route_manager_with_recorder() mgr._voice_lease_owner = "game" @@ -3030,7 +3126,7 @@ async def test_audio_start_ack_is_not_duplicated_for_a_single_window(): async def test_audio_start_ack_does_not_reach_the_game_microphone(): # Same exemption the text path carries: the galgame gate owns the mic - # through its own consumer binding, and a session_started handler that + # through its built-in consumer route, and a session_started handler that # calls stopRecording would release a lease this ack never meant to touch. recorder, chat = _fake_socket_pair() mgr = _make_routable_audio_manager(True) diff --git a/tests/unit/test_check_core_contracts.py b/tests/unit/test_check_core_contracts.py index fd3e6d864d..e5508c3951 100644 --- a/tests/unit/test_check_core_contracts.py +++ b/tests/unit/test_check_core_contracts.py @@ -52,7 +52,10 @@ def test_imported_paths_resolves_package_alias_attribute_chains( assert expected in referenced -def _dynamic_import_results(contract_checker, source: str) -> list[tuple[str | None, bool]]: +def _dynamic_import_results( + contract_checker, + source: str, +) -> list[tuple[tuple[str, ...] | None, bool]]: tree = ast.parse(source) aliases = contract_checker.module_alias_paths(tree, "main_logic.asr_client") return [ @@ -80,7 +83,7 @@ def test_dynamic_import_target_resolves_string_literal_forms( source: str, ) -> None: assert _dynamic_import_results(contract_checker, source) == [ - ("main_logic.core", True) + (("main_logic.core",), True) ] @@ -91,6 +94,31 @@ def test_dynamic_import_target_reports_non_literal_argument(contract_checker) -> assert _dynamic_import_results(contract_checker, source) == [(None, True)] +@pytest.mark.unit +@pytest.mark.parametrize( + "source", + [ + ( + "import importlib\n\n" + "def load():\n" + ' return importlib.import_module(".core", "main_logic")\n' + ), + ( + "def load():\n" + ' return __import__("main_logic", fromlist=["core"])\n' + ), + ], +) +def test_dynamic_import_gate_resolves_relative_and_fromlist_targets( + contract_checker, + source: str, +) -> None: + assert ( + "asr_client must not import main_logic.core (dynamic import)" + in _dynamic_import_violation_messages(contract_checker, source) + ) + + @pytest.mark.unit def test_dynamic_import_target_ignores_unrelated_calls(contract_checker) -> None: source = "def import_module(name):\n return name\nmod = import_module('main_logic.core')" @@ -109,6 +137,15 @@ def _dynamic_import_violation_messages(contract_checker, source: str) -> list[st ] +@pytest.mark.unit +def test_dynamic_import_docstring_describes_multiple_forbidden_prefixes( + contract_checker, +) -> None: + docstring = contract_checker._dynamic_import_violations.__doc__ or "" + + assert "forbidden prefixes" in docstring + + @pytest.mark.unit @pytest.mark.parametrize( "source", @@ -334,6 +371,129 @@ def test_run_flags_dynamic_imports_of_core_inside_asr_client( assert any("non-literal module name" in message for message in messages) +@pytest.mark.unit +@pytest.mark.parametrize( + "source", + [ + "import main_logic.core\n", + "from main_logic import asr_client\n", + "from main_logic.voice_turn import audio_input\n", + "import main_routers.game_router\n", + "from utils import preferences\n", + "import plugin.plugins.demo\n", + "import importlib\nimportlib.import_module('main_logic.core')\n", + ], +) +def test_run_flags_forbidden_voice_input_dependencies( + contract_checker, + tmp_path, + source: str, +) -> None: + _write_minimal_core_layout(tmp_path) + for package in (tmp_path / "main_routers", tmp_path / "utils"): + package.mkdir() + (package / "__init__.py").write_text("", encoding="utf-8") + plugin = tmp_path / "plugin" / "plugins" / "demo" + plugin.mkdir(parents=True) + for package in (plugin.parent.parent, plugin.parent, plugin): + (package / "__init__.py").write_text("", encoding="utf-8") + voice_input = tmp_path / "main_logic" / "voice_input" + voice_input.mkdir() + probe = voice_input / "probe.py" + probe.write_text(source, encoding="utf-8") + + messages = [ + violation.message + for violation in contract_checker.run(tmp_path) + if violation.path == probe + and violation.code == "VOICE_INPUT_LAYERING" + ] + + assert messages + + +@pytest.mark.unit +def test_missing_voice_input_registry_uses_voice_input_violation_code( + contract_checker, + tmp_path, +) -> None: + _write_minimal_core_layout(tmp_path) + missing = tmp_path / "main_logic" / "voice_input" + + violations = [ + violation + for violation in contract_checker.run(tmp_path) + if violation.path == missing + ] + + assert len(violations) == 1 + assert violations[0].code == "VOICE_INPUT_LAYERING" + assert ( + violations[0].message + == "required layering path is missing (VOICE_INPUT_LAYERING)" + ) + + +@pytest.mark.unit +def test_run_accepts_frozen_voice_input_dependency_direction( + contract_checker, + tmp_path, +) -> None: + _write_minimal_core_layout(tmp_path) + voice_input = tmp_path / "main_logic" / "voice_input" + consumers = voice_input / "consumers" + consumers.mkdir(parents=True) + probe = consumers / "game.py" + probe.write_text( + "from main_logic.voice_input.contracts import VoiceInputConsumer\n" + "from main_logic.voice_turn.contracts import VoiceTurnToken\n" + "from utils.game_route_state import is_game_route_active\n", + encoding="utf-8", + ) + + violations = [ + violation + for violation in contract_checker.run(tmp_path) + if violation.path == probe + and violation.code == "VOICE_INPUT_LAYERING" + ] + + assert violations == [] + + +@pytest.mark.unit +@pytest.mark.parametrize( + "source", + [ + "import main_logic.voice_input\n", + "import importlib\n" + "importlib.import_module('main_logic.voice_input.registry')\n", + ], +) +def test_run_flags_asr_client_importing_core_owned_registry( + contract_checker, + tmp_path, + source: str, +) -> None: + _write_minimal_core_layout(tmp_path) + (tmp_path / "main_logic" / "voice_input").mkdir() + asr_client = tmp_path / "main_logic" / "asr_client" + asr_client.mkdir() + probe = asr_client / "probe.py" + probe.write_text(source, encoding="utf-8") + + messages = [ + violation.message + for violation in contract_checker.run(tmp_path) + if violation.path == probe and violation.code == "ASR_LAYERING" + ] + + assert any( + "asr_client must not import main_logic.voice_input" in message + for message in messages + ) + + @pytest.mark.unit def test_run_flags_forbidden_runtime_reads_through_local_alias( contract_checker, diff --git a/tests/unit/test_core_independent_asr.py b/tests/unit/test_core_independent_asr.py index c00b1b3581..b5c0e15433 100644 --- a/tests/unit/test_core_independent_asr.py +++ b/tests/unit/test_core_independent_asr.py @@ -14,6 +14,8 @@ from main_logic.core.asr_runtime import AsrRuntimeMixin, _HotSwapAudioFrame from main_logic.asr_client.runtime import AsrStartResult, AsrStartStatus from main_logic.asr_client.detector_runtime import DetectorFeedResult, DetectorRuntime +from main_logic.voice_input import VoiceInputDispatchResult +from main_logic.voice_input.consumers import CoreChatTurnContext from main_logic.asr_client.lifecycle import ( VoiceLifecycleEvent, VoiceLifecycleState, @@ -726,10 +728,23 @@ async def test_game_takeover_wins_even_if_provider_clear_fails() -> None: assert runtime._asr_lifecycle.snapshot.state.value == "suspended" -async def test_bound_game_consumer_reuses_smart_turn_asr_without_core() -> None: +async def test_game_consumer_reuses_smart_turn_asr_without_core( + monkeypatch, +) -> None: runtime = _Runtime() - on_final = AsyncMock() - binding = runtime.bind_voice_input_consumer("game", on_final) + route_transcript = AsyncMock(return_value=True) + monkeypatch.setattr( + "main_logic.voice_input.consumers.game.is_game_route_active", + lambda _name: True, + ) + monkeypatch.setattr( + "main_logic.voice_input.consumers.game.get_active_game_route_identity", + lambda _name: ("game", "session-a"), + ) + monkeypatch.setattr( + "main_logic.voice_input.consumers.game.route_external_voice_transcript", + route_transcript, + ) assert ( await runtime._handle_voice_input_control( @@ -749,18 +764,18 @@ async def test_bound_game_consumer_reuses_smart_turn_asr_without_core() -> None: await runtime._handle_independent_asr_final("play", epoch, "qwen") await runtime._wait_asr_transcript_dispatch_idle() - event = on_final.await_args.args[0] - assert isinstance(event, VoiceTranscriptEvent) - assert event.text == "play" - assert event.provider == "qwen" - assert event.turn_token.turn_id > 0 + route_transcript.assert_awaited_once_with( + "Test", + "play", + request_id=f"asr-{epoch}-1", + game_type="game", + session_id="session-a", + ) runtime.handle_new_message.assert_not_awaited() runtime.handle_input_transcript.assert_not_awaited() runtime.session.create_response.assert_not_awaited() assert runtime._omni_mic_audio_bytes == 0 - with pytest.raises(RuntimeError, match="RELEASE_LEASE_FIRST"): - runtime.unbind_voice_input_consumer(binding) assert ( await runtime._handle_voice_input_control( "lease_sync", @@ -771,13 +786,23 @@ async def test_bound_game_consumer_reuses_smart_turn_asr_without_core() -> None: ) is True ) - assert runtime.unbind_voice_input_consumer(binding) is True -async def test_bound_game_consumer_ignores_empty_final() -> None: +async def test_game_consumer_ignores_empty_final(monkeypatch) -> None: runtime = _Runtime() - on_final = AsyncMock() - runtime.bind_voice_input_consumer("game", on_final) + route_transcript = AsyncMock(return_value=True) + monkeypatch.setattr( + "main_logic.voice_input.consumers.game.is_game_route_active", + lambda _name: True, + ) + monkeypatch.setattr( + "main_logic.voice_input.consumers.game.get_active_game_route_identity", + lambda _name: ("game", "session-a"), + ) + monkeypatch.setattr( + "main_logic.voice_input.consumers.game.route_external_voice_transcript", + route_transcript, + ) assert ( await runtime._handle_voice_input_control( "lease_sync", @@ -797,19 +822,156 @@ async def test_bound_game_consumer_ignores_empty_final() -> None: text="", ) - await runtime._dispatch_core_asr_transcript(event) + assert await runtime._prepare_voice_input_turn(event.turn_token) is True + await runtime._dispatch_voice_input_final(event) + await runtime._voice_input_registry.wait_idle() - on_final.assert_not_awaited() + route_transcript.assert_not_awaited() runtime.handle_new_message.assert_not_awaited() runtime.handle_input_transcript.assert_not_awaited() runtime.session.create_response.assert_not_awaited() -async def test_bound_game_consumer_accepts_real_pcm_through_pipeline() -> None: +async def test_game_takeover_pre_abort_window_rejects_stale_core_turn( + monkeypatch, +) -> None: + runtime = _Runtime() + route_transcript = AsyncMock(return_value=True) + monkeypatch.setattr( + "main_logic.voice_input.consumers.game.is_game_route_active", + lambda _name: True, + ) + monkeypatch.setattr( + "main_logic.voice_input.consumers.game.get_active_game_route_identity", + lambda _name: ("game", "session-a"), + ) + monkeypatch.setattr( + "main_logic.voice_input.consumers.game.route_external_voice_transcript", + route_transcript, + ) + runtime.session.abandon_external_voice_turn = MagicMock() + assert ( + await runtime._handle_voice_input_control( + "lease_sync", + 1, + owner="core", + hard_muted=False, + focus_suppressed=False, + ) + is True + ) + _install_ready_lifecycle(runtime, "openai") + runtime._asr_session.close = AsyncMock() + runtime._asr_session.signal_user_activity_end = AsyncMock() + + preview_clear_started = asyncio.Event() + release_preview_clear = asyncio.Event() + + async def block_preview_clear(payload: dict[str, object]) -> None: + if ( + payload.get("type") == "user_transcript_preview" + and payload.get("text") == "" + ): + preview_clear_started.set() + await release_preview_clear.wait() + + runtime.websocket = SimpleNamespace( + send_json=AsyncMock(side_effect=block_preview_clear), + ) + epoch = runtime._asr_session_epoch + await _start_and_seal_turn(runtime, "openai") + sealed = runtime._asr_runtime._asr_sealed_turn_token + assert sealed is not None + stale_ingress = sealed.turn.ingress + + await runtime._handle_independent_asr_final("", epoch, "openai") + await asyncio.wait_for(preview_clear_started.wait(), 1) + + takeover = asyncio.create_task( + runtime._handle_voice_input_control("game_takeover", 2) + ) + endpoint: asyncio.Task[None] | None = None + try: + for _ in range(100): + if runtime._voice_lease_owner == "game": + break + await asyncio.sleep(0) + assert runtime._voice_lease_owner == "game" + assert runtime._voice_lease_generation == 2 + assert takeover.done() is False + + await runtime._handle_independent_asr_activity( + SpeechActivityEvent.SPEECH_STARTED, + epoch, + ) + prepared_token = runtime._asr_runtime._asr_partial_turn_token + endpoint = asyncio.create_task( + runtime._handle_independent_asr_endpoint(epoch) + ) + for _ in range(100): + if endpoint.done(): + break + await asyncio.sleep(0) + await runtime._handle_independent_asr_final( + "stale core audio", + epoch, + "openai", + ) + await runtime._asr_runtime.wait_transcript_idle() + + assert stale_ingress.lease_generation == 1 + assert prepared_token is None + route_transcript.assert_not_awaited() + finally: + release_preview_clear.set() + pending = [takeover] + if endpoint is not None: + pending.append(endpoint) + results = await asyncio.wait_for(asyncio.gather(*pending), 1) + assert results[0] is True + await runtime._voice_input_registry.wait_idle() + + runtime.session.abandon_external_voice_turn.assert_called_once_with( + f"asr-{epoch}-1" + ) + + +async def test_rejected_voice_input_final_is_observable(monkeypatch) -> None: + runtime = _Runtime() + _install_ready_lifecycle(runtime, "qwen") + event = VoiceTranscriptEvent( + turn_token=runtime._asr_runtime._capture_turn_token( + runtime._asr_lifecycle + ), + provider="qwen", + text="hello", + ) + runtime._voice_input_registry.dispatch_final = AsyncMock( + return_value=VoiceInputDispatchResult.REJECTED + ) + debug = MagicMock() + monkeypatch.setattr(core_asr_runtime_module.logger, "debug", debug) + + await runtime._dispatch_voice_input_final(event) + + debug.assert_called_once() + assert "voice input final rejected" in debug.call_args.args[0] + + +async def test_game_consumer_accepts_real_pcm_through_pipeline( + monkeypatch, +) -> None: runtime = _Runtime() runtime.is_active = True runtime.is_hot_swap_imminent = False - runtime.bind_voice_input_consumer("game", AsyncMock()) + monkeypatch.setattr( + "main_logic.voice_input.consumers.game.is_game_route_active", + lambda _name: True, + ) + monkeypatch.setattr( + "main_logic.voice_input.consumers.game.get_active_game_route_identity", + lambda _name: ("game", "session-a"), + ) assert ( await runtime._handle_voice_input_control( "lease_sync", @@ -852,9 +1014,12 @@ async def test_bound_game_consumer_accepts_real_pcm_through_pipeline() -> None: ) -async def test_bound_game_consumer_submit_preserves_owner_identity() -> None: +async def test_game_consumer_submit_preserves_owner_identity(monkeypatch) -> None: runtime = _Runtime() - runtime.bind_voice_input_consumer("game", AsyncMock()) + monkeypatch.setattr( + "main_logic.voice_input.consumers.game.is_game_route_active", + lambda _name: True, + ) assert ( await runtime._handle_voice_input_control( "lease_sync", @@ -892,12 +1057,22 @@ async def test_bound_game_consumer_submit_preserves_owner_identity() -> None: ) -async def test_game_consumer_failure_never_falls_back_to_core() -> None: +async def test_game_consumer_failure_never_falls_back_to_core( + monkeypatch, +) -> None: runtime = _Runtime() - on_final = AsyncMock(side_effect=RuntimeError("consumer failed")) - runtime.bind_voice_input_consumer( - "game", - on_final, + route_transcript = AsyncMock(side_effect=RuntimeError("consumer failed")) + monkeypatch.setattr( + "main_logic.voice_input.consumers.game.is_game_route_active", + lambda _name: True, + ) + monkeypatch.setattr( + "main_logic.voice_input.consumers.game.get_active_game_route_identity", + lambda _name: ("game", "session-a"), + ) + monkeypatch.setattr( + "main_logic.voice_input.consumers.game.route_external_voice_transcript", + route_transcript, ) await runtime._handle_voice_input_control( "lease_sync", @@ -913,21 +1088,34 @@ async def test_game_consumer_failure_never_falls_back_to_core() -> None: await runtime._handle_independent_asr_final("play", epoch, "qwen") await runtime._wait_asr_transcript_dispatch_idle() - on_final.assert_awaited_once() - event = on_final.await_args.args[0] - assert isinstance(event, VoiceTranscriptEvent) - assert event.text == "play" - assert event.provider == "qwen" + route_transcript.assert_awaited_once_with( + "Test", + "play", + request_id=f"asr-{epoch}-1", + game_type="game", + session_id="session-a", + ) runtime.handle_new_message.assert_not_awaited() runtime.handle_input_transcript.assert_not_awaited() runtime.session.create_response.assert_not_awaited() assert runtime._omni_mic_audio_bytes == 0 -async def test_game_final_cannot_cross_lease_back_to_core() -> None: +async def test_game_final_cannot_cross_lease_back_to_core(monkeypatch) -> None: runtime = _Runtime() - on_final = AsyncMock() - runtime.bind_voice_input_consumer("game", on_final) + route_transcript = AsyncMock(return_value=True) + monkeypatch.setattr( + "main_logic.voice_input.consumers.game.is_game_route_active", + lambda _name: True, + ) + monkeypatch.setattr( + "main_logic.voice_input.consumers.game.get_active_game_route_identity", + lambda _name: ("game", "session-a"), + ) + monkeypatch.setattr( + "main_logic.voice_input.consumers.game.route_external_voice_transcript", + route_transcript, + ) await runtime._handle_voice_input_control( "lease_sync", 1, @@ -949,15 +1137,18 @@ async def test_game_final_cannot_cross_lease_back_to_core() -> None: await runtime._handle_independent_asr_final("stale", epoch, "qwen") await runtime._wait_asr_transcript_dispatch_idle() - on_final.assert_not_awaited() + route_transcript.assert_not_awaited() runtime.handle_input_transcript.assert_not_awaited() runtime.session.create_response.assert_not_awaited() assert runtime._omni_mic_audio_bytes == 0 -async def test_hard_mute_overrides_bound_game_consumer() -> None: +async def test_hard_mute_overrides_game_consumer(monkeypatch) -> None: runtime = _Runtime() - runtime.bind_voice_input_consumer("game", AsyncMock()) + monkeypatch.setattr( + "main_logic.voice_input.consumers.game.is_game_route_active", + lambda _name: True, + ) await runtime._handle_voice_input_control( "lease_sync", @@ -1080,11 +1271,46 @@ async def block_prepare(*, turn_id: str) -> None: runtime.handle_new_message.assert_not_awaited() external_turn_id = f"asr-{token.ingress.session_epoch}-{token.turn_id}" assert runtime.session.abandon_external_voice_turn.call_args_list == [ - call(None), call(external_turn_id), ] +async def test_stale_core_prepare_restores_previous_preview_owner() -> None: + runtime = _Runtime() + _install_ready_lifecycle(runtime) + prepare_started = asyncio.Event() + release_prepare = asyncio.Event() + + async def block_prepare(*, turn_id: str) -> None: + del turn_id + prepare_started.set() + await release_prepare.wait() + + runtime.session.prepare_external_voice_turn = AsyncMock(side_effect=block_prepare) + runtime.session.abandon_external_voice_turn = MagicMock() + token = runtime._asr_runtime._capture_turn_token(runtime._asr_lifecycle) + previous_token = replace(token, turn_id=token.turn_id + 100) + previous_turn_id = ( + f"asr-{previous_token.ingress.session_epoch}-{previous_token.turn_id}" + ) + runtime._core_asr_preview_turn_id = previous_turn_id + runtime._core_asr_preview_turn_token = previous_token + runtime._core_asr_preview_text = "previous partial" + + prepare_task = asyncio.create_task(runtime._prepare_core_voice_turn(token)) + await asyncio.wait_for(prepare_started.wait(), 1) + runtime._voice_input_transition_generation += 1 + release_prepare.set() + + assert await asyncio.wait_for(prepare_task, 1) is False + assert runtime._core_asr_preview_turn_id == previous_turn_id + assert runtime._core_asr_preview_turn_token == previous_token + assert runtime._core_asr_preview_text == "previous partial" + runtime.session.abandon_external_voice_turn.assert_called_once_with( + f"asr-{token.ingress.session_epoch}-{token.turn_id}" + ) + + async def test_turn_endpoint_seals_immediately_before_provider_final() -> None: runtime = _Runtime() runtime._asr_session = type("Asr", (), {"is_ready": True})() @@ -1169,6 +1395,16 @@ async def test_empty_final_completes_turn_without_core_injection() -> None: await _start_and_seal_turn(runtime) turn_id = runtime.session.prepare_external_voice_turn.await_args.kwargs["turn_id"] + await runtime._handle_independent_asr_final( + "", + runtime._asr_session_epoch, + "qwen", + ) + # Teardown racing the queued empty final may win or lose, but both paths + # terminate the same pinned route. Repeated invalidation and a duplicate + # provider final must not produce a second cancellation/abandonment. + runtime._invalidate_voice_pcm_sync("duplicate_after_empty_final") + runtime._invalidate_voice_pcm_sync("duplicate_after_empty_final") await runtime._handle_independent_asr_final( "", runtime._asr_session_epoch, @@ -1184,6 +1420,36 @@ async def test_empty_final_completes_turn_without_core_injection() -> None: assert runtime._omni_mic_audio_bytes == 0 +async def test_blocked_consumer_callback_does_not_block_next_turn_lifecycle() -> ( + None +): + runtime = _Runtime() + callback_started = asyncio.Event() + release_callback = asyncio.Event() + + async def block_first_final(*_args, **_kwargs) -> bool: + callback_started.set() + await release_callback.wait() + return True + + runtime.handle_input_transcript.side_effect = block_first_final + await _start_and_seal_turn(runtime, "qwen") + epoch = runtime._asr_session_epoch + + await runtime._handle_independent_asr_final("first", epoch, "qwen") + await asyncio.wait_for(callback_started.wait(), 1) + await runtime._handle_independent_asr_activity( + SpeechActivityEvent.SPEECH_STARTED, + epoch, + ) + + assert runtime._asr_lifecycle.snapshot.state is VoiceLifecycleState.ACTIVE + assert runtime._asr_turn_prepared is True + release_callback.set() + await runtime._wait_asr_transcript_dispatch_idle() + runtime.session.create_response.assert_awaited_once_with("first") + + async def test_prepare_failure_releases_keyed_external_turn_pause() -> None: runtime = _Runtime() _install_ready_lifecycle(runtime) @@ -1200,13 +1466,42 @@ async def test_prepare_failure_releases_keyed_external_turn_pause() -> None: ) -async def test_final_transcript_submits_to_the_session_it_was_validated_against() -> None: - # Codex P2. _dispatch_core_asr_transcript validates `self.session is - # session_ref`, then awaits _restore_core_asr_preview_after_final -- a - # websocket send. _submit_core_voice_turn used to re-read self.session after - # that await, discarding the validation: a hot swap promoting a replacement - # session inside the window made it inject this conversation's transcript - # into the next one, producing a reply in the wrong conversation. +async def test_registry_prepare_rejection_releases_keyed_external_turn_pause() -> ( + None +): + runtime = _Runtime() + _install_ready_lifecycle(runtime) + runtime.session.abandon_external_voice_turn = MagicMock() + runtime.handle_new_message = AsyncMock(side_effect=RuntimeError("history failed")) + token = runtime._asr_runtime._capture_turn_token(runtime._asr_lifecycle) + + assert await runtime._prepare_voice_input_turn(token) is False + + runtime.session.abandon_external_voice_turn.assert_called_once_with( + f"asr-{token.ingress.session_epoch}-{token.turn_id}" + ) + + +async def test_registry_cancelled_prepare_releases_keyed_external_turn_pause() -> ( + None +): + runtime = _Runtime() + _install_ready_lifecycle(runtime) + runtime.session.abandon_external_voice_turn = MagicMock() + runtime.handle_new_message = AsyncMock(side_effect=asyncio.CancelledError) + token = runtime._asr_runtime._capture_turn_token(runtime._asr_lifecycle) + + with pytest.raises(asyncio.CancelledError): + await runtime._prepare_voice_input_turn(token) + await runtime._voice_input_registry.wait_idle() + + runtime.session.abandon_external_voice_turn.assert_called_once_with( + f"asr-{token.ingress.session_epoch}-{token.turn_id}" + ) + + +async def test_final_transcript_drops_new_conversation_swap_mid_restore() -> None: + """A real conversation transition still invalidates the prepared final.""" runtime = _Runtime() _install_ready_lifecycle(runtime) runtime.session.abandon_external_voice_turn = MagicMock() @@ -1218,6 +1513,7 @@ async def test_final_transcript_submits_to_the_session_it_was_validated_against( replacement.abandon_external_voice_turn = MagicMock() async def _hot_swap_mid_restore(*_args, **_kwargs) -> None: + runtime._voice_input_transition_generation += 1 runtime.session = replacement runtime._restore_core_asr_preview_after_final = _hot_swap_mid_restore @@ -1233,13 +1529,118 @@ async def _hot_swap_mid_restore(*_args, **_kwargs) -> None: # assertions below would pass while modelling an ordinary final with no hot # swap at all. Pin that the swap really happened first. assert runtime.session is replacement - # The turn lands on the session that produced it... - timed_session.create_response.assert_awaited_once_with("hello") - # ...and never touches the one that replaced it. + timed_session.create_response.assert_not_awaited() replacement.create_response.assert_not_awaited() replacement.submit_external_voice_turn.assert_not_awaited() +async def test_pre_dispatch_hot_swap_reprepares_turn_on_promoted_session() -> None: + """A same-route hot swap transfers the final off the closed old arbiter.""" + runtime = _Runtime() + _install_ready_lifecycle(runtime) + runtime.session.abandon_external_voice_turn = MagicMock() + prepared_session = runtime.session + prepared_session.create_response.side_effect = RuntimeError("closed arbiter") + + replacement = type("Omni", (), {})() + replacement.create_response = AsyncMock() + replacement.submit_external_voice_turn = AsyncMock() + replacement.prepare_external_voice_turn = AsyncMock() + replacement.abandon_external_voice_turn = MagicMock() + + token = runtime._asr_runtime._capture_turn_token(runtime._asr_lifecycle) + runtime.session = replacement + await runtime._dispatch_core_asr_transcript( + VoiceTranscriptEvent(turn_token=token, provider="qwen", text="prepared"), + session_ref=prepared_session, + ) + + prepared_session.create_response.assert_not_awaited() + replacement.prepare_external_voice_turn.assert_awaited_once_with( + turn_id=f"asr-{token.ingress.session_epoch}-{token.turn_id}" + ) + replacement.submit_external_voice_turn.assert_awaited_once_with( + "prepared", + turn_id=f"asr-{token.ingress.session_epoch}-{token.turn_id}", + ) + + +async def test_final_waits_for_shared_swap_barrier_then_uses_promoted_session() -> None: + runtime = _Runtime() + _install_ready_lifecycle(runtime) + prepared_session = runtime.session + prepared_session.abandon_external_voice_turn = MagicMock() + + replacement = type("Omni", (), {})() + replacement.submit_external_voice_turn = AsyncMock() + replacement.prepare_external_voice_turn = AsyncMock() + replacement.abandon_external_voice_turn = MagicMock() + + token = runtime._asr_runtime._capture_turn_token(runtime._asr_lifecycle) + await runtime._core_voice_session_swap_lock.acquire() + dispatch = asyncio.create_task( + runtime._dispatch_core_asr_transcript( + VoiceTranscriptEvent( + turn_token=token, + provider="qwen", + text="after swap", + ), + session_ref=prepared_session, + ) + ) + try: + await asyncio.sleep(0) + assert dispatch.done() is False + runtime.session = replacement + finally: + runtime._core_voice_session_swap_lock.release() + await dispatch + + replacement.prepare_external_voice_turn.assert_awaited_once_with( + turn_id=f"asr-{token.ingress.session_epoch}-{token.turn_id}" + ) + replacement.submit_external_voice_turn.assert_awaited_once_with( + "after swap", + turn_id=f"asr-{token.ingress.session_epoch}-{token.turn_id}", + ) + + +async def test_final_swap_barrier_timeout_drops_without_blocking_dispatcher() -> None: + runtime = _Runtime() + _install_ready_lifecycle(runtime) + runtime.session.abandon_external_voice_turn = MagicMock() + runtime._core_voice_session_swap_barrier_timeout_s = 0.01 + token = runtime._asr_runtime._capture_turn_token(runtime._asr_lifecycle) + + await runtime._core_voice_session_swap_lock.acquire() + try: + await asyncio.wait_for( + runtime._dispatch_core_asr_transcript( + VoiceTranscriptEvent( + turn_token=token, + provider="qwen", + text="bounded", + ) + ), + timeout=0.5, + ) + finally: + runtime._core_voice_session_swap_lock.release() + + runtime.session.create_response.assert_not_awaited() + + +async def test_hot_swap_lifecycle_guards_close_and_promote_with_voice_barrier() -> None: + source = inspect.getsource( + core_module.LLMSessionManager._perform_final_swap_sequence + ) + + barrier = source.index("async with core_voice_session_lock") + close = source.index("await old_main_session.close()") + promote = source.index("self.session = new_session") + assert barrier < close < promote + + async def test_final_transcript_is_dropped_when_the_route_leaves_core_mid_restore() -> None: # Codex P2, the other half of the case above. Pinning session_ref protects # only the SESSION: a game or text takeover landing inside the preview @@ -1295,12 +1696,142 @@ async def test_transcript_dispatch_failure_releases_keyed_external_turn_pause() ) +async def test_cancelled_preview_clear_still_releases_keyed_external_turn_pause() -> None: + runtime = _Runtime() + session = runtime.session + session.abandon_external_voice_turn = MagicMock() + runtime._send_core_asr_preview_clear = AsyncMock( + side_effect=asyncio.CancelledError + ) + token = VoiceTurnToken(ingress=runtime._capture_ingress_token(), turn_id=7) + context = CoreChatTurnContext( + token=token, + external_turn_id="asr-cancelled-preview", + session_ref=session, + ) + + with pytest.raises(asyncio.CancelledError): + await runtime._cancel_core_chat_voice_turn(context, "takeover") + + session.abandon_external_voice_turn.assert_called_once_with( + "asr-cancelled-preview" + ) + + +@pytest.mark.parametrize("stale_guard", ["ingress", "owner"]) +async def test_stale_final_guard_releases_keyed_external_turn_pause( + stale_guard: str, +) -> None: + runtime = _Runtime() + _install_ready_lifecycle(runtime) + runtime.session.abandon_external_voice_turn = MagicMock() + token = runtime._asr_runtime._capture_turn_token(runtime._asr_lifecycle) + event = VoiceTranscriptEvent( + turn_token=token, + provider="qwen", + text="hello", + ) + if stale_guard == "ingress": + runtime._asr_audio_generation += 1 + else: + runtime._voice_lease_owner = "game" + + await runtime._dispatch_core_asr_transcript(event) + + runtime.session.abandon_external_voice_turn.assert_called_once_with( + f"asr-{token.ingress.session_epoch}-{token.turn_id}" + ) + + +async def test_abort_bumps_generation_before_waiting_for_registry_cancel() -> None: + runtime = _Runtime() + order: list[str] = [] + runtime._asr_runtime.abort = AsyncMock( + side_effect=lambda _reason: order.append("abort") + ) + runtime._invalidate_voice_pcm_sync = MagicMock( + side_effect=lambda _reason: order.append("invalidate") + ) + runtime._voice_input_registry.wait_idle = AsyncMock( + side_effect=lambda: order.append("wait_idle") + ) + + await runtime._abort_independent_asr("ingress_backpressure") + + assert order == ["abort", "invalidate", "wait_idle"] + + +async def test_suspend_advances_runtime_barrier_before_waiting_for_registry_cancel() -> ( + None +): + runtime = _Runtime() + order: list[str] = [] + runtime._invalidate_voice_pcm_sync = MagicMock( + side_effect=lambda _reason: order.append("invalidate") + ) + runtime._asr_runtime.suspend = AsyncMock( + side_effect=lambda _reason: order.append("suspend") + ) + runtime._voice_input_registry.wait_idle = AsyncMock( + side_effect=lambda: order.append("wait_idle") + ) + + await runtime._suspend_independent_asr("game_takeover") + + assert order == ["suspend", "invalidate", "wait_idle"] + + +@pytest.mark.parametrize( + ("previous_owner", "owner", "reason", "barrier_method"), + [ + ("core", "game", "game_takeover", "suspend"), + ("game", "core", "game_release", "abort"), + ("core", "none", "connection_closed", "abort"), + ], +) +async def test_voice_lease_advances_runtime_barrier_before_waiting_for_registry( + previous_owner: str, + owner: str, + reason: str, + barrier_method: str, +) -> None: + runtime = _Runtime() + runtime._voice_lease_owner = previous_owner + order: list[str] = [] + runtime._invalidate_voice_pcm_sync = MagicMock( + side_effect=lambda _reason: order.append("invalidate") + ) + runtime._asr_runtime.suspend = AsyncMock( + side_effect=lambda _reason: order.append("suspend") + ) + runtime._asr_runtime.abort = AsyncMock( + side_effect=lambda _reason: order.append("abort") + ) + runtime._asr_runtime.resume = AsyncMock() + runtime._voice_input_registry.wait_idle = AsyncMock( + side_effect=lambda: order.append("wait_idle") + ) + + await runtime._apply_voice_lease_state( + owner=owner, + hard_muted=False, + focus_suppressed=False, + reason=reason, + force_abort=True, + ) + + assert order == ["invalidate", barrier_method, "wait_idle"] + + @pytest.mark.parametrize("operation", ["abort", "close"]) async def test_core_asr_teardown_force_releases_external_turn_pause( operation: str, ) -> None: runtime = _Runtime() + _install_ready_lifecycle(runtime, "qwen") runtime.session.abandon_external_voice_turn = MagicMock() + token = runtime._asr_runtime._capture_turn_token(runtime._asr_lifecycle) + assert await runtime._prepare_voice_input_turn(token) is True if operation == "abort": runtime._asr_runtime.abort = AsyncMock() @@ -1311,12 +1842,17 @@ async def test_core_asr_teardown_force_releases_external_turn_pause( await runtime._close_independent_asr(next_route_mode="blocked") runtime._asr_runtime.close.assert_awaited_once_with() - runtime.session.abandon_external_voice_turn.assert_called_once_with(None) + runtime.session.abandon_external_voice_turn.assert_called_once_with( + f"asr-{token.ingress.session_epoch}-{token.turn_id}", + ) async def test_current_asr_failure_force_releases_external_turn_pause() -> None: runtime = _Runtime() + _install_ready_lifecycle(runtime, "qwen") runtime.session.abandon_external_voice_turn = MagicMock() + token = runtime._asr_runtime._capture_turn_token(runtime._asr_lifecycle) + assert await runtime._prepare_voice_input_turn(token) is True await runtime._handle_core_asr_failure( AsrFailureEvent( @@ -1326,7 +1862,73 @@ async def test_current_asr_failure_force_releases_external_turn_pause() -> None: ) ) - runtime.session.abandon_external_voice_turn.assert_called_once_with(None) + runtime.session.abandon_external_voice_turn.assert_called_once_with( + f"asr-{token.ingress.session_epoch}-{token.turn_id}", + ) + + +async def test_registry_cancellation_abandons_the_prepared_session_after_swap() -> ( + None +): + runtime = _Runtime() + _install_ready_lifecycle(runtime, "qwen") + original_session = runtime.session + original_session.abandon_external_voice_turn = MagicMock() + token = runtime._asr_runtime._capture_turn_token(runtime._asr_lifecycle) + assert await runtime._prepare_voice_input_turn(token) is True + + replacement = type("Omni", (), {})() + replacement.abandon_external_voice_turn = MagicMock() + runtime.session = replacement + assert runtime._voice_input_registry.invalidate_utterance( + token, + reason="session_hot_swap", + ) + await runtime._voice_input_registry.wait_idle() + + original_session.abandon_external_voice_turn.assert_called_once_with( + f"asr-{token.ingress.session_epoch}-{token.turn_id}", + ) + replacement.abandon_external_voice_turn.assert_not_called() + + +async def test_runtime_close_preserves_manager_lifetime_registry_builtins() -> None: + runtime = _Runtime() + _install_ready_lifecycle(runtime, "qwen") + registry = runtime._voice_input_registry + core_registration = runtime._core_chat_voice_input_registration + game_registration = runtime._game_voice_input_registration + token = runtime._asr_runtime._capture_turn_token(runtime._asr_lifecycle) + assert await runtime._prepare_voice_input_turn(token) is True + runtime._asr_runtime.close = AsyncMock() + + await runtime._close_independent_asr(next_route_mode="blocked") + runtime._ensure_asr_runtime_state() + runtime._ensure_asr_runtime_state() + + assert runtime._voice_input_registry is registry + assert runtime._core_chat_voice_input_registration is core_registration + assert runtime._game_voice_input_registration is game_registration + assert core_registration.closed is False + assert game_registration.closed is False + assert len(registry._records) == 2 + + +async def test_runtime_state_initializes_and_backfills_phase4a_fields() -> None: + runtime = _Runtime() + + assert runtime._voice_input_resource_optimization_handshake_override is None + assert runtime._voice_input_resource_optimization_session_value is None + assert runtime._core_asr_preview_turn_token is None + + del runtime._voice_input_resource_optimization_handshake_override + del runtime._voice_input_resource_optimization_session_value + del runtime._core_asr_preview_turn_token + runtime._ensure_asr_runtime_state() + + assert runtime._voice_input_resource_optimization_handshake_override is None + assert runtime._voice_input_resource_optimization_session_value is None + assert runtime._core_asr_preview_turn_token is None async def test_provider_final_watchdog_blocks_only_independent_asr() -> None: @@ -1760,9 +2362,10 @@ async def test_stale_overlap_onset_is_not_replayed_after_final() -> None: assert runtime._asr_lifecycle.snapshot.state is VoiceLifecycleState.WARM_IDLE assert runtime._asr_turn_prepared is False - assert [ - call.args[0] for call in runtime.handle_input_transcript.await_args_list - ] == ["first"] + # The prepared Registry route retains the original full VoiceTurnToken. + # Rotating audio_generation makes the later final a different route, so + # strict routing drops it together with the stale overlap onset. + runtime.handle_input_transcript.assert_not_awaited() assert runtime.handle_new_message.await_count == 1 @@ -1981,9 +2584,9 @@ async def test_stale_completed_overlap_is_dropped_at_next_endpoint() -> None: await runtime._handle_independent_asr_final("ghost", epoch, "openai") await runtime._wait_asr_transcript_dispatch_idle() - assert [ - call.args[0] for call in runtime.handle_input_transcript.await_args_list - ] == ["first"] + # Both the overlap credit and the final belong to the superseded full + # VoiceTurnToken once audio_generation rotates. + runtime.handle_input_transcript.assert_not_awaited() assert runtime.handle_new_message.await_count == 1 @@ -3078,6 +3681,13 @@ async def test_asr_stream_failure_never_replays_the_failed_frame_to_omni() -> No async def test_asr_backpressure_reports_specific_blocking_status() -> None: runtime = _Runtime() + blocking_status_sent = asyncio.Event() + + async def record_status(message: str) -> None: + if "ASR_STREAM_BACKPRESSURE" in message: + blocking_status_sent.set() + + runtime.send_status.side_effect = record_status asr = type("Asr", (), {})() asr.is_ready = True asr.stream_audio = AsyncMock( @@ -3094,6 +3704,7 @@ async def test_asr_backpressure_reports_specific_blocking_status() -> None: sample_rate_hz=16_000, ) await runtime._asr_audio_dispatcher.wait_idle() + await asyncio.wait_for(blocking_status_sent.wait(), 1) assert "ASR_STREAM_BACKPRESSURE" in runtime.send_status.await_args.args[0] assert runtime._asr_route_mode == "blocked" @@ -4064,7 +4675,11 @@ async def test_partial_preview_is_display_only_and_epoch_guarded() -> None: runtime.websocket = websocket runtime.current_speech_id = "speech-current" runtime._set_microphone_route("independent") + await _install_active_smart_turn(runtime) epoch = runtime._asr_session_epoch + token = runtime._asr_runtime._asr_partial_turn_token + assert token is not None + assert runtime._activate_asr_audio_dispatcher(runtime._asr_lifecycle, token) await runtime._send_independent_asr_preview(" draft ", epoch) await runtime._send_independent_asr_preview("stale", epoch + 1) @@ -4074,11 +4689,42 @@ async def test_partial_preview_is_display_only_and_epoch_guarded() -> None: "type": "user_transcript_preview", "text": "draft", "turn_id": "speech-current", + "asr_turn_id": f"asr-{epoch}-1", } ) runtime.handle_input_transcript.assert_not_awaited() +async def test_partial_preview_keeps_prepared_token_and_rejects_after_abort() -> None: + runtime = _Runtime() + runtime._set_microphone_route("independent") + on_partial = AsyncMock() + runtime._asr_runtime._callbacks = replace( + runtime._asr_runtime._callbacks, + on_partial=on_partial, + ) + await _install_active_smart_turn(runtime) + epoch = runtime._asr_session_epoch + captured_token = runtime._asr_runtime._asr_partial_turn_token + assert captured_token is not None + assert runtime._activate_asr_audio_dispatcher( + runtime._asr_lifecycle, + captured_token, + ) + + await runtime._send_independent_asr_preview("current", epoch) + + event = on_partial.await_args.args[0] + assert event.turn_token is captured_token + assert event.session_epoch == epoch + on_partial.reset_mock() + + runtime._asr_audio_dispatcher.abort(captured_token) + await runtime._send_independent_asr_preview("late", epoch) + + on_partial.assert_not_awaited() + + async def test_start_failure_blocks_omni_without_leaking_error(monkeypatch) -> None: import main_logic.asr_client.runtime as runtime_module @@ -4296,6 +4942,110 @@ async def test_start_session_handshake_false_overrides_persisted_enabled( assert runtime._asr_route_mode == "native" +async def test_resource_optimization_handshake_false_overrides_persisted_enabled( + monkeypatch, +) -> None: + runtime = _Runtime() + runtime.core_api_type = "gemini" + monkeypatch.setattr( + core_module, + "aload_global_conversation_settings", + AsyncMock( + return_value={ + "independentAsrEnabled": True, + "voiceInputResourceOptimizationEnabled": True, + } + ), + ) + start_mock = AsyncMock( + return_value=AsrStartResult( + status=AsrStartStatus.FAILED, + failure_code="ASR_START_STALE", + ) + ) + monkeypatch.setattr(runtime._asr_runtime, "start", start_mock) + + runtime.set_voice_input_resource_optimization_handshake(False) + await runtime._start_independent_asr_if_enabled("audio") + + assert start_mock.await_args.kwargs["resource_optimization_enabled"] is False + + +async def test_provider_restart_reuses_accepted_session_optimization( + monkeypatch, +) -> None: + runtime = _Runtime() + runtime.core_api_type = "gemini" + runtime.input_mode = "audio" + monkeypatch.setattr( + core_module, + "aload_global_conversation_settings", + AsyncMock( + return_value={ + "independentAsrEnabled": True, + "voiceInputResourceOptimizationEnabled": True, + } + ), + ) + start_mock = AsyncMock( + return_value=AsrStartResult( + status=AsrStartStatus.READY, + provider="qwen", + session_epoch=0, + ) + ) + monkeypatch.setattr(runtime._asr_runtime, "start", start_mock) + + await runtime._start_independent_asr_if_enabled( + "audio", + resource_optimization_override=False, + ) + assert runtime._voice_input_resource_optimization_session_value is False + + # A losing/deduplicated request may overwrite the shared handshake, but a + # provider-changing restart still belongs to the already accepted session. + runtime.set_voice_input_resource_optimization_handshake(True) + runtime.core_api_type = "openai" + await runtime._reconcile_independent_asr_after_core_change() + + assert start_mock.await_count == 2 + assert all( + call.kwargs["resource_optimization_enabled"] is False + for call in start_mock.await_args_list + ) + + +@pytest.mark.parametrize("malformed", ["false", 0, 1, [False], {"enabled": False}]) +async def test_resource_optimization_handshake_malformed_falls_back_to_persisted( + monkeypatch, + malformed, +) -> None: + runtime = _Runtime() + runtime.core_api_type = "gemini" + monkeypatch.setattr( + core_module, + "aload_global_conversation_settings", + AsyncMock( + return_value={ + "independentAsrEnabled": True, + "voiceInputResourceOptimizationEnabled": True, + } + ), + ) + start_mock = AsyncMock( + return_value=AsrStartResult( + status=AsrStartStatus.FAILED, + failure_code="ASR_START_STALE", + ) + ) + monkeypatch.setattr(runtime._asr_runtime, "start", start_mock) + + runtime.set_voice_input_resource_optimization_handshake(malformed) + await runtime._start_independent_asr_if_enabled("audio") + + assert start_mock.await_args.kwargs["resource_optimization_enabled"] is True + + async def test_start_session_handshake_missing_falls_back_to_persisted( monkeypatch, ) -> None: @@ -4323,6 +5073,29 @@ async def test_start_session_handshake_missing_falls_back_to_persisted( start_mock.assert_awaited_once() +async def test_missing_independent_asr_setting_defaults_enabled(monkeypatch) -> None: + runtime = _Runtime() + runtime.core_api_type = "gemini" + monkeypatch.setattr( + core_module, + "aload_global_conversation_settings", + AsyncMock(return_value={}), + ) + start_mock = AsyncMock( + return_value=AsrStartResult( + status=AsrStartStatus.FAILED, + failure_code="ASR_START_STALE", + ) + ) + monkeypatch.setattr(runtime._asr_runtime, "start", start_mock) + + await runtime._start_independent_asr_if_enabled("audio") + + start_mock.assert_awaited_once() + assert start_mock.await_args.kwargs["resource_optimization_enabled"] is True + assert runtime._asr_route_mode != "native" + + @pytest.mark.parametrize("malformed", ["true", 1, 0, [True], {"enabled": True}]) async def test_start_session_handshake_malformed_value_is_ignored( monkeypatch, @@ -5295,10 +6068,20 @@ async def test_injection_failure_is_reported_once_without_provider_body() -> Non runtime.session.create_response.assert_awaited_once_with("hello") -async def test_session_swap_during_transcript_drops_old_final_injection() -> None: +async def test_session_swap_during_transcript_reprepares_promoted_final() -> None: runtime = _Runtime() old_session = runtime.session - new_session = type("Omni", (), {"create_response": AsyncMock()})() + old_session.create_response.side_effect = RuntimeError("closed arbiter") + new_session = type( + "Omni", + (), + { + "create_response": AsyncMock(), + "prepare_external_voice_turn": AsyncMock(), + "submit_external_voice_turn": AsyncMock(), + "abandon_external_voice_turn": MagicMock(), + }, + )() async def swap_session(*_args, **_kwargs) -> bool: runtime.session = new_session @@ -5316,6 +6099,8 @@ async def swap_session(*_args, **_kwargs) -> bool: old_session.create_response.assert_not_awaited() new_session.create_response.assert_not_awaited() + new_session.prepare_external_voice_turn.assert_awaited_once() + new_session.submit_external_voice_turn.assert_awaited_once() async def test_game_takeover_during_transcript_drops_stale_core_final() -> None: @@ -5543,27 +6328,35 @@ async def test_partial_preview_requires_current_core_lease() -> None: runtime.websocket = websocket runtime._set_microphone_route("independent") epoch = runtime._asr_session_epoch + token = VoiceTurnToken( + ingress=runtime._capture_ingress_token(), + turn_id=1, + ) + stale_token = VoiceTurnToken( + ingress=replace(token.ingress, session_epoch=epoch + 1), + turn_id=token.turn_id, + ) runtime._voice_lease_owner = "game" await runtime._send_core_asr_preview( - VoicePartialEvent(text="game", session_epoch=epoch) + VoicePartialEvent(turn_token=token, text="game") ) runtime._voice_lease_owner = "core" runtime._voice_lease_hard_muted = True await runtime._send_core_asr_preview( - VoicePartialEvent(text="muted", session_epoch=epoch) + VoicePartialEvent(turn_token=token, text="muted") ) runtime._voice_lease_hard_muted = False runtime._voice_lease_focus_suppressed = True await runtime._send_core_asr_preview( - VoicePartialEvent(text="focused", session_epoch=epoch) + VoicePartialEvent(turn_token=token, text="focused") ) runtime._voice_lease_focus_suppressed = False await runtime._send_core_asr_preview( - VoicePartialEvent(text="stale", session_epoch=epoch + 1) + VoicePartialEvent(turn_token=stale_token, text="stale") ) await runtime._send_core_asr_preview( - VoicePartialEvent(text="current", session_epoch=epoch) + VoicePartialEvent(turn_token=token, text="current") ) websocket.send_json.assert_awaited_once_with( @@ -5721,7 +6514,7 @@ def _set_route_then_supersede(mode: str) -> None: async def test_runtime_failure_leaves_the_game_lease_alone() -> None: - # The galgame route holds the mic through its own consumer binding and tears + # The galgame route holds the mic through its built-in consumer route and tears # down via GAME_ROUTE_ENDED; re-basing the identity must not start # collaterally revoking it. runtime = _Runtime() @@ -6157,6 +6950,41 @@ async def test_notification_waiting_on_lock_drops_same_epoch_stale_identity( assert runtime._asr_route_mode == "independent" +async def test_failure_cancellation_can_publish_without_notification_deadlock() -> ( + None +): + runtime = _Runtime() + runtime._set_microphone_route("independent") + current_epoch = runtime._asr_session_epoch + + async def cancellation_wait_idle() -> None: + assert runtime._asr_notification_lock.locked() is False + await runtime._send_core_asr_status( + AsrStatusEvent( + code="ASR_CANCEL_CLEANUP", + provider="plugin-consumer", + session_epoch=current_epoch, + ) + ) + + runtime._voice_input_registry.wait_idle = AsyncMock( + side_effect=cancellation_wait_idle + ) + + await asyncio.wait_for( + runtime._handle_core_asr_failure( + AsrFailureEvent( + code="ASR_INDEPENDENT_FAILED", + provider="current-provider", + session_epoch=current_epoch, + ) + ), + 1, + ) + + assert "ASR_CANCEL_CLEANUP" in str(runtime.send_status.await_args_list) + + def _lease_resync_statuses(runtime: _Runtime) -> list[dict]: statuses = [ json.loads(call.args[0]) for call in runtime.send_status.await_args_list @@ -6534,11 +7362,8 @@ def resolver(core_type: str): "_create_asr_session_from_selection", lambda _core_type, **_kwargs: session, ) - monkeypatch.setattr( - runtime_module, - "DetectorRuntime", - MagicMock(return_value=_ReadyDetector()), - ) + detector_factory = MagicMock(return_value=_ReadyDetector()) + monkeypatch.setattr(runtime_module, "DetectorRuntime", detector_factory) result = await runtime._asr_runtime.start( route_key="qwen", @@ -6548,6 +7373,9 @@ def resolver(core_type: str): assert result.status is AsrStartStatus.READY assert len(resolver_threads) == 1 assert resolver_threads[0] is not threading.main_thread() + assert ( + detector_factory.call_args.kwargs["resource_optimization_enabled"] is False + ) async def test_teardown_routines_share_one_turn_state_reset() -> None: diff --git a/tests/unit/test_session_start_guard.py b/tests/unit/test_session_start_guard.py index 3e5aba6188..f880dc5152 100644 --- a/tests/unit/test_session_start_guard.py +++ b/tests/unit/test_session_start_guard.py @@ -151,7 +151,13 @@ async def test_cross_mode_start_waits_then_restarts_in_requested_mode(): # 重入禁用二次跨模式重启（深度封顶 1）。 restart_mock.assert_awaited_once_with( - ws, False, "audio", user_initiated=True, _allow_cross_mode_restart=False + ws, + False, + "audio", + user_initiated=True, + _allow_cross_mode_restart=False, + handshake_override=None, + resource_optimization_override=None, ) @@ -243,7 +249,13 @@ async def test_cross_mode_start_restarts_even_if_inflight_failed_internally(): # param ws still connected + self.websocket is None ⇒ restart proceeds. restart_mock.assert_awaited_once_with( - ws, False, "audio", user_initiated=True, _allow_cross_mode_restart=False + ws, + False, + "audio", + user_initiated=True, + _allow_cross_mode_restart=False, + handshake_override=None, + resource_optimization_override=None, ) diff --git a/tests/unit/test_voice_input_consumers.py b/tests/unit/test_voice_input_consumers.py new file mode 100644 index 0000000000..760a1c8632 --- /dev/null +++ b/tests/unit/test_voice_input_consumers.py @@ -0,0 +1,233 @@ +from __future__ import annotations + +import asyncio +from unittest.mock import AsyncMock + +import pytest + +from main_logic.voice_input.consumers.core_chat import CoreChatVoiceInputConsumer +from main_logic.voice_input.consumers.game import GameVoiceInputConsumer +from main_logic.voice_turn.contracts import ( + VoiceIngressToken, + VoicePartialEvent, + VoiceTranscriptEvent, + VoiceTurnToken, +) + + +pytestmark = pytest.mark.asyncio + + +def _token(turn_id: int = 1) -> VoiceTurnToken: + return VoiceTurnToken( + ingress=VoiceIngressToken( + connection_id="connection", + lease_generation=7, + route_generation=9, + audio_generation=13, + session_epoch=11, + ), + turn_id=turn_id, + ) + + +async def test_core_consumer_cancels_the_session_captured_at_prepare() -> None: + original_session = object() + current_session = [original_session] + prepared = AsyncMock(return_value=True) + cancelled = AsyncMock() + consumer = CoreChatVoiceInputConsumer( + session_ref=lambda: current_session[0], + on_prepare=prepared, + on_partial_event=AsyncMock(), + on_final_event=AsyncMock(), + on_cancelled_event=cancelled, + ) + token = _token() + + assert await consumer.prepare_turn(token) is True + current_session[0] = object() + await consumer.on_cancelled(token, "consumer_switched") + + context = cancelled.await_args.args[0] + assert context.token == token + assert context.external_turn_id == "asr-11-1" + assert context.session_ref is original_session + assert cancelled.await_args.args[1] == "consumer_switched" + + +async def test_core_consumer_final_uses_prepared_context_once() -> None: + original_session = object() + final = AsyncMock() + cancelled = AsyncMock() + consumer = CoreChatVoiceInputConsumer( + session_ref=lambda: original_session, + on_prepare=AsyncMock(return_value=True), + on_partial_event=AsyncMock(), + on_final_event=final, + on_cancelled_event=cancelled, + ) + token = _token() + event = VoiceTranscriptEvent(turn_token=token, provider="qwen", text="hello") + + assert await consumer.prepare_turn(token) is True + await consumer.on_final(event) + await consumer.on_cancelled(token, "late_cancel") + + assert final.await_count == 1 + assert final.await_args.args[0] == event + assert final.await_args.args[1].session_ref is original_session + cancelled.assert_not_awaited() + + +async def test_core_consumer_rejected_prepare_retains_context_for_cancel() -> None: + session = object() + cancelled = AsyncMock() + consumer = CoreChatVoiceInputConsumer( + session_ref=lambda: session, + on_prepare=AsyncMock(return_value=False), + on_partial_event=AsyncMock(), + on_final_event=AsyncMock(), + on_cancelled_event=cancelled, + ) + token = _token() + + assert await consumer.prepare_turn(token) is False + await consumer.on_cancelled(token, "prepare_rejected") + await consumer.on_cancelled(token, "duplicate_cancel") + + cancelled.assert_awaited_once() + context, reason = cancelled.await_args.args + assert context.token == token + assert context.session_ref is session + assert reason == "prepare_rejected" + + +async def test_core_consumer_cancelled_prepare_retains_context_for_cancel() -> None: + session = object() + cancelled = AsyncMock() + consumer = CoreChatVoiceInputConsumer( + session_ref=lambda: session, + on_prepare=AsyncMock(side_effect=asyncio.CancelledError), + on_partial_event=AsyncMock(), + on_final_event=AsyncMock(), + on_cancelled_event=cancelled, + ) + token = _token() + + with pytest.raises(asyncio.CancelledError): + await consumer.prepare_turn(token) + await consumer.on_cancelled(token, "prepare_cancelled") + + context, reason = cancelled.await_args.args + assert context.token == token + assert context.session_ref is session + assert reason == "prepare_cancelled" + + +async def test_game_consumer_uses_token_derived_request_id(monkeypatch) -> None: + routed = AsyncMock(return_value=True) + monkeypatch.setattr( + "main_logic.voice_input.consumers.game.is_game_route_active", + lambda name: name == "Lan", + ) + monkeypatch.setattr( + "main_logic.voice_input.consumers.game.get_active_game_route_identity", + lambda name: ("puzzle", "session-a") if name == "Lan" else None, + ) + monkeypatch.setattr( + "main_logic.voice_input.consumers.game.route_external_voice_transcript", + routed, + ) + consumer = GameVoiceInputConsumer(lanlan_name=lambda: "Lan") + token = _token(turn_id=3) + event = VoiceTranscriptEvent(turn_token=token, provider="qwen", text="play") + + assert consumer.is_available() is True + assert await consumer.prepare_turn(token) is True + await consumer.on_final(event) + + routed.assert_awaited_once_with( + "Lan", + "play", + request_id="asr-11-3", + game_type="puzzle", + session_id="session-a", + ) + + +async def test_game_consumer_surfaces_route_delivery_failure(monkeypatch) -> None: + routed = AsyncMock(return_value=False) + monkeypatch.setattr( + "main_logic.voice_input.consumers.game.get_active_game_route_identity", + lambda _name: ("puzzle", "session-a"), + ) + monkeypatch.setattr( + "main_logic.voice_input.consumers.game.route_external_voice_transcript", + routed, + ) + consumer = GameVoiceInputConsumer(lanlan_name=lambda: "Lan") + token = _token(turn_id=4) + event = VoiceTranscriptEvent( + turn_token=token, + provider="qwen", + text="play", + ) + + assert await consumer.prepare_turn(token) is True + with pytest.raises(RuntimeError, match="GAME_VOICE_TRANSCRIPT_NOT_ROUTED"): + await consumer.on_final(event) + + routed.assert_awaited_once_with( + "Lan", + "play", + request_id="asr-11-4", + game_type="puzzle", + session_id="session-a", + ) + + +async def test_game_consumer_is_fail_closed_when_route_is_unavailable( + monkeypatch, +) -> None: + monkeypatch.setattr( + "main_logic.voice_input.consumers.game.is_game_route_active", + lambda _name: False, + ) + monkeypatch.setattr( + "main_logic.voice_input.consumers.game.get_active_game_route_identity", + lambda _name: None, + ) + consumer = GameVoiceInputConsumer(lanlan_name=lambda: "Lan") + + assert consumer.is_available() is False + assert await consumer.prepare_turn(_token()) is False + + +async def test_game_consumer_pins_route_identity_at_prepare(monkeypatch) -> None: + routed = AsyncMock(return_value=True) + active_identity = ["maze", "session-a"] + monkeypatch.setattr( + "main_logic.voice_input.consumers.game.get_active_game_route_identity", + lambda _name: tuple(active_identity), + ) + monkeypatch.setattr( + "main_logic.voice_input.consumers.game.route_external_voice_transcript", + routed, + ) + consumer = GameVoiceInputConsumer(lanlan_name=lambda: "Lan") + token = _token(turn_id=5) + + assert await consumer.prepare_turn(token) is True + active_identity[:] = ["maze", "session-b"] + await consumer.on_final( + VoiceTranscriptEvent(turn_token=token, provider="qwen", text="left") + ) + + routed.assert_awaited_once_with( + "Lan", + "left", + request_id="asr-11-5", + game_type="maze", + session_id="session-a", + ) diff --git a/tests/unit/test_voice_input_registry.py b/tests/unit/test_voice_input_registry.py new file mode 100644 index 0000000000..dc3c544067 --- /dev/null +++ b/tests/unit/test_voice_input_registry.py @@ -0,0 +1,585 @@ +from __future__ import annotations + +import asyncio +from dataclasses import replace +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest + +from main_logic.voice_input import ( + BuiltinVoiceInputConsumer, + VoiceInputConsumerCapabilities, + VoiceInputDispatchResult, + VoiceInputHandleError, + VoiceInputRegistry, +) +from main_logic.voice_input.plugin_api import PluginVoiceInputRegistrar +from main_logic.voice_turn.contracts import ( + VoiceIngressToken, + VoicePartialEvent, + VoiceTranscriptEvent, + VoiceTurnToken, +) + + +pytestmark = pytest.mark.asyncio + + +def _turn(turn_id: int = 1) -> VoiceTurnToken: + return VoiceTurnToken( + ingress=VoiceIngressToken( + session_epoch=7, + connection_id="socket-a", + lease_generation=3, + route_generation=5, + audio_generation=11, + ), + turn_id=turn_id, + ) + + +def _consumer(*, available: bool = True) -> SimpleNamespace: + return SimpleNamespace( + is_available=lambda: available, + prepare_turn=AsyncMock(return_value=True), + on_partial=AsyncMock(), + on_final=AsyncMock(), + on_cancelled=AsyncMock(), + ) + + +def _register_chat( + registry: VoiceInputRegistry, + consumer: SimpleNamespace, +): + return registry.register_builtin( + BuiltinVoiceInputConsumer.CORE_CHAT, + consumer, + capabilities=VoiceInputConsumerCapabilities( + accepts_partial=True, + accepts_final=True, + ), + ) + + +async def test_builtin_route_delivers_partial_and_final_once() -> None: + registry = VoiceInputRegistry() + chat = _consumer() + registration = _register_chat(registry, chat) + registry.activate(registration.handle) + turn = _turn() + + assert registry.begin_utterance(turn) is True + assert await registry.prepare_utterance(turn) is True + partial = VoicePartialEvent(turn_token=turn, text="hel") + assert ( + await registry.dispatch_partial(partial) + is VoiceInputDispatchResult.DELIVERED + ) + event = VoiceTranscriptEvent(turn_token=turn, provider="qwen", text="hello") + assert ( + await registry.dispatch_final(event) + is VoiceInputDispatchResult.DELIVERED + ) + assert ( + await registry.dispatch_final(event) + is VoiceInputDispatchResult.REJECTED + ) + + chat.prepare_turn.assert_awaited_once_with(turn) + chat.on_partial.assert_awaited_once_with(partial) + chat.on_final.assert_awaited_once_with(event) + + +async def test_partial_routes_only_by_its_own_full_turn_token() -> None: + registry = VoiceInputRegistry() + chat = _consumer() + registration = _register_chat(registry, chat) + registry.activate(registration.handle) + first = _turn(1) + second = _turn(2) + assert registry.begin_utterance(first) + assert registry.begin_utterance(second) + assert await registry.prepare_utterance(first) + assert await registry.prepare_utterance(second) + + second_partial = VoicePartialEvent(turn_token=second, text="second") + first_partial = VoicePartialEvent(turn_token=first, text="first") + assert ( + await registry.dispatch_partial(second_partial) + is VoiceInputDispatchResult.DELIVERED + ) + assert ( + await registry.dispatch_partial(first_partial) + is VoiceInputDispatchResult.DELIVERED + ) + unknown = VoicePartialEvent(turn_token=_turn(3), text="unknown") + assert ( + await registry.dispatch_partial(unknown) + is VoiceInputDispatchResult.REJECTED + ) + + assert chat.on_partial.await_args_list == [ + ((second_partial,), {}), + ((first_partial,), {}), + ] + + +async def test_switch_invalidates_pinned_routes_without_fallback() -> None: + registry = VoiceInputRegistry() + chat = _consumer() + game = _consumer() + chat_registration = _register_chat(registry, chat) + game_registration = registry.register_builtin( + BuiltinVoiceInputConsumer.GAME, + game, + ) + registry.activate(game_registration.handle) + turn = _turn() + assert registry.begin_utterance(turn) + assert await registry.prepare_utterance(turn) + + registry.activate(chat_registration.handle) + await registry.wait_idle() + stale = VoiceTranscriptEvent(turn_token=turn, provider="qwen", text="play") + + assert ( + await registry.dispatch_final(stale) + is VoiceInputDispatchResult.REJECTED + ) + game.on_cancelled.assert_awaited_once_with(turn, "consumer_switched") + game.on_final.assert_not_awaited() + chat.on_final.assert_not_awaited() + + +async def test_closed_registration_rejects_stale_handle_and_final() -> None: + registry = VoiceInputRegistry() + game = _consumer() + registration = registry.register_builtin( + BuiltinVoiceInputConsumer.GAME, + game, + ) + registry.activate(registration.handle) + turn = _turn() + assert registry.begin_utterance(turn) + + assert registration.close() is True + assert registration.close() is False + await registry.wait_idle() + + with pytest.raises(VoiceInputHandleError, match="STALE"): + registry.activate(registration.handle) + assert ( + await registry.dispatch_final( + VoiceTranscriptEvent( + turn_token=turn, + provider="qwen", + text="stale", + ) + ) + is VoiceInputDispatchResult.REJECTED + ) + game.on_cancelled.assert_awaited_once_with( + turn, + "consumer_unregistered", + ) + game.on_final.assert_not_awaited() + + +async def test_foreign_and_forged_handles_are_rejected() -> None: + registry = VoiceInputRegistry() + other = VoiceInputRegistry() + first = _register_chat(registry, _consumer()) + second = registry.register_builtin( + BuiltinVoiceInputConsumer.GAME, + _consumer(), + ) + foreign = other.register_builtin( + BuiltinVoiceInputConsumer.CORE_CHAT, + _consumer(), + ) + + with pytest.raises(VoiceInputHandleError, match="FOREIGN"): + registry.activate(foreign.handle) + + forged = replace(first.handle, identity=second.handle.identity) + with pytest.raises(VoiceInputHandleError, match="STALE"): + registry.activate(forged) + + +async def test_consumer_capability_blocks_partial_delivery() -> None: + registry = VoiceInputRegistry() + game = _consumer() + registration = registry.register_builtin( + BuiltinVoiceInputConsumer.GAME, + game, + capabilities=VoiceInputConsumerCapabilities( + accepts_partial=False, + accepts_final=True, + ), + ) + registry.activate(registration.handle) + turn = _turn() + assert registry.begin_utterance(turn) + + assert ( + await registry.dispatch_partial( + VoicePartialEvent(turn_token=turn, text="hidden") + ) + is VoiceInputDispatchResult.REJECTED + ) + game.on_partial.assert_not_awaited() + + +async def test_fake_plugin_registers_through_namespaced_registrar() -> None: + registry = VoiceInputRegistry() + plugin = _consumer() + registrar = registry.issue_plugin_registrar("study-companion") + + assert isinstance(registrar, PluginVoiceInputRegistrar) + registration = registrar.register_consumer( + plugin, + capabilities=VoiceInputConsumerCapabilities( + accepts_partial=True, + accepts_final=True, + ), + ) + assert registration.handle.identity.namespace == "plugin" + assert registration.handle.identity.name == "study-companion" + + registry.activate(registration.handle) + turn = _turn() + assert registry.begin_utterance(turn) + assert await registry.prepare_utterance(turn) + event = VoiceTranscriptEvent( + turn_token=turn, + provider="soniox", + text="note", + ) + assert ( + await registry.dispatch_final(event) + is VoiceInputDispatchResult.DELIVERED + ) + plugin.on_final.assert_awaited_once_with(event) + + +@pytest.mark.parametrize( + "plugin_id", + ("core_chat", "game", "", "spaces are invalid", "../escape"), +) +async def test_plugin_registrar_rejects_reserved_or_invalid_ids( + plugin_id: str, +) -> None: + registry = VoiceInputRegistry() + + with pytest.raises(ValueError, match="PLUGIN_ID_INVALID"): + registry.issue_plugin_registrar(plugin_id) + + +async def test_unavailable_consumer_keeps_input_fail_closed() -> None: + registry = VoiceInputRegistry() + registration = registry.register_builtin( + BuiltinVoiceInputConsumer.GAME, + _consumer(available=False), + ) + registry.activate(registration.handle) + + assert registry.active_accepts_input is False + assert registry.begin_utterance(_turn()) is False + + +async def test_consumer_becoming_unavailable_before_prepare_consumes_route() -> None: + registry = VoiceInputRegistry() + game = _consumer() + registration = registry.register_builtin( + BuiltinVoiceInputConsumer.GAME, + game, + ) + registry.activate(registration.handle) + turn = _turn() + + assert registry.begin_utterance(turn) is True + game.is_available = lambda: False + + assert await registry.prepare_utterance(turn) is False + game.prepare_turn.assert_not_awaited() + game.on_cancelled.assert_awaited_once_with(turn, "consumer_unavailable") + + game.is_available = lambda: True + assert registry.begin_utterance(turn) is True + + +async def test_availability_error_keeps_input_fail_closed() -> None: + registry = VoiceInputRegistry() + game = _consumer() + game.is_available = lambda: (_ for _ in ()).throw( + RuntimeError("availability failed") + ) + registration = registry.register_builtin( + BuiltinVoiceInputConsumer.GAME, + game, + ) + registry.activate(registration.handle) + + assert registry.active_accepts_input is False + assert registry.begin_utterance(_turn()) is False + + +async def test_switch_during_prepare_rejects_old_route_once() -> None: + registry = VoiceInputRegistry() + game = _consumer() + chat = _consumer() + prepare_entered = asyncio.Event() + release_prepare = asyncio.Event() + callback_order: list[str] = [] + prepared_state = False + + async def slow_prepare(_token: VoiceTurnToken) -> bool: + nonlocal prepared_state + prepare_entered.set() + await release_prepare.wait() + prepared_state = True + callback_order.append("prepare") + return True + + async def cancel_after_prepare( + _token: VoiceTurnToken, + _reason: str, + ) -> None: + nonlocal prepared_state + callback_order.append("cancel") + assert prepared_state is True + prepared_state = False + + game.prepare_turn.side_effect = slow_prepare + game.on_cancelled.side_effect = cancel_after_prepare + game_registration = registry.register_builtin( + BuiltinVoiceInputConsumer.GAME, + game, + ) + chat_registration = _register_chat(registry, chat) + registry.activate(game_registration.handle) + turn = _turn() + assert registry.begin_utterance(turn) + + prepare_task = asyncio.create_task(registry.prepare_utterance(turn)) + await prepare_entered.wait() + registry.activate(chat_registration.handle) + await asyncio.sleep(0) + game.on_cancelled.assert_not_awaited() + release_prepare.set() + + assert await prepare_task is False + await registry.wait_idle() + game.on_cancelled.assert_awaited_once_with(turn, "consumer_switched") + assert callback_order == ["prepare", "cancel"] + assert prepared_state is False + chat.prepare_turn.assert_not_awaited() + + +@pytest.mark.parametrize("failure_mode", ("false", "error")) +async def test_prepare_failure_consumes_route_without_fallback( + failure_mode: str, +) -> None: + registry = VoiceInputRegistry() + game = _consumer() + if failure_mode == "false": + game.prepare_turn.return_value = False + else: + game.prepare_turn.side_effect = RuntimeError("prepare failed") + registration = registry.register_builtin( + BuiltinVoiceInputConsumer.GAME, + game, + ) + registry.activate(registration.handle) + turn = _turn() + assert registry.begin_utterance(turn) + + assert await registry.prepare_utterance(turn) is False + await registry.wait_idle() + assert ( + await registry.dispatch_final( + VoiceTranscriptEvent( + turn_token=turn, + provider="qwen", + text="stale", + ) + ) + is VoiceInputDispatchResult.REJECTED + ) + game.on_cancelled.assert_awaited_once_with(turn, "prepare_rejected") + + +async def test_prepare_rejection_finishes_cancellation_before_same_token_retry() -> ( + None +): + registry = VoiceInputRegistry() + chat = _consumer() + chat.prepare_turn.side_effect = [False, True] + cancellation_started = asyncio.Event() + release_cancellation = asyncio.Event() + + async def slow_cancel(_token: VoiceTurnToken, _reason: str) -> None: + cancellation_started.set() + await release_cancellation.wait() + + chat.on_cancelled.side_effect = slow_cancel + registration = _register_chat(registry, chat) + registry.activate(registration.handle) + turn = _turn() + assert registry.begin_utterance(turn) + + rejected = asyncio.create_task(registry.prepare_utterance(turn)) + await cancellation_started.wait() + assert rejected.done() is False + release_cancellation.set() + assert await rejected is False + + assert registry.begin_utterance(turn) is True + assert await registry.prepare_utterance(turn) is True + event = VoiceTranscriptEvent( + turn_token=turn, + provider="qwen", + text="retry", + ) + assert ( + await registry.dispatch_final(event) + is VoiceInputDispatchResult.DELIVERED + ) + chat.on_final.assert_awaited_once_with(event) + + +async def test_empty_final_consumes_route_before_terminal_cancellation() -> None: + registry = VoiceInputRegistry() + chat = _consumer() + registration = _register_chat(registry, chat) + registry.activate(registration.handle) + turn = _turn() + assert registry.begin_utterance(turn) + assert await registry.prepare_utterance(turn) + callback_saw_consumed_route = False + + async def on_cancelled(token: VoiceTurnToken, reason: str) -> None: + nonlocal callback_saw_consumed_route + duplicate = VoiceTranscriptEvent( + turn_token=token, + provider="qwen", + text="duplicate", + ) + callback_saw_consumed_route = ( + await registry.dispatch_final(duplicate) + is VoiceInputDispatchResult.REJECTED + ) + assert reason == "empty_final" + + chat.on_cancelled.side_effect = on_cancelled + empty = VoiceTranscriptEvent( + turn_token=turn, + provider="qwen", + text=" \t ", + ) + + assert ( + await registry.dispatch_final(empty) + is VoiceInputDispatchResult.EMPTY_CONSUMED + ) + await registry.wait_idle() + assert callback_saw_consumed_route is True + chat.on_final.assert_not_awaited() + chat.on_cancelled.assert_awaited_once_with(turn, "empty_final") + assert ( + await registry.dispatch_final(empty) + is VoiceInputDispatchResult.REJECTED + ) + + +async def test_final_callback_error_never_restores_consumed_route() -> None: + registry = VoiceInputRegistry() + chat = _consumer() + chat.on_final.side_effect = RuntimeError("consumer failed") + registration = _register_chat(registry, chat) + registry.activate(registration.handle) + turn = _turn() + assert registry.begin_utterance(turn) + event = VoiceTranscriptEvent( + turn_token=turn, + provider="qwen", + text="hello", + ) + + assert ( + await registry.dispatch_final(event) + is VoiceInputDispatchResult.CALLBACK_FAILED + ) + assert ( + await registry.dispatch_final(event) + is VoiceInputDispatchResult.REJECTED + ) + chat.on_final.assert_awaited_once_with(event) + + +async def test_partial_callback_error_is_rejected_without_losing_route() -> None: + registry = VoiceInputRegistry() + chat = _consumer() + chat.on_partial.side_effect = RuntimeError("consumer failed") + registration = _register_chat(registry, chat) + registry.activate(registration.handle) + turn = _turn() + assert registry.begin_utterance(turn) + partial = VoicePartialEvent(turn_token=turn, text="hel") + + assert ( + await registry.dispatch_partial(partial) + is VoiceInputDispatchResult.CALLBACK_FAILED + ) + + final = VoiceTranscriptEvent( + turn_token=turn, + provider="qwen", + text="hello", + ) + assert ( + await registry.dispatch_final(final) + is VoiceInputDispatchResult.DELIVERED + ) + + +async def test_targeted_and_global_invalidation_cancel_each_route_once() -> None: + registry = VoiceInputRegistry() + chat = _consumer() + registration = _register_chat(registry, chat) + registry.activate(registration.handle) + first = _turn(1) + second = _turn(2) + assert registry.begin_utterance(first) + assert registry.begin_utterance(second) + + assert registry.invalidate_utterance(first, reason="pcm_hole") is True + assert registry.invalidate_utterance(first, reason="duplicate") is False + assert registry.invalidate_utterance(reason="route_swap") is True + assert registry.invalidate_utterance(reason="duplicate") is False + await registry.wait_idle() + + assert chat.on_cancelled.await_args_list == [ + ((first, "pcm_hole"), {}), + ((second, "route_swap"), {}), + ] + + +async def test_close_cancels_routes_and_invalidates_registrations() -> None: + registry = VoiceInputRegistry() + chat = _consumer() + registration = _register_chat(registry, chat) + registry.activate(registration.handle) + turn = _turn() + assert registry.begin_utterance(turn) + + await registry.close() + await registry.close() + + assert registry.active_identity is None + assert registry.active_accepts_input is False + chat.on_cancelled.assert_awaited_once_with(turn, "registry_closed") + with pytest.raises(VoiceInputHandleError): + registry.activate(registration.handle) diff --git a/tests/unit/test_voice_recognition_settings_static.py b/tests/unit/test_voice_recognition_settings_static.py new file mode 100644 index 0000000000..9be8028ac3 --- /dev/null +++ b/tests/unit/test_voice_recognition_settings_static.py @@ -0,0 +1,154 @@ +import json +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] +APP_STATE = ROOT / "static" / "app" / "app-state.js" +APP_SETTINGS = ROOT / "static" / "app" / "app-settings.js" +APP_AUDIO_CAPTURE = ROOT / "static" / "app" / "app-audio-capture.js" +LOCALE_DIR = ROOT / "static" / "locales" +LOCALES = ("en", "es", "ja", "ko", "pt", "ru", "zh-CN", "zh-TW") + + +def test_new_profile_voice_settings_default_enabled_without_becoming_authoritative() -> None: + state = APP_STATE.read_text(encoding="utf-8") + + assert "independentAsrEnabled: true" in state + assert "voiceInputResourceOptimizationEnabled: true" in state + assert "settingsHydrated: false" in state + assert "independentAsrAuthoritative: false" in state + assert "voiceInputResourceOptimizationAuthoritative: false" in state + + +def test_voice_settings_preserve_explicit_false_during_boot_merge() -> None: + settings = APP_SETTINGS.read_text(encoding="utf-8") + + assert "settings.independentAsrEnabled ?? true" in settings + assert "settings.voiceInputResourceOptimizationEnabled ?? true" in settings + assert "settings.independentAsrEnabled || true" not in settings + assert "settings.voiceInputResourceOptimizationEnabled || true" not in settings + + +def test_reset_defaults_match_new_profile_voice_defaults() -> None: + settings = APP_SETTINGS.read_text(encoding="utf-8") + reset_defaults = settings.split( + "function _defaultConversationSettingsForReset()", + maxsplit=1, + )[1].split("function _serverSettingsForMerge", maxsplit=1)[0] + + assert "independentAsrEnabled: true" in reset_defaults + assert "voiceInputResourceOptimizationEnabled: true" in reset_defaults + + +def test_resource_optimization_uses_only_the_canonical_shared_setting_key() -> None: + settings = APP_SETTINGS.read_text(encoding="utf-8") + + assert "'voiceInputResourceOptimizationEnabled'" in settings + assert ( + "voiceInputResourceOptimizationEnabled: " + "S.voiceInputResourceOptimizationEnabled" + ) in settings + assert ( + "voiceInputResourceOptimizationEnabled: currentVoiceResourceOptimization" + ) in settings + assert "voice_input_resource_optimization_enabled" not in settings + + +def test_voice_recognition_popover_has_explicit_portal_lifecycle() -> None: + source = APP_AUDIO_CAPTURE.read_text(encoding="utf-8") + + for function_name in ( + "createVoicePanel", + "openVoicePanel", + "closeVoicePanel", + "destroyVoicePanel", + ): + assert f"function {function_name}(" in source + + assert "document.body.appendChild(voicePanel)" in source + assert "position: 'fixed'" in source + assert "asrContainer.setAttribute('aria-expanded', 'true')" in source + assert "asrContainer.setAttribute('aria-expanded', 'false')" in source + assert "event.key === 'Escape'" in source + assert "document.addEventListener('pointerdown'" in source + assert "document.removeEventListener('pointerdown'" in source + assert "window.addEventListener('resize'" in source + assert "window.removeEventListener('resize'" in source + assert "window.addEventListener('scroll'" in source + assert "window.removeEventListener('scroll'" in source + assert "asrContainer.addEventListener('mouseenter'" in source + assert "asrContainer.addEventListener('focusin'" in source + assert "asrContainer.addEventListener('pointerup'" in source + + +def test_cross_window_voice_settings_publish_a_shared_pending_route_snapshot() -> None: + state = APP_STATE.read_text(encoding="utf-8") + settings = APP_SETTINGS.read_text(encoding="utf-8") + audio_capture = APP_AUDIO_CAPTURE.read_text(encoding="utf-8") + + assert "voiceSettingsPendingUntilEpoch: null" in state + assert "pendingVoiceRouteIndependentAsr: null" in state + assert "S.voiceSettingsPendingUntilEpoch" in settings + assert "S.pendingVoiceRouteIndependentAsr" in settings + assert "neko:voice-settings-pending-changed" in settings + assert "S.voiceSettingsPendingUntilEpoch" in audio_capture + assert "S.pendingVoiceRouteIndependentAsr" in audio_capture + assert "neko:voice-settings-pending-changed" in audio_capture + + +def test_voice_recognition_copy_keeps_native_and_fail_closed_routes_distinct() -> None: + source = APP_AUDIO_CAPTURE.read_text(encoding="utf-8") + + assert "window.t('microphone.voiceRecognitionDisabled')" in source + assert "window.t('microphone.voiceRecognitionDisabledHint')" in source + assert "window.t('microphone.voiceRecognitionUnavailable')" in source + assert "语音输入已关闭" not in source + assert "自动回退到 Omni" not in source + assert "自动选择其他识别服务" not in source + + +def test_voice_recognition_popover_keys_match_across_all_locales() -> None: + required = { + "noiseReduction", + "noiseReductionHint", + "independentAsr", + "independentAsrSummary", + "independentAsrSummaryGeneric", + "independentAsrNative", + "voiceRecognitionSettings", + "voiceRecognitionDisabled", + "voiceRecognitionDisabledHint", + "voiceRecognitionUnavailable", + "voiceRecognitionStatusReady", + "voiceRecognitionSettingsPending", + "voiceResourceOptimization", + "voiceResourceOptimizationHintOn", + "voiceResourceOptimizationHintOff", + } + + key_sets: list[set[str]] = [] + for locale_name in LOCALES: + locale = json.loads( + (LOCALE_DIR / f"{locale_name}.json").read_text(encoding="utf-8") + ) + microphone = locale["microphone"] + assert required <= set(microphone), locale_name + assert "RNNoise" not in microphone["noiseReductionHint"] + assert "Silero" not in microphone["noiseReductionHint"] + key_sets.append(set(microphone)) + + assert all(keys == key_sets[0] for keys in key_sets[1:]) + + +def test_async_asr_status_copy_uses_the_caller_provider_key() -> None: + for locale_name in LOCALES: + locale = json.loads( + (LOCALE_DIR / f"{locale_name}.json").read_text(encoding="utf-8") + ) + microphone = locale["microphone"] + for key in ( + "independentAsrActive", + "independentAsrProviderUnavailable", + ): + assert "{{providerKey}}" in microphone[key], (locale_name, key) + assert "{{provider}}" not in microphone[key], (locale_name, key) diff --git a/tests/unit/test_websocket_binary_audio.py b/tests/unit/test_websocket_binary_audio.py index 27946284ec..02c81a30cb 100644 --- a/tests/unit/test_websocket_binary_audio.py +++ b/tests/unit/test_websocket_binary_audio.py @@ -76,6 +76,9 @@ def reset_session_start_circuit(self) -> None: def set_independent_asr_handshake(self, value) -> None: self.calls.append(("asr_handshake", value)) + def set_voice_input_resource_optimization_handshake(self, value) -> None: + self.calls.append(("resource_optimization_handshake", value)) + def start_session(self, *_args, **_kwargs): self.calls.append(("start_session", None)) @@ -127,6 +130,35 @@ async def close(self) -> None: self.closed = True +class _DeferredHandshakeManager(_ProtocolManager): + """Model the real async start task reading manager fallback state late.""" + + def __init__(self) -> None: + super().__init__() + self.asr_override = None + self.optimization_override = None + self.started_overrides: list[tuple[object, object]] = [] + + def set_independent_asr_handshake(self, value) -> None: + super().set_independent_asr_handshake(value) + self.asr_override = value if isinstance(value, bool) else None + + def set_voice_input_resource_optimization_handshake(self, value) -> None: + super().set_voice_input_resource_optimization_handshake(value) + self.optimization_override = value if isinstance(value, bool) else None + + async def start_session(self, *_args, **kwargs) -> None: + self.started_overrides.append( + ( + kwargs.get("handshake_override", self.asr_override), + kwargs.get( + "resource_optimization_override", + self.optimization_override, + ), + ) + ) + + def _install_protocol_endpoint( monkeypatch, *, @@ -344,6 +376,7 @@ async def test_start_session_forwards_independent_asr_handshake_before_dispatch( "action": "start_session", "input_type": "audio", "independent_asr_enabled": True, + "voice_input_resource_optimization_enabled": False, }, {"action": "start_session", "input_type": "audio"}, ] @@ -362,8 +395,36 @@ async def test_start_session_forwards_independent_asr_handshake_before_dispatch( assert [ payload for name, payload in manager.calls if name == "asr_handshake" ] == [True, None] + assert [ + payload + for name, payload in manager.calls + if name == "resource_optimization_handshake" + ] == [False, None] call_names = [name for name, _payload in manager.calls] - assert call_names.index("asr_handshake") < call_names.index("start_session") + start_indices = [ + index for index, name in enumerate(call_names) if name == "start_session" + ] + asr_indices = [ + index for index, name in enumerate(call_names) if name == "asr_handshake" + ] + optimization_indices = [ + index + for index, name in enumerate(call_names) + if name == "resource_optimization_handshake" + ] + assert len(start_indices) == len(asr_indices) == len(optimization_indices) == 2 + assert all( + asr_handshake < start + for asr_handshake, start in zip(asr_indices, start_indices, strict=True) + ) + assert all( + optimization_handshake < start + for optimization_handshake, start in zip( + optimization_indices, + start_indices, + strict=True, + ) + ) @pytest.mark.asyncio @@ -1438,3 +1499,41 @@ async def _replace_connection_before_dispatch() -> dict: assert "authorize" not in [name for name, _payload in manager.calls] assert "start_session" not in [name for name, _payload in manager.calls] assert websocket.closed is True + + +@pytest.mark.asyncio +async def test_each_start_task_keeps_its_own_voice_handshake_overrides( + monkeypatch, +) -> None: + manager = _DeferredHandshakeManager() + websocket = _EventWebSocket( + [ + { + "action": "start_session", + "input_type": "text", + "independent_asr_enabled": False, + "voice_input_resource_optimization_enabled": True, + }, + { + "action": "start_session", + "input_type": "text", + "independent_asr_enabled": True, + "voice_input_resource_optimization_enabled": False, + }, + ] + ) + _install_protocol_endpoint( + monkeypatch, + manager=manager, + websocket=websocket, + ) + deferred: list[object] = [] + monkeypatch.setattr(websocket_router, "_fire_task", deferred.append) + + await websocket_router.websocket_endpoint(websocket, "Lan") + await asyncio.gather(*deferred) + + assert manager.started_overrides == [ + (False, True), + (True, False), + ] diff --git a/tests/unit/voice_turn/test_contracts.py b/tests/unit/voice_turn/test_contracts.py index bf8c9ce494..a255b77b45 100644 --- a/tests/unit/voice_turn/test_contracts.py +++ b/tests/unit/voice_turn/test_contracts.py @@ -12,12 +12,27 @@ SmartTurnConfig, TurnDecision, TurnEvaluation, + VoiceIngressToken, VoicePartialEvent, + VoiceTurnToken, build_turn_detector_if_required, requires_external_turn_detector, ) +def _turn_token(*, session_epoch: int = 1, turn_id: int = 2) -> VoiceTurnToken: + return VoiceTurnToken( + ingress=VoiceIngressToken( + session_epoch=session_epoch, + connection_id="connection", + lease_generation=3, + route_generation=4, + audio_generation=5, + ), + turn_id=turn_id, + ) + + def test_semantic_endpoint_provider_does_not_require_smart_turn(): assert requires_external_turn_detector(AsrTurnCapabilities(semantic_endpoint=True)) is False assert requires_external_turn_detector(AsrTurnCapabilities(semantic_endpoint=False)) is True @@ -62,7 +77,7 @@ def test_config_rejects_missing_vad_hysteresis(): @pytest.mark.parametrize( "event", [ - VoicePartialEvent(text="hello", session_epoch=1), + VoicePartialEvent(turn_token=_turn_token(), text="hello"), AsrStatusEvent(code="ASR_READY", provider="qwen"), AsrLifecycleNotification( state="local_listen", @@ -75,3 +90,12 @@ def test_config_rejects_missing_vad_hysteresis(): def test_cross_layer_asr_events_are_immutable(event): with pytest.raises(FrozenInstanceError): event.__setattr__(next(iter(event.__dataclass_fields__)), object()) + + +def test_partial_event_exposes_read_only_epoch_from_full_turn_identity() -> None: + token = _turn_token(session_epoch=7, turn_id=11) + + event = VoicePartialEvent(turn_token=token, text="draft") + + assert event.turn_token is token + assert event.session_epoch == 7 diff --git a/utils/game_route_state.py b/utils/game_route_state.py index b0a3f522b9..1e479997c7 100644 --- a/utils/game_route_state.py +++ b/utils/game_route_state.py @@ -147,6 +147,21 @@ def is_game_route_active(lanlan_name: str, game_type: str | None = None) -> bool return _get_active_game_route_state(lanlan_name, game_type) is not None +def get_active_game_route_identity( + lanlan_name: str, +) -> tuple[str, str] | None: + """Return the concrete active ``(game_type, session_id)`` identity.""" + + target_lanlan = str(lanlan_name or "") + for (key_lanlan, key_game), state in _game_route_states.items(): + if key_lanlan != target_lanlan or not state.get("game_route_active"): + continue + session_id = str(state.get("session_id") or "").strip() + if session_id: + return key_game, session_id + return None + + _VoiceTranscriptHandler = Callable[..., Awaitable[bool]] _voice_transcript_handler: Optional[_VoiceTranscriptHandler] = None