Skip to content

Commit 263683f

Browse files
author
Harsh Patadia
committed
fix(llm): make client timeout env-overridable (LLM_TIMEOUT)
Hardcoded 30s aborted the deep-tier archetype call (max_tokens=8192) on slow local backends like Ollama 7B. Default stays 30s for hosted providers.
1 parent 56ae3a0 commit 263683f

1 file changed

Lines changed: 5 additions & 3 deletions

File tree

backend/app/utils/llm.py

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -218,7 +218,7 @@ def __init__(
218218
model: str | None = None,
219219
tracker: UsageTracker | None = None,
220220
max_retries: int = 3,
221-
timeout: float = 30.0,
221+
timeout: float | None = None,
222222
):
223223
self.tier = tier
224224
if api_key and base_url and model:
@@ -227,9 +227,11 @@ def __init__(
227227
self.api_key, self.base_url, self.model = _resolve_model(tier)
228228
self.tracker = tracker
229229
self.max_retries = max_retries
230-
self.timeout = timeout
230+
# Env-overridable so slow local backends (Ollama 7B) don't abort the
231+
# longer deep-tier generations. Default 30s for hosted providers.
232+
self.timeout = timeout if timeout is not None else float(os.environ.get("LLM_TIMEOUT", "30"))
231233

232-
self._sync = OpenAI(api_key=self.api_key, base_url=self.base_url, timeout=timeout)
234+
self._sync = OpenAI(api_key=self.api_key, base_url=self.base_url, timeout=self.timeout)
233235
self._async: AsyncOpenAI | None = None # lazy
234236

235237
# Fallback provider (e.g. Gemini). Used after primary exhausts retries.

0 commit comments

Comments
 (0)