From 23e44472671f9c5104f3e0fdf05ec546f221a523 Mon Sep 17 00:00:00 2001 From: Fabian Date: Fri, 6 Feb 2026 22:26:01 +0100 Subject: [PATCH 1/3] feat: add OpenRouter as alternative VLM and image generation provider Add support for OpenRouter's OpenAI-compatible API as an alternative to the direct Google Gemini SDK. This allows using any model available on OpenRouter (Gemini, Claude, GPT, etc.) for both text planning and image generation via a single OPENROUTER_API_KEY. New providers: - openrouter (VLM): text generation via /chat/completions - openrouter_imagen (ImageGen): image generation via modalities param Usage: --vlm-provider openrouter --vlm-model google/gemini-2.0-flash-001 --image-provider openrouter_imagen --image-model google/gemini-3-pro-image-preview --- paperbanana/core/config.py | 1 + .../providers/image_gen/openrouter_imagen.py | 137 ++++++++++++++++++ paperbanana/providers/registry.py | 20 ++- paperbanana/providers/vlm/openrouter.py | 112 ++++++++++++++ 4 files changed, 268 insertions(+), 2 deletions(-) create mode 100644 paperbanana/providers/image_gen/openrouter_imagen.py create mode 100644 paperbanana/providers/vlm/openrouter.py diff --git a/paperbanana/core/config.py b/paperbanana/core/config.py index b23c843b..c6d902bc 100644 --- a/paperbanana/core/config.py +++ b/paperbanana/core/config.py @@ -73,6 +73,7 @@ class Settings(BaseSettings): # API Keys (loaded from environment) google_api_key: Optional[str] = Field(default=None, alias="GOOGLE_API_KEY") + openrouter_api_key: Optional[str] = Field(default=None, alias="OPENROUTER_API_KEY") # SSL skip_ssl_verification: bool = Field(default=False, alias="SKIP_SSL_VERIFICATION") diff --git a/paperbanana/providers/image_gen/openrouter_imagen.py b/paperbanana/providers/image_gen/openrouter_imagen.py new file mode 100644 index 00000000..ed5d5ed8 --- /dev/null +++ b/paperbanana/providers/image_gen/openrouter_imagen.py @@ -0,0 +1,137 @@ +"""OpenRouter image generation provider — uses any image model via the OpenAI-compatible API.""" + +from __future__ import annotations + +import base64 +import re +from io import BytesIO +from typing import Optional + +import structlog +from PIL import Image +from tenacity import retry, stop_after_attempt, wait_exponential + +from paperbanana.providers.base import ImageGenProvider + +logger = structlog.get_logger() + + +class OpenRouterImageGen(ImageGenProvider): + """Image generation routed through OpenRouter. + + Talks to models that support ``modalities: ["image", "text"]`` + (e.g. google/gemini-3-pro-image-preview) and returns a PIL Image + decoded from the base64 response. + + Get an API key at https://openrouter.ai/keys + """ + + def __init__( + self, + api_key: Optional[str] = None, + model: str = "google/gemini-3-pro-image-preview", + ): + self._api_key = api_key + self._model = model + self._client = None + + @property + def name(self) -> str: + return "openrouter_imagen" + + @property + def model_name(self) -> str: + return self._model + + def _get_client(self): + """Lazy-init an httpx client pointed at the OpenRouter API.""" + if self._client is None: + import httpx + + self._client = httpx.Client( + base_url="https://openrouter.ai/api/v1", + headers={ + "Authorization": f"Bearer {self._api_key}", + "HTTP-Referer": "https://github.com/llmsresearch/paperbanana", + "X-Title": "PaperBanana", + }, + # Image generation can take a while + timeout=180.0, + ) + return self._client + + def is_available(self) -> bool: + return self._api_key is not None + + def _aspect_ratio_hint(self, width: int, height: int) -> str: + """Turn pixel dimensions into a human-readable aspect ratio hint for the prompt.""" + ratio = width / height + if ratio > 1.5: + return "wide landscape format (16:9)" + if ratio > 1.2: + return "landscape format (3:2)" + if ratio < 0.67: + return "tall portrait format (9:16)" + if ratio < 0.83: + return "portrait format (2:3)" + return "square format (1:1)" + + @retry(stop=stop_after_attempt(3), wait=wait_exponential(min=2, max=30)) + async def generate( + self, + prompt: str, + negative_prompt: Optional[str] = None, + width: int = 1024, + height: int = 1024, + seed: Optional[int] = None, + ) -> Image.Image: + client = self._get_client() + + # OpenRouter doesn't have native aspect-ratio params like the Google SDK, + # so we bake the desired format into the prompt itself. + aspect_hint = self._aspect_ratio_hint(width, height) + full_prompt = f"{prompt}\n\nGenerate this as a {aspect_hint} image." + if negative_prompt: + full_prompt += f"\n\nAvoid: {negative_prompt}" + + payload = { + "model": self._model, + "messages": [ + {"role": "user", "content": full_prompt}, + ], + # This tells OpenRouter we want an image back, not just text + "modalities": ["image", "text"], + } + + if seed is not None: + payload["seed"] = seed + + response = client.post("/chat/completions", json=payload) + response.raise_for_status() + data = response.json() + + message = data["choices"][0]["message"] + + # Primary path: images come as base64 data-URLs in the "images" array + images = message.get("images", []) + if images: + for img_entry in images: + url = img_entry.get("image_url", {}).get("url", "") + if url.startswith("data:image/"): + b64_data = url.split(",", 1)[1] + image_bytes = base64.b64decode(b64_data) + return Image.open(BytesIO(image_bytes)) + + # Fallback: some models inline the base64 data directly in the text content + content = message.get("content", "") + if "data:image/" in content: + match = re.search(r"data:image/[^;]+;base64,([A-Za-z0-9+/=]+)", content) + if match: + image_bytes = base64.b64decode(match.group(1)) + return Image.open(BytesIO(image_bytes)) + + logger.error("No image data in OpenRouter response", model=self._model) + raise ValueError( + f"OpenRouter response for {self._model} did not contain image data. " + f"Content preview: {content[:200]}" + ) diff --git a/paperbanana/providers/registry.py b/paperbanana/providers/registry.py index de57766c..f8f3247b 100644 --- a/paperbanana/providers/registry.py +++ b/paperbanana/providers/registry.py @@ -26,8 +26,15 @@ def create_vlm(settings: Settings) -> VLMProvider: api_key=settings.google_api_key, model=settings.vlm_model, ) + elif provider == "openrouter": + from paperbanana.providers.vlm.openrouter import OpenRouterVLM + + return OpenRouterVLM( + api_key=settings.openrouter_api_key, + model=settings.vlm_model, + ) else: - raise ValueError(f"Unknown VLM provider: {provider}. Available: gemini") + raise ValueError(f"Unknown VLM provider: {provider}. Available: gemini, openrouter") @staticmethod def create_image_gen(settings: Settings) -> ImageGenProvider: @@ -42,5 +49,14 @@ def create_image_gen(settings: Settings) -> ImageGenProvider: api_key=settings.google_api_key, model=settings.image_model, ) + elif provider == "openrouter_imagen": + from paperbanana.providers.image_gen.openrouter_imagen import OpenRouterImageGen + + return OpenRouterImageGen( + api_key=settings.openrouter_api_key, + model=settings.image_model, + ) else: - raise ValueError(f"Unknown image provider: {provider}. Available: google_imagen") + raise ValueError( + f"Unknown image provider: {provider}. Available: google_imagen, openrouter_imagen" + ) diff --git a/paperbanana/providers/vlm/openrouter.py b/paperbanana/providers/vlm/openrouter.py new file mode 100644 index 00000000..80b92663 --- /dev/null +++ b/paperbanana/providers/vlm/openrouter.py @@ -0,0 +1,112 @@ +"""OpenRouter VLM provider — OpenAI-compatible API for any model.""" + +from __future__ import annotations + +import re +from typing import Optional + +import structlog +from PIL import Image +from tenacity import retry, stop_after_attempt, wait_exponential + +from paperbanana.core.utils import image_to_base64 +from paperbanana.providers.base import VLMProvider + +logger = structlog.get_logger() + + +class OpenRouterVLM(VLMProvider): + """VLM provider that routes through OpenRouter's OpenAI-compatible API. + + Works with any model on OpenRouter (Gemini, Claude, GPT, Llama, etc.). + Get an API key at https://openrouter.ai/keys + """ + + def __init__( + self, + api_key: Optional[str] = None, + model: str = "google/gemini-2.0-flash-001", + ): + self._api_key = api_key + self._model = model + self._client = None + + @property + def name(self) -> str: + return "openrouter" + + @property + def model_name(self) -> str: + return self._model + + def _get_client(self): + """Lazy-init an httpx client pointed at the OpenRouter API.""" + if self._client is None: + import httpx + + self._client = httpx.Client( + base_url="https://openrouter.ai/api/v1", + headers={ + "Authorization": f"Bearer {self._api_key}", + "HTTP-Referer": "https://github.com/llmsresearch/paperbanana", + "X-Title": "PaperBanana", + }, + timeout=120.0, + ) + return self._client + + def is_available(self) -> bool: + return self._api_key is not None + + @retry(stop=stop_after_attempt(3), wait=wait_exponential(min=2, max=30)) + async def generate( + self, + prompt: str, + images: Optional[list[Image.Image]] = None, + system_prompt: Optional[str] = None, + temperature: float = 1.0, + max_tokens: int = 4096, + response_format: Optional[str] = None, + ) -> str: + client = self._get_client() + + messages = [] + if system_prompt: + messages.append({"role": "system", "content": system_prompt}) + + # Build multimodal content array (vision images + text) + content = [] + if images: + for img in images: + b64 = image_to_base64(img) + content.append( + { + "type": "image_url", + "image_url": {"url": f"data:image/png;base64,{b64}"}, + } + ) + content.append({"type": "text", "text": prompt}) + messages.append({"role": "user", "content": content}) + + payload = { + "model": self._model, + "messages": messages, + "temperature": temperature, + "max_tokens": max_tokens, + } + + if response_format == "json": + payload["response_format"] = {"type": "json_object"} + + response = client.post("/chat/completions", json=payload) + response.raise_for_status() + + data = response.json() + text = data["choices"][0]["message"]["content"] + + logger.debug( + "OpenRouter response", + model=self._model, + usage=data.get("usage"), + ) + return text From 9282be4c1f2fbf761a9bfe539bf99cb244c5fc3f Mon Sep 17 00:00:00 2001 From: Fabian Date: Fri, 6 Feb 2026 22:28:04 +0100 Subject: [PATCH 2/3] chore: change default VLM model to gemini-3-flash for better performance --- paperbanana/providers/vlm/openrouter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paperbanana/providers/vlm/openrouter.py b/paperbanana/providers/vlm/openrouter.py index 80b92663..bfbb2224 100644 --- a/paperbanana/providers/vlm/openrouter.py +++ b/paperbanana/providers/vlm/openrouter.py @@ -25,7 +25,7 @@ class OpenRouterVLM(VLMProvider): def __init__( self, api_key: Optional[str] = None, - model: str = "google/gemini-2.0-flash-001", + model: str = "google/gemini-3-flash-preview", ): self._api_key = api_key self._model = model From 18f74d9741c0a728a1f4400acb4738194d265c05 Mon Sep 17 00:00:00 2001 From: Fabian Date: Sat, 7 Feb 2026 12:34:56 +0100 Subject: [PATCH 3/3] fix: remove unused re import and switch to httpx.AsyncClient Address PR review feedback: - Remove unused `import re` from VLM provider (F401) - Switch both OpenRouter providers from httpx.Client to httpx.AsyncClient to avoid blocking the event loop --- paperbanana/providers/image_gen/openrouter_imagen.py | 6 +++--- paperbanana/providers/vlm/openrouter.py | 7 +++---- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/paperbanana/providers/image_gen/openrouter_imagen.py b/paperbanana/providers/image_gen/openrouter_imagen.py index ed5d5ed8..c9bc157b 100644 --- a/paperbanana/providers/image_gen/openrouter_imagen.py +++ b/paperbanana/providers/image_gen/openrouter_imagen.py @@ -44,11 +44,11 @@ def model_name(self) -> str: return self._model def _get_client(self): - """Lazy-init an httpx client pointed at the OpenRouter API.""" + """Lazy-init an async httpx client pointed at the OpenRouter API.""" if self._client is None: import httpx - self._client = httpx.Client( + self._client = httpx.AsyncClient( base_url="https://openrouter.ai/api/v1", headers={ "Authorization": f"Bearer {self._api_key}", @@ -106,7 +106,7 @@ async def generate( if seed is not None: payload["seed"] = seed - response = client.post("/chat/completions", json=payload) + response = await client.post("/chat/completions", json=payload) response.raise_for_status() data = response.json() diff --git a/paperbanana/providers/vlm/openrouter.py b/paperbanana/providers/vlm/openrouter.py index bfbb2224..4605857c 100644 --- a/paperbanana/providers/vlm/openrouter.py +++ b/paperbanana/providers/vlm/openrouter.py @@ -2,7 +2,6 @@ from __future__ import annotations -import re from typing import Optional import structlog @@ -40,11 +39,11 @@ def model_name(self) -> str: return self._model def _get_client(self): - """Lazy-init an httpx client pointed at the OpenRouter API.""" + """Lazy-init an async httpx client pointed at the OpenRouter API.""" if self._client is None: import httpx - self._client = httpx.Client( + self._client = httpx.AsyncClient( base_url="https://openrouter.ai/api/v1", headers={ "Authorization": f"Bearer {self._api_key}", @@ -98,7 +97,7 @@ async def generate( if response_format == "json": payload["response_format"] = {"type": "json_object"} - response = client.post("/chat/completions", json=payload) + response = await client.post("/chat/completions", json=payload) response.raise_for_status() data = response.json()