[codex] fix(providers): normalize qwen system messages (#725)

Author: penso

.github/workflows/provider-integration.yml

@@ -324,6 +324,83 @@ jobs:
           if-no-files-found: ignore
           retention-days: 14
 
+  ollama-qwen-live-e2e:
+    name: Ollama Qwen Live E2E
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    continue-on-error: true
+    env:
+      MOLTIS_E2E_OLLAMA_QWEN_LIVE: "1"
+      MOLTIS_E2E_OLLAMA_QWEN_MODEL: qwen2.5:0.5b
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
+        with:
+          persist-credentials: false
+
+      - uses: dtolnay/rust-toolchain@f7ccc83f9ed1e5b9c81d8a67d7ad1a747e22a561 # master
+        with:
+          toolchain: stable
+
+      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
+        with:
+          node-version: "22"
+          package-manager-cache: false
+
+      - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2
+        with:
+          shared-key: ollama-qwen-live-e2e-v1
+          cache-all-crates: true
+
+      - name: Install build dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y cmake build-essential clang libclang-dev pkg-config curl zstd
+
+      - name: Install Ollama
+        env:
+          OLLAMA_VERSION: v0.20.7
+          OLLAMA_TARBALL: ollama-linux-amd64.tar.zst
+          OLLAMA_TARBALL_SHA256: 193c41ffee30411a76af4484dae9cdd4c2d6f8f877b7096925c36f2489af131f
+        run: |
+          curl -fsSL -o ollama.tar.zst "https://github.com/ollama/ollama/releases/download/${OLLAMA_VERSION}/${OLLAMA_TARBALL}"
+          echo "${OLLAMA_TARBALL_SHA256}  ollama.tar.zst" | sha256sum -c -
+          sudo tar --use-compress-program=unzstd -C /usr -xf ollama.tar.zst
+          ollama --version
+
+      - name: Install npm dependencies
+        working-directory: crates/web/ui
+        run: npm ci
+
+      - name: Build Tailwind CSS
+        run: |
+          ./scripts/download-tailwindcss-cli.sh tailwindcss-linux-x64
+          cd crates/web/ui && TAILWINDCSS=../../../tailwindcss-linux-x64 ./build.sh
+
+      - name: Build moltis binary
+        run: cargo build --bin moltis
+
+      - name: Install Playwright browsers
+        working-directory: crates/web/ui
+        run: npx playwright install --with-deps chromium
+
+      - name: Run Ollama Qwen live E2E
+        working-directory: crates/web/ui
+        env:
+          CI: "true"
+        run: npx playwright test --project=ollama-qwen-live e2e/specs/ollama-qwen-live.spec.js
+
+      - name: Upload Ollama Qwen live E2E results
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: ollama-qwen-live-e2e-${{ github.run_id }}-${{ github.run_attempt }}
+          path: |
+            crates/web/ui/playwright-report/
+            crates/web/ui/test-results/
+          if-no-files-found: ignore
+          retention-days: 14
+
   summary:
     name: Integration Summary
     runs-on: ubuntu-latest
@@ -333,6 +410,7 @@ jobs:
       - provider-tests
       - provider-e2e-scenarios
       - openai-live-e2e
+      - ollama-qwen-live-e2e
     if: always()
     steps:
       - name: Report

crates/providers/src/openai/provider/request.rs

@@ -2,10 +2,17 @@ use std::collections::{HashMap, HashSet};
 
 use tracing::warn;
 
-use moltis_agents::model::ChatMessage;
+use {crate::raw_model_id, moltis_agents::model::ChatMessage};
 
 use super::OpenAiProvider;
 
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum SystemMessageRewriteStrategy {
+    None,
+    MergeLeadingSystem,
+    InlineIntoFirstUser,
+}
+
 impl OpenAiProvider {
     /// Returns `true` when this provider targets an Anthropic model via
     /// OpenRouter, which supports prompt caching when `cache_control`
@@ -117,18 +124,60 @@ impl OpenAiProvider {
             || self.base_url.to_ascii_lowercase().contains("minimax")
     }
 
-    /// For providers that reject `role: "system"` in the messages array,
-    /// extract all system messages from `body["messages"]`, join their
-    /// content, and prepend it to the first user message.
+    fn is_custom_openai_compatible_provider(&self) -> bool {
+        self.provider_name.starts_with("custom-")
+    }
+
+    fn is_alibaba_qwen_backend(&self) -> bool {
+        self.provider_name.eq_ignore_ascii_case("alibaba-coding")
+            || self.provider_name.eq_ignore_ascii_case("alibaba")
+            || self.provider_name.eq_ignore_ascii_case("dashscope-coding")
+            || self.base_url.contains("dashscope.aliyuncs.com")
+            || self.base_url.contains("alibabacloud.com")
+    }
+
+    fn is_qwen_single_system_backend(&self) -> bool {
+        self.provider_name.eq_ignore_ascii_case("ollama")
+            || self.provider_name.to_ascii_lowercase().contains("ollama")
+            || self.is_custom_openai_compatible_provider()
+            || self.is_alibaba_qwen_backend()
+    }
+
+    /// Some backends ship chat templates that only accept a single system
+    /// message at the front of the conversation. Qwen-based OpenAI-compatible
+    /// backends commonly behave this way (e.g. llama.cpp chat templates).
+    fn requires_single_leading_system_message(&self) -> bool {
+        raw_model_id(&self.model)
+            .to_ascii_lowercase()
+            .contains("qwen")
+            && self.is_qwen_single_system_backend()
+    }
+
+    fn system_message_rewrite_strategy(&self) -> SystemMessageRewriteStrategy {
+        if self.rejects_system_role() {
+            return SystemMessageRewriteStrategy::InlineIntoFirstUser;
+        }
+        if self.requires_single_leading_system_message() {
+            return SystemMessageRewriteStrategy::MergeLeadingSystem;
+        }
+        SystemMessageRewriteStrategy::None
+    }
+
+    /// Rewrite system messages for providers with stricter chat template rules.
     ///
     /// MiniMax's `/v1/chat/completions` endpoint returns error 2013 for
     /// `role: "system"` entries and silently ignores a top-level `"system"`
     /// field. The only reliable way to deliver the system prompt is to
     /// inline it into the first user message.
     ///
+    /// Qwen-based OpenAI-compatible backends often only accept a single system
+    /// message at the very front. For those, join all system messages with
+    /// blank lines and emit exactly one leading `role: "system"` message.
+    ///
     /// Must be called on the request body **after** it is fully assembled.
     pub(super) fn apply_system_prompt_rewrite(&self, body: &mut serde_json::Value) {
-        if !self.rejects_system_role() {
+        let rewrite_strategy = self.system_message_rewrite_strategy();
+        if matches!(rewrite_strategy, SystemMessageRewriteStrategy::None) {
             return;
         }
         let Some(messages) = body
@@ -145,7 +194,10 @@ impl OpenAiProvider {
                 {
                     system_parts.push(content.to_string());
                 } else if msg.get("content").is_some() {
-                    warn!("MiniMax system message has non-string content; it will be dropped");
+                    warn!(
+                        ?rewrite_strategy,
+                        "system message has non-string content; it will be dropped"
+                    );
                 }
                 return false;
             }
@@ -156,6 +208,20 @@ impl OpenAiProvider {
         }
         let system_text = system_parts.join("\n\n");
 
+        if matches!(
+            rewrite_strategy,
+            SystemMessageRewriteStrategy::MergeLeadingSystem
+        ) {
+            messages.insert(
+                0,
+                serde_json::json!({
+                    "role": "system",
+                    "content": system_text,
+                }),
+            );
+            return;
+        }
+
         // Find the first user message and prepend system content to it.
         let system_block =
             format!("[System Instructions]\n{system_text}\n[End System Instructions]\n\n");
@@ -347,3 +413,145 @@ fn assign_openai_tool_call_id(
     remapped_tool_call_ids.insert(raw.to_string(), candidate.clone());
     candidate
 }
+
+#[cfg(test)]
+mod tests {
+    use secrecy::Secret;
+
+    use super::*;
+
+    fn provider(model: &str, provider_name: &str, base_url: &str) -> OpenAiProvider {
+        OpenAiProvider::new_with_name(
+            Secret::new("test-key".to_string()),
+            model.to_string(),
+            base_url.to_string(),
+            provider_name.to_string(),
+        )
+    }
+
+    fn body_messages(body: &serde_json::Value) -> &[serde_json::Value] {
+        let Some(messages) = body.get("messages").and_then(serde_json::Value::as_array) else {
+            panic!("messages should be an array");
+        };
+        messages
+    }
+
+    #[test]
+    fn system_message_rewrite_qwen_merges_multiple_messages_into_one_leading_message() {
+        let provider = provider(
+            "qwen3:0.6b",
+            "custom-ollama-qwen",
+            "http://127.0.0.1:11435/v1",
+        );
+        let mut body = serde_json::json!({
+            "messages": [
+                {"role": "system", "content": "You are a helpful assistant."},
+                {"role": "user", "content": "hello"},
+                {"role": "assistant", "content": "hi"},
+                {"role": "system", "content": "The current user datetime is 2026-04-15 18:22:00 UTC."},
+                {"role": "user", "content": "what time is it?"}
+            ]
+        });
+
+        provider.apply_system_prompt_rewrite(&mut body);
+
+        let messages = body_messages(&body);
+        assert_eq!(messages.len(), 4);
+        assert_eq!(messages[0]["role"], "system");
+        assert_eq!(
+            messages[0]["content"],
+            "You are a helpful assistant.\n\nThe current user datetime is 2026-04-15 18:22:00 UTC."
+        );
+        assert_eq!(messages[1]["role"], "user");
+        assert_eq!(messages[2]["role"], "assistant");
+        assert_eq!(messages[3]["role"], "user");
+    }
+
+    #[test]
+    fn system_message_rewrite_minimax_inlines_messages_into_first_user_message() {
+        let provider = provider("MiniMax-M2.7", "minimax", "https://api.minimax.io/v1");
+        let mut body = serde_json::json!({
+            "messages": [
+                {"role": "system", "content": "You are a helpful assistant."},
+                {"role": "user", "content": "hello"},
+                {"role": "system", "content": "The current user datetime is 2026-04-15 18:22:00 UTC."}
+            ]
+        });
+
+        provider.apply_system_prompt_rewrite(&mut body);
+
+        let messages = body_messages(&body);
+        assert_eq!(messages.len(), 1);
+        assert_eq!(messages[0]["role"], "user");
+        assert_eq!(
+            messages[0]["content"],
+            "[System Instructions]\nYou are a helpful assistant.\n\nThe current user datetime is 2026-04-15 18:22:00 UTC.\n[End System Instructions]\n\nhello"
+        );
+    }
+
+    #[test]
+    fn system_message_rewrite_default_openai_request_is_unchanged() {
+        let provider = provider("gpt-4o-mini", "openai", "https://api.openai.com/v1");
+        let mut body = serde_json::json!({
+            "messages": [
+                {"role": "system", "content": "sys1"},
+                {"role": "user", "content": "hello"},
+                {"role": "system", "content": "sys2"}
+            ]
+        });
+
+        provider.apply_system_prompt_rewrite(&mut body);
+
+        let messages = body_messages(&body);
+        assert_eq!(messages.len(), 3);
+        assert_eq!(messages[0]["role"], "system");
+        assert_eq!(messages[1]["role"], "user");
+        assert_eq!(messages[2]["role"], "system");
+    }
+
+    #[test]
+    fn system_message_rewrite_qwen_model_on_openai_provider_is_unchanged() {
+        let provider = provider("qwen3-coder-plus", "openai", "https://api.openai.com/v1");
+        let mut body = serde_json::json!({
+            "messages": [
+                {"role": "system", "content": "sys1"},
+                {"role": "user", "content": "hello"},
+                {"role": "system", "content": "sys2"}
+            ]
+        });
+
+        provider.apply_system_prompt_rewrite(&mut body);
+
+        let messages = body_messages(&body);
+        assert_eq!(messages.len(), 3);
+        assert_eq!(messages[0]["role"], "system");
+        assert_eq!(messages[0]["content"], "sys1");
+        assert_eq!(messages[1]["role"], "user");
+        assert_eq!(messages[2]["role"], "system");
+        assert_eq!(messages[2]["content"], "sys2");
+    }
+
+    #[test]
+    fn system_message_rewrite_alibaba_qwen_merges_multiple_messages_into_one_leading_message() {
+        let provider = provider(
+            "qwen3.5-plus",
+            "alibaba-coding",
+            "https://coding-intl.dashscope.aliyuncs.com/v1",
+        );
+        let mut body = serde_json::json!({
+            "messages": [
+                {"role": "system", "content": "sys1"},
+                {"role": "user", "content": "hello"},
+                {"role": "system", "content": "sys2"}
+            ]
+        });
+
+        provider.apply_system_prompt_rewrite(&mut body);
+
+        let messages = body_messages(&body);
+        assert_eq!(messages.len(), 2);
+        assert_eq!(messages[0]["role"], "system");
+        assert_eq!(messages[0]["content"], "sys1\n\nsys2");
+        assert_eq!(messages[1]["role"], "user");
+    }
+}

crates/web/ui/e2e/README.md

@@ -42,7 +42,8 @@ app enters onboarding mode. Uses a random free port by default.
 
 ## Playwright Projects
 
-The test suite is split into six Playwright projects:
+The test suite is split into seven Playwright projects, plus one opt-in live
+project for local Ollama/Qwen validation:
 
 | Project | Port | Spec files | Notes |
 |---------|------|------------|-------|
@@ -52,6 +53,7 @@ The test suite is split into six Playwright projects:
 | `onboarding-auth` | Random free port (`MOLTIS_E2E_ONBOARDING_AUTH_PORT`) | `onboarding-auth.spec.js` | Separate server with remote-auth simulation |
 | `onboarding-anthropic` | Random free port (`MOLTIS_E2E_ONBOARDING_ANTHROPIC_PORT`) | `onboarding-anthropic.spec.js` | Separate server proving first-run Anthropic onboarding with zero providers at startup |
 | `openai-live` | Random free port (`MOLTIS_E2E_OPENAI_LIVE_PORT`) | `openai-live.spec.js` | Separate server that preserves only the existing OpenAI env and proves a real OpenAI chat turn works |
+| `ollama-qwen-live` | Random free port (`MOLTIS_E2E_OLLAMA_QWEN_LIVE_PORT`) + Ollama API port (`MOLTIS_E2E_OLLAMA_QWEN_API_PORT`, default `11435`) | `ollama-qwen-live.spec.js` | Opt-in server that starts a local Ollama instance, seeds a custom OpenAI-compatible Qwen provider, and proves the multiple-system-message regression is fixed |
 
 ## Spec Files
 
@@ -74,6 +76,7 @@ The test suite is split into six Playwright projects:
 | `onboarding-auth.spec.js` | 1 | Remote onboarding auth flow with setup code and identity save |
 | `onboarding-anthropic.spec.js` | 1 | Anthropic onboarding from empty startup, model discovery, model selection |
 | `openai-live.spec.js` | 1 | Live OpenAI provider smoke test using the existing env and a real chat turn |
+| `ollama-qwen-live.spec.js` | 1 | Opt-in live Ollama smoke test for the custom OpenAI-compatible Qwen regression path |
 
 ## Shared Helpers
 
@@ -94,6 +97,9 @@ cd crates/web/ui && npx playwright test e2e/specs/sessions.spec.js
 # Run a specific project
 npx playwright test --project=auth
 
+# Run the opt-in Ollama/Qwen live project
+MOLTIS_E2E_OLLAMA_QWEN_LIVE=1 npx playwright test --project=ollama-qwen-live e2e/specs/ollama-qwen-live.spec.js
+
 # Run with visible browser
 just ui-e2e-headed