[Fix] Fix vLLM chat template BOS handling (#2554)

Author: ssiq

opencompass/models/vllm_with_tf_above_v4_33.py

@@ -125,13 +125,10 @@ def generate(self, inputs: List[str], max_out_len: int, stopping_criteria: List[
             messages = _format_with_fast_chat_template(messages, self.fastchat_template)
         else:
             messages = [self.tokenizer.apply_chat_template(m, add_generation_prompt=True, tokenize=False, **self.chat_template_kwargs) for m in messages]
-            # vLLM tokenize prompts by AutoTokenizer with its default parameter "add_special_token=True"
-            # OC add bos_token in the prompt, which requires tokenizing prompts using "add_speicial_token=False"
-            # But vLLM doesn't have "add_speicial_token" in the pipeline API. So, we remove bos_token
-            # from messages as a workaround
-            if self.tokenizer.bos_token:
-                bos_token = self.tokenizer.bos_token
-                messages = [message.removeprefix(bos_token) if message.startswith(bos_token) else message for message in messages]
+            messages = [{
+                'prompt': message,
+                'prompt_token_ids': self.tokenizer.encode(message, add_special_tokens=False),
+            } for message in messages]
         DEFAULT_GENERATION_KWARGS = {
             'temperature': 0,
             'max_tokens': max_out_len,

tests/models/test_vllm_with_tf_above_v4_33.py

@@ -118,6 +118,49 @@ def test_generate_basic(self, mock_ray, mock_sampling_params_class,
         self.assertEqual(results[0], 'Generated response')
         mock_model.generate.assert_called_once()
 
+    @patch('opencompass.models.vllm_with_tf_above_v4_33._convert_chat_messages'
+           )
+    @patch('opencompass.models.vllm_with_tf_above_v4_33.SamplingParams')
+    def test_generate_preserves_bos_token_with_token_ids(
+            self, mock_sampling_params_class, mock_convert_messages):
+        """Test BOS token is preserved in vLLM prompts."""
+        model = VLLMwithChatTemplate.__new__(VLLMwithChatTemplate)
+        model.fastchat_template = None
+        model.chat_template_kwargs = {}
+        model.generation_kwargs = {}
+        model.stop_words = []
+        model.lora_path = None
+        model.logger = MagicMock()
+        mock_model = MagicMock()
+        mock_tokenizer = MagicMock()
+        mock_tokenizer.bos_token = '<bos>'
+        mock_tokenizer.eos_token = None
+        mock_tokenizer.apply_chat_template.return_value = '<bos>Formatted prompt'
+        mock_tokenizer.encode.return_value = [1, 2, 3]
+        model.tokenizer = mock_tokenizer
+        model.model = mock_model
+
+        mock_convert_messages.return_value = [{
+            'role': 'user',
+            'content': 'Hello'
+        }]
+        mock_output = MagicMock()
+        mock_output.outputs = [MagicMock(text='Generated response')]
+        mock_model.generate.return_value = [mock_output]
+        mock_sampling_params = MagicMock()
+        mock_sampling_params_class.return_value = mock_sampling_params
+
+        results = model.generate(['Hello'], max_out_len=100)
+
+        self.assertEqual(results, ['Generated response'])
+        mock_tokenizer.encode.assert_called_once_with(
+            '<bos>Formatted prompt', add_special_tokens=False)
+        prompts = mock_model.generate.call_args[0][0]
+        self.assertEqual(prompts, [{
+            'prompt': '<bos>Formatted prompt',
+            'prompt_token_ids': [1, 2, 3],
+        }])
+
     @patch('opencompass.models.vllm_with_tf_above_v4_33.LLM')
     @patch('transformers.AutoTokenizer')
     @patch(