[Fix] Fix per-turn statistics in dump-res-length and repeat detection issues for multi-round conversations (#2569)

Author: Myhs-phz

opencompass/openicl/icl_inferencer/icl_base_inferencer.py

@@ -169,9 +169,9 @@ def save_results(self,
         }
         if gold:
             self.results_dict[str(idx)]['gold'] = gold
-        if res_length:
+        if res_length is not None:
             self.results_dict[str(idx)]['res_length'] = res_length
-        if input_length:
+        if input_length is not None:
             self.results_dict[str(idx)]['all_input_length'] = input_length
 
 

opencompass/openicl/icl_inferencer/icl_gen_inferencer.py

@@ -190,17 +190,20 @@ def inference(self,
             num_return_sequences = getattr(self.model, 'generation_kwargs',
                                            {}).get('num_return_sequences', 1)
             # 5-3. Save current output
-            for prompt, prediction, gold in zip(
-                    parsed_entries, batched(generated, num_return_sequences),
-                    golds):
+            for batch_idx, (prompt, prediction, gold) in enumerate(
+                    zip(parsed_entries, batched(generated,
+                                                num_return_sequences), golds)):
                 if num_return_sequences == 1:
                     prediction = prediction[0]
 
                 if self.dump_res_length:
-                    input_length = 0
-                    if isinstance(prompt, str):
+                    if self.multiround and isinstance(prompt, list):
+                        input_length = self._compute_multiround_input_lengths(
+                            entry[batch_idx])
+                    elif isinstance(prompt, str):
                         input_length = self.model.get_token_len(prompt)
                     elif isinstance(prompt, list):
+                        input_length = 0
                         for i in range(len(prompt)):
                             if 'prompt' in prompt[i]:
                                 prompt[i][
@@ -275,6 +278,29 @@ def inference(self,
             for sample in output_handler.results_dict.values()
         ]
 
+    def _compute_multiround_input_lengths(self, chat: List) -> List[int]:
+        """Compute cumulative input token length at each generation turn.
+
+        Expects ``chat`` to be the filled multi-round conversation where
+        assistant slots already contain the generated responses. Returns a
+        list of cumulative token counts, one per generation turn, i.e. the
+        input length the model actually sees when generating each turn.
+        """
+        input_lengths = []
+        cumulative = 0
+        for msg in chat:
+            if isinstance(msg, dict):
+                role = msg.get('role', '')
+                content = msg.get('content', msg.get('prompt', ''))
+            else:
+                role, content = '', msg
+            if not isinstance(content, str):
+                content = str(content)
+            if role == 'assistant':
+                input_lengths.append(cumulative)
+            cumulative += self.model.get_token_len(content)
+        return input_lengths
+
     def _generate_multiround(self, entry: List,
                              extra_gen_kwargs: dict) -> List[List[str]]:
         """Multi-turn generation with dynamic turn-level scheduling.

opencompass/openicl/icl_inferencer/icl_gen_inferencer_parallel.py

@@ -154,10 +154,13 @@ def _infer_one(entry, gold, idx):
             )
 
             if self.dump_res_length:
-                input_length = 0
-                if isinstance(parsed_entry, str):
+                if self.multiround and isinstance(parsed_entry, list):
+                    input_length = self._compute_multiround_input_lengths(
+                        entry)
+                elif isinstance(parsed_entry, str):
                     input_length = self.model.get_token_len(parsed_entry)
                 elif isinstance(parsed_entry, list):
+                    input_length = 0
                     for i in range(len(parsed_entry)):
                         if 'prompt' in parsed_entry[i]:
                             parsed_entry[i][

opencompass/utils/repeat_analysis.py

@@ -367,8 +367,10 @@ def _collect_predictions_by_benchmark(
                         raw_res_length = sample.get('res_length', None)
                         if raw_res_length is not None:
                             if isinstance(raw_res_length, list):
-                                res_length = (raw_res_length[0]
-                                              if raw_res_length else 0)
+                                # Multi-turn: each turn's response is joined
+                                # by _prediction_to_text with '\n', so sum
+                                # across turns to match the combined text.
+                                res_length = sum(raw_res_length)
                             else:
                                 res_length = raw_res_length
                     else: