Skip to content

Commit af8fbcd

Browse files
authored
[Fix] Prevent return-value deception in code evaluators (#2565)
1 parent 63b0199 commit af8fbcd

9 files changed

Lines changed: 192 additions & 35 deletions

File tree

opencompass/datasets/LCBench.py

Lines changed: 20 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,9 @@
1515
from opencompass.openicl.icl_evaluator import BaseEvaluator
1616
from opencompass.registry import ICL_EVALUATORS, LOAD_DATASET
1717
from opencompass.utils import get_data_path
18+
from opencompass.utils.code_execution import (TYPE_AWARE_EQUAL_NAME,
19+
make_assertions_type_aware,
20+
type_aware_equal)
1821

1922
from .base import BaseDataset
2023

@@ -130,21 +133,18 @@ def score(self, predictions, references):
130133

131134
# Try each code block until one passes
132135
for code_idx, code_block in enumerate(code_blocks):
133-
test_programs = self._process_test(refer, code_block)
134-
135-
# Submit each test program variant for execution
136-
for prog_idx, program in enumerate(test_programs):
137-
future = executor.submit(
138-
execution,
139-
program,
140-
(
141-
i,
142-
code_idx,
143-
prog_idx,
144-
), # Pass indices for tracking
145-
3,
146-
)
147-
futures.append(future)
136+
test_program = self._process_test(refer, code_block)
137+
future = executor.submit(
138+
execution,
139+
test_program,
140+
(
141+
i,
142+
code_idx,
143+
0,
144+
),
145+
3,
146+
)
147+
futures.append(future)
148148

149149
from tqdm import tqdm
150150

@@ -291,10 +291,7 @@ def _process_test(self, test_case, code):
291291
# Use the modified test
292292
test_case = modified_test
293293

294-
formatted = code + '\n'
295-
formatted += test_case
296-
# breakpoint()
297-
return formatted
294+
return code, make_assertions_type_aware(test_case)
298295

299296

300297
def execution(programs, task_ids, timeout):
@@ -313,10 +310,13 @@ def _execution(programs, timeout):
313310
try:
314311
# Add exec globals to prevent the exec to raise
315312
# unnecessary NameError for correct answer
313+
code, test_case = programs
316314
exec_globals = {}
317315
with swallow_io():
318316
with time_limit(timeout):
319-
exec(programs, exec_globals)
317+
exec(code, exec_globals)
318+
exec_globals[TYPE_AWARE_EQUAL_NAME] = type_aware_equal
319+
exec(test_case, exec_globals)
320320
key.append('pass')
321321
except TimeOutException:
322322
key.append('timeout')

opencompass/datasets/apps.py

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727

2828
from opencompass.openicl.icl_evaluator import BaseEvaluator
2929
from opencompass.registry import ICL_EVALUATORS, LOAD_DATASET
30+
from opencompass.utils.code_execution import type_aware_equal
3031

3132
from .base import BaseDataset
3233

@@ -490,11 +491,12 @@ def run_test(sample, test=None, debug=False):
490491
if isinstance(output, tuple):
491492
output = list(output)
492493

493-
tmp_result = output == in_outs['outputs'][index]
494+
tmp_result = type_aware_equal(output,
495+
in_outs['outputs'][index])
494496
if isinstance(in_outs['outputs'][index],
495497
list) and in_outs['outputs'][index]:
496-
tmp_result = tmp_result or (
497-
output == in_outs['outputs'][index][0])
498+
tmp_result = tmp_result or type_aware_equal(
499+
output, in_outs['outputs'][index][0])
498500

499501
# ground truth sequences are not tuples
500502
try:

opencompass/datasets/livecodebench/evaluator.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
from opencompass.openicl.icl_evaluator import BaseEvaluator
1111
from opencompass.registry import ICL_EVALUATORS
1212
from opencompass.utils import get_logger
13+
from opencompass.utils.code_execution import TYPE_AWARE_EQUAL_NAME
1314

1415
from .execute_utils import BASE_IMPORTS, codeexecute_check_correctness
1516
from .extract_utils import (extract_code_execution, extract_code_generation,
@@ -365,7 +366,8 @@ def evaluate_score(args) -> list[bool]:
365366
if i in g:
366367
pass
367368
else:
368-
code_to_execute = f'{BASE_IMPORTS}\n{c}\nassert {o} == {g}'
369+
code_to_execute = (f'{BASE_IMPORTS}\n{c}',
370+
f'assert {TYPE_AWARE_EQUAL_NAME}({o}, {g})')
369371
execution_results.append(
370372
codeexecute_check_correctness(code_to_execute, 3))
371373
if len(execution_results) == 0:

opencompass/datasets/livecodebench/execute_utils.py

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,9 @@
2525
import signal
2626
import tempfile
2727

28+
from opencompass.utils.code_execution import (TYPE_AWARE_EQUAL_NAME,
29+
type_aware_equal)
30+
2831
BASE_IMPORTS = """from itertools import accumulate, chain, combinations, count, permutations, product, groupby, islice, repeat
2932
from copy import deepcopy
3033
from string import ascii_lowercase
@@ -103,10 +106,13 @@ def unsafe_execute(check_program, result, timeout):
103106

104107
# Run program.
105108
try:
109+
code, test_case = check_program
106110
exec_globals = {}
107111
with swallow_io():
108112
with time_limit(timeout):
109-
exec(check_program, exec_globals)
113+
exec(code, exec_globals)
114+
exec_globals[TYPE_AWARE_EQUAL_NAME] = type_aware_equal
115+
exec(test_case, exec_globals)
110116
result.append('passed')
111117
except TimeoutException:
112118
result.append('timed out')

opencompass/datasets/livecodebench/testing_util.py

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@
1818

1919
import numpy as np
2020

21+
from opencompass.utils.code_execution import type_aware_equal
22+
2123
try:
2224
from pyext import RuntimeModule
2325
except ImportError:
@@ -285,11 +287,12 @@ def run_test(sample,
285287
if isinstance(output, tuple):
286288
output = list(output)
287289

288-
tmp_result = output == in_outs['outputs'][index]
290+
tmp_result = type_aware_equal(output,
291+
in_outs['outputs'][index])
289292
if (isinstance(in_outs['outputs'][index], list)
290293
and in_outs['outputs'][index]):
291-
tmp_result = tmp_result or (
292-
output == in_outs['outputs'][index][0])
294+
tmp_result = tmp_result or type_aware_equal(
295+
output, in_outs['outputs'][index][0])
293296

294297
# ground truth sequences are not tuples
295298
try:

opencompass/datasets/mbpp.py

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,9 @@
1818
from opencompass.openicl.icl_evaluator import BaseEvaluator
1919
from opencompass.registry import ICL_EVALUATORS, LOAD_DATASET
2020
from opencompass.utils import get_data_path
21+
from opencompass.utils.code_execution import (TYPE_AWARE_EQUAL_NAME,
22+
make_assertions_type_aware,
23+
type_aware_equal)
2124

2225
from .base import BaseDataset
2326

@@ -345,9 +348,7 @@ def _process_answer(self, text):
345348
return text
346349

347350
def _process_test(self, test_case, pred):
348-
formatted = pred + '\n'
349-
formatted += test_case
350-
return formatted
351+
return pred, make_assertions_type_aware(test_case)
351352

352353

353354
@ICL_EVALUATORS.register_module()
@@ -392,10 +393,13 @@ def _execution(programs, timeout, key):
392393
try:
393394
# Add exec globals to prevent the exec to raise
394395
# unnecessary NameError for correct answer
396+
code, test_case = programs
395397
exec_globals = {}
396398
with swallow_io():
397399
with time_limit(timeout):
398-
exec(programs, exec_globals)
400+
exec(code, exec_globals)
401+
exec_globals[TYPE_AWARE_EQUAL_NAME] = type_aware_equal
402+
exec(test_case, exec_globals)
399403
key.append('pass')
400404
except TimeOutException:
401405
key.append('timeout')

opencompass/datasets/taco.py

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828

2929
from opencompass.openicl.icl_evaluator import BaseEvaluator
3030
from opencompass.registry import ICL_EVALUATORS, LOAD_DATASET
31+
from opencompass.utils.code_execution import type_aware_equal
3132

3233
from .base import BaseDataset
3334

@@ -451,11 +452,12 @@ def run_test(sample, test=None, debug=False):
451452
if isinstance(output, tuple):
452453
output = list(output)
453454

454-
tmp_result = output == in_outs['outputs'][index]
455+
tmp_result = type_aware_equal(output,
456+
in_outs['outputs'][index])
455457
if isinstance(in_outs['outputs'][index],
456458
list) and in_outs['outputs'][index]:
457-
tmp_result = tmp_result or (
458-
output == in_outs['outputs'][index][0])
459+
tmp_result = tmp_result or type_aware_equal(
460+
output, in_outs['outputs'][index][0])
459461

460462
# ground truth sequences are not tuples
461463
try:
Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
import ast
2+
from collections.abc import Mapping
3+
from typing import Any
4+
5+
TYPE_AWARE_EQUAL_NAME = '__opencompass_type_aware_equal'
6+
7+
8+
def type_aware_equal(actual: Any, expected: Any) -> bool:
9+
"""Compare values without accepting forged ``__eq__`` on return objects."""
10+
if type(actual) is not type(expected):
11+
return False
12+
13+
if isinstance(actual, (list, tuple)):
14+
return (len(actual) == len(expected) and all(
15+
type_aware_equal(a, e) for a, e in zip(actual, expected)))
16+
17+
if isinstance(actual, Mapping):
18+
if len(actual) != len(expected):
19+
return False
20+
21+
matched_expected_keys = set()
22+
for actual_key, actual_value in actual.items():
23+
match_idx = None
24+
for idx, (expected_key,
25+
expected_value) in enumerate(expected.items()):
26+
if idx in matched_expected_keys:
27+
continue
28+
if (type_aware_equal(actual_key, expected_key)
29+
and type_aware_equal(actual_value, expected_value)):
30+
match_idx = idx
31+
break
32+
if match_idx is None:
33+
return False
34+
matched_expected_keys.add(match_idx)
35+
return True
36+
37+
if isinstance(actual, (set, frozenset)):
38+
if len(actual) != len(expected):
39+
return False
40+
41+
expected_items = list(expected)
42+
matched_expected_items = set()
43+
for actual_item in actual:
44+
match_idx = None
45+
for idx, expected_item in enumerate(expected_items):
46+
if idx in matched_expected_items:
47+
continue
48+
if type_aware_equal(actual_item, expected_item):
49+
match_idx = idx
50+
break
51+
if match_idx is None:
52+
return False
53+
matched_expected_items.add(match_idx)
54+
return True
55+
56+
try:
57+
result = actual == expected
58+
except Exception:
59+
return False
60+
return type(result) is bool and result
61+
62+
63+
class _TypeAwareAssertTransformer(ast.NodeTransformer):
64+
"""Replace ``assert actual == expected`` with a type-aware comparison."""
65+
66+
def visit_Assert(self, node: ast.Assert) -> ast.Assert:
67+
self.generic_visit(node)
68+
comparison = node.test
69+
if (isinstance(comparison, ast.Compare) and len(comparison.ops) == 1
70+
and isinstance(comparison.ops[0], ast.Eq)):
71+
node.test = ast.Call(
72+
func=ast.Name(id=TYPE_AWARE_EQUAL_NAME, ctx=ast.Load()),
73+
args=[comparison.left, comparison.comparators[0]],
74+
keywords=[],
75+
)
76+
return node
77+
78+
79+
def make_assertions_type_aware(source: str) -> str:
80+
"""Harden direct equality assertions in a dataset-provided test program."""
81+
try:
82+
tree = ast.parse(source)
83+
except SyntaxError:
84+
return source
85+
tree = _TypeAwareAssertTransformer().visit(tree)
86+
ast.fix_missing_locations(tree)
87+
return ast.unparse(tree)
Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
import unittest
2+
3+
from opencompass.utils.code_execution import (TYPE_AWARE_EQUAL_NAME,
4+
make_assertions_type_aware,
5+
type_aware_equal)
6+
7+
8+
class _AlwaysEqual:
9+
10+
def __eq__(self, other):
11+
return True
12+
13+
14+
class _AlwaysEqualInt(int):
15+
16+
def __new__(cls):
17+
return int.__new__(cls, 999)
18+
19+
def __eq__(self, other):
20+
return True
21+
22+
23+
class TestCodeExecution(unittest.TestCase):
24+
25+
def test_type_aware_equal_rejects_eq_override(self):
26+
self.assertFalse(type_aware_equal(_AlwaysEqual(), 1))
27+
28+
def test_type_aware_equal_rejects_nested_eq_override(self):
29+
self.assertFalse(type_aware_equal([_AlwaysEqualInt()], [1]))
30+
31+
def test_type_aware_equal_accepts_nested_builtin_value(self):
32+
self.assertTrue(type_aware_equal({'value': [1, 2]}, {'value': [1, 2]}))
33+
34+
def test_type_aware_assertion_rejects_eq_override(self):
35+
test_program = make_assertions_type_aware('assert solve() == 1')
36+
namespace = {
37+
'solve': _AlwaysEqual,
38+
TYPE_AWARE_EQUAL_NAME: type_aware_equal,
39+
}
40+
41+
with self.assertRaises(AssertionError):
42+
exec(test_program, namespace)
43+
44+
def test_type_aware_assertion_accepts_matching_builtin_value(self):
45+
test_program = make_assertions_type_aware('assert solve() == 1')
46+
namespace = {
47+
'solve': lambda: 1,
48+
TYPE_AWARE_EQUAL_NAME: type_aware_equal,
49+
}
50+
51+
exec(test_program, namespace)

0 commit comments

Comments
 (0)