Skip to content

Commit 135facf

Browse files
jc200808jc200808
authored
feat: Standardize PromptTemplate format to include system + user message pair for all graders (#168)
1 parent 55b2172 commit 135facf

14 files changed

Lines changed: 200 additions & 30 deletions

‎openjudge/graders/code/code_bug_detection.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -188,12 +188,20 @@
188188
DEFAULT_CODE_BUG_DETECTION_TEMPLATE = PromptTemplate(
189189
messages={
190190
LanguageEnum.EN: [
191+
ChatMessage(
192+
role="system",
193+
content=LLMGrader.SYSTEM_PROMPT_EN,
194+
),
191195
ChatMessage(
192196
role="user",
193197
content=CODE_BUG_DETECTION_PROMPT_EN,
194198
),
195199
],
196200
LanguageEnum.ZH: [
201+
ChatMessage(
202+
role="system",
203+
content=LLMGrader.SYSTEM_PROMPT_ZH,
204+
),
197205
ChatMessage(
198206
role="user",
199207
content=CODE_BUG_DETECTION_PROMPT_ZH,

‎openjudge/graders/code/code_complexity.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -208,12 +208,20 @@ class hierarchies, redundant intermediate variables, deeply nested logic, over-g
208208
DEFAULT_CODE_COMPLEXITY_TEMPLATE = PromptTemplate(
209209
messages={
210210
LanguageEnum.EN: [
211+
ChatMessage(
212+
role="system",
213+
content=LLMGrader.SYSTEM_PROMPT_EN,
214+
),
211215
ChatMessage(
212216
role="user",
213217
content=CODE_COMPLEXITY_PROMPT_EN,
214218
),
215219
],
216220
LanguageEnum.ZH: [
221+
ChatMessage(
222+
role="system",
223+
content=LLMGrader.SYSTEM_PROMPT_ZH,
224+
),
217225
ChatMessage(
218226
role="user",
219227
content=CODE_COMPLEXITY_PROMPT_ZH,

‎openjudge/graders/code/code_security.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -182,12 +182,20 @@
182182
DEFAULT_CODE_SECURITY_TEMPLATE = PromptTemplate(
183183
messages={
184184
LanguageEnum.EN: [
185+
ChatMessage(
186+
role="system",
187+
content=LLMGrader.SYSTEM_PROMPT_EN,
188+
),
185189
ChatMessage(
186190
role="user",
187191
content=CODE_SECURITY_PROMPT_EN,
188192
),
189193
],
190194
LanguageEnum.ZH: [
195+
ChatMessage(
196+
role="system",
197+
content=LLMGrader.SYSTEM_PROMPT_ZH,
198+
),
191199
ChatMessage(
192200
role="user",
193201
content=CODE_SECURITY_PROMPT_ZH,

‎openjudge/graders/common/search_correctness.py‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414

1515
from openjudge.agentic import BaseTool, ReActAgent, ToolResult
1616
from openjudge.graders.agentic_grader import AgenticGrader
17+
from openjudge.graders.llm_grader import LLMGrader
1718
from openjudge.graders.schema import GraderError, GraderMode, GraderScore
1819
from openjudge.models.base_chat_model import BaseChatModel
1920
from openjudge.models.schema.oai.message import ChatMessage
@@ -203,10 +204,24 @@ async def aexecute(self, query: str, search_depth: str = "advanced", **kwargs) -
203204
DEFAULT_SEARCH_CORRECTNESS_TEMPLATE = PromptTemplate(
204205
messages={
205206
LanguageEnum.EN: [
206-
ChatMessage(role="user", content=SEARCH_CORRECTNESS_PROMPT_EN),
207+
ChatMessage(
208+
role="system",
209+
content=LLMGrader.SYSTEM_PROMPT_EN,
210+
),
211+
ChatMessage(
212+
role="user",
213+
content=SEARCH_CORRECTNESS_PROMPT_EN,
214+
),
207215
],
208216
LanguageEnum.ZH: [
209-
ChatMessage(role="user", content=SEARCH_CORRECTNESS_PROMPT_ZH),
217+
ChatMessage(
218+
role="system",
219+
content=LLMGrader.SYSTEM_PROMPT_ZH,
220+
),
221+
ChatMessage(
222+
role="user",
223+
content=SEARCH_CORRECTNESS_PROMPT_ZH,
224+
),
210225
],
211226
},
212227
)

‎openjudge/graders/multi_turn/anaphora_resolution_grader.py‎

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -143,11 +143,25 @@
143143

144144
DEFAULT_ANAPHORA_RESOLUTION_TEMPLATE = PromptTemplate(
145145
messages={
146-
LanguageEnum.ZH: [
147-
ChatMessage(role="user", content=ANAPHORA_RESOLUTION_PROMPT_ZH),
148-
],
149146
LanguageEnum.EN: [
150-
ChatMessage(role="user", content=ANAPHORA_RESOLUTION_PROMPT_EN),
147+
ChatMessage(
148+
role="system",
149+
content=LLMGrader.SYSTEM_PROMPT_EN,
150+
),
151+
ChatMessage(
152+
role="user",
153+
content=ANAPHORA_RESOLUTION_PROMPT_EN,
154+
),
155+
],
156+
LanguageEnum.ZH: [
157+
ChatMessage(
158+
role="system",
159+
content=LLMGrader.SYSTEM_PROMPT_ZH,
160+
),
161+
ChatMessage(
162+
role="user",
163+
content=ANAPHORA_RESOLUTION_PROMPT_ZH,
164+
),
151165
],
152166
},
153167
)

‎openjudge/graders/multi_turn/context_memory_grader.py‎

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -140,11 +140,25 @@
140140

141141
DEFAULT_CONTEXT_MEMORY_TEMPLATE = PromptTemplate(
142142
messages={
143-
LanguageEnum.ZH: [
144-
ChatMessage(role="user", content=CONTEXT_MEMORY_PROMPT_ZH),
145-
],
146143
LanguageEnum.EN: [
147-
ChatMessage(role="user", content=CONTEXT_MEMORY_PROMPT_EN),
144+
ChatMessage(
145+
role="system",
146+
content=LLMGrader.SYSTEM_PROMPT_EN,
147+
),
148+
ChatMessage(
149+
role="user",
150+
content=CONTEXT_MEMORY_PROMPT_EN,
151+
),
152+
],
153+
LanguageEnum.ZH: [
154+
ChatMessage(
155+
role="system",
156+
content=LLMGrader.SYSTEM_PROMPT_ZH,
157+
),
158+
ChatMessage(
159+
role="user",
160+
content=CONTEXT_MEMORY_PROMPT_ZH,
161+
),
148162
],
149163
},
150164
)

‎openjudge/graders/multi_turn/instruction_clarification_grader.py‎

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -142,11 +142,25 @@
142142

143143
DEFAULT_INSTRUCTION_CLARIFICATION_TEMPLATE = PromptTemplate(
144144
messages={
145-
LanguageEnum.ZH: [
146-
ChatMessage(role="user", content=INSTRUCTION_CLARIFICATION_PROMPT_ZH),
147-
],
148145
LanguageEnum.EN: [
149-
ChatMessage(role="user", content=INSTRUCTION_CLARIFICATION_PROMPT_EN),
146+
ChatMessage(
147+
role="system",
148+
content=LLMGrader.SYSTEM_PROMPT_EN,
149+
),
150+
ChatMessage(
151+
role="user",
152+
content=INSTRUCTION_CLARIFICATION_PROMPT_EN,
153+
),
154+
],
155+
LanguageEnum.ZH: [
156+
ChatMessage(
157+
role="system",
158+
content=LLMGrader.SYSTEM_PROMPT_ZH,
159+
),
160+
ChatMessage(
161+
role="user",
162+
content=INSTRUCTION_CLARIFICATION_PROMPT_ZH,
163+
),
150164
],
151165
},
152166
)

‎openjudge/graders/multi_turn/proactive_interaction_grader.py‎

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -145,11 +145,25 @@
145145

146146
DEFAULT_PROACTIVE_INTERACTION_TEMPLATE = PromptTemplate(
147147
messages={
148-
LanguageEnum.ZH: [
149-
ChatMessage(role="user", content=PROACTIVE_INTERACTION_PROMPT_ZH),
150-
],
151148
LanguageEnum.EN: [
152-
ChatMessage(role="user", content=PROACTIVE_INTERACTION_PROMPT_EN),
149+
ChatMessage(
150+
role="system",
151+
content=LLMGrader.SYSTEM_PROMPT_EN,
152+
),
153+
ChatMessage(
154+
role="user",
155+
content=PROACTIVE_INTERACTION_PROMPT_EN,
156+
),
157+
],
158+
LanguageEnum.ZH: [
159+
ChatMessage(
160+
role="system",
161+
content=LLMGrader.SYSTEM_PROMPT_ZH,
162+
),
163+
ChatMessage(
164+
role="user",
165+
content=PROACTIVE_INTERACTION_PROMPT_ZH,
166+
),
153167
],
154168
},
155169
)

‎openjudge/graders/multi_turn/response_repetition_grader.py‎

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -148,11 +148,25 @@
148148

149149
DEFAULT_RESPONSE_REPETITION_TEMPLATE = PromptTemplate(
150150
messages={
151-
LanguageEnum.ZH: [
152-
ChatMessage(role="user", content=RESPONSE_REPETITION_PROMPT_ZH),
153-
],
154151
LanguageEnum.EN: [
155-
ChatMessage(role="user", content=RESPONSE_REPETITION_PROMPT_EN),
152+
ChatMessage(
153+
role="system",
154+
content=LLMGrader.SYSTEM_PROMPT_EN,
155+
),
156+
ChatMessage(
157+
role="user",
158+
content=RESPONSE_REPETITION_PROMPT_EN,
159+
),
160+
],
161+
LanguageEnum.ZH: [
162+
ChatMessage(
163+
role="system",
164+
content=LLMGrader.SYSTEM_PROMPT_ZH,
165+
),
166+
ChatMessage(
167+
role="user",
168+
content=RESPONSE_REPETITION_PROMPT_ZH,
169+
),
156170
],
157171
},
158172
)

‎openjudge/graders/multi_turn/self_correction_grader.py‎

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -145,11 +145,25 @@
145145

146146
DEFAULT_SELF_CORRECTION_TEMPLATE = PromptTemplate(
147147
messages={
148-
LanguageEnum.ZH: [
149-
ChatMessage(role="user", content=SELF_CORRECTION_PROMPT_ZH),
150-
],
151148
LanguageEnum.EN: [
152-
ChatMessage(role="user", content=SELF_CORRECTION_PROMPT_EN),
149+
ChatMessage(
150+
role="system",
151+
content=LLMGrader.SYSTEM_PROMPT_EN,
152+
),
153+
ChatMessage(
154+
role="user",
155+
content=SELF_CORRECTION_PROMPT_EN,
156+
),
157+
],
158+
LanguageEnum.ZH: [
159+
ChatMessage(
160+
role="system",
161+
content=LLMGrader.SYSTEM_PROMPT_ZH,
162+
),
163+
ChatMessage(
164+
role="user",
165+
content=SELF_CORRECTION_PROMPT_ZH,
166+
),
153167
],
154168
},
155169
)

0 commit comments

Comments
 (0)