fix(session-goal): judge goal completion on host evidence, not the model's claim (#4852)

Author: YauhenBichel

core/agent_harness/AGENTS.md

@@ -26,7 +26,12 @@ are separate layers.
 
 Outer `SessionGoal` (`session/session_goal.py` + `turns/session_goal_loop.py`) is
 **not** the inner ReAct `Goal` / `goal_review`. While a session goal is active,
-`run_turn` suppresses investigation Want-me-to closers.
+`run_turn` suppresses investigation Want-me-to closers. Completion is judged by
+`session/session_goal_evaluate.py` (independent of model self-report): checklist
+complete via `done=` indices, or `session_goal:achieved` **only** with tool
+evidence when there is no checklist — bare `achieved` is ignored. Optional LLM
+confirm for the tool-evidence path: `build_session_goal_llm_evaluator` in
+`session/session_goal_review.py` (pass as `evaluate=` to the outer loop).
 
 **Evidence kinds (open/closed):** vocabulary + per-kind policy live in
 `turns/evidence_kind.py` (`EvidenceKind` + `EvidenceKindPolicy`). Add a kind by

core/agent_harness/prompts/action/multi_step_policy.py

@@ -60,9 +60,11 @@
 shell_run): when the user wants a multi-step chat checklist / walkthrough
 with no local create/run/file work — "walk through these N steps in chat",
 "checklist without asking whether to continue", "keep going until every
-item is done" — emit assistant_handoff with session_goal (and preferably
-session_goal_items). The host outer loop continues turns; do not invent
-shell side effects to "prove" progress.
+item is done" — emit ONE assistant_handoff and set the schema field
+session_goal=true (REQUIRED — omitting it leaves the host unable to continue
+across turns). Prefer session_goal_items with one string per checklist item
+in order. Do not invent shell side effects to "prove" progress; do not put
+session_goal only in content prose when the boolean field is available.
 Still assistant_handoff (no execution requested):
 * capability questions — "do you support consecutive steps?", "can you loop?"
 * explicit plan-only requests — "do not write any code yet; first create a

core/agent_harness/prompts/action/text.py

@@ -582,12 +582,21 @@
   decides L0 vs L1 from that field plus live connectivity. Omit only for pure
   explain/docs chat about analytics with no number request.
 - evidence_kind=incident — bare incident / symptom handoffs.
-- session_goal=true — multi-step or "keep going until done" work that should
-  continue across turns without asking whether to continue.
+- session_goal=true — REQUIRED on every handoff for multi-step or
+  "keep going until done" chat checklists / walkthroughs (no local shell
+  work). The host outer loop keys off this boolean; omitting it drops
+  continuation. Prefer session_goal_items=["…", …] for checklist criteria.
 - session_goal_max_turns=<n> — optional outer-turn cap for that goal.
 - session_goal_items=["…", …] — checklist success criteria (one string per
   item, in order). The host tracks completion via session_goal:done=<index>
   in later replies; do not invent checklist items from synonyms.
+When a host session goal is active (or you just emitted session_goal:achieved
+for one), finish the reply without a Want me to: closer — the outer loop owns
+continuation; do not ask the user whether to continue or clear the goal.
+Never emit session_goal:achieved in the same turn as investigation_start —
+starting RCA is not finishing the goal. After investigation results (or other
+real tool answers) are in the reply and the condition's deliverables are met
+(issue id, count, next action, …), then emit session_goal:achieved.
 Legacy content-string tags still work if you must put them in content
 (``evidence_kind:metric_read`` or ``evidence_kind=metric_read``, same for
 ``session_goal`` / ``session_goal_item``). Prefer the schema fields above so the

core/agent_harness/prompts/assistant/observation.py

@@ -12,6 +12,62 @@
     "incident_description:",
 )
 
+_ON_SCREEN_FRAMING = (
+    "A read-only discovery command was just run to answer the user's question; "
+    "its output is below. Summarize it to answer the user's question directly, "
+    "citing the relevant status. The output is already on screen, so keep "
+    "**Here's what that looks like:** brief or omit it when it would repeat "
+    "what the user just saw. Still end with **Want me to:** and a specific "
+    "next step tied to the finding (for integration questions: connect another "
+    "integration, verify a failed service, or set up a missing one)."
+)
+
+_ON_SCREEN_FRAMING_NO_WANT_ME_TO = (
+    "A read-only discovery command was just run to answer the user's question; "
+    "its output is below. Summarize it to answer the user's question directly, "
+    "citing the relevant status. The output is already on screen, so keep "
+    "**Here's what that looks like:** brief or omit it when it would repeat "
+    "what the user just saw."
+)
+
+_OFF_SCREEN_FRAMING = (
+    "Live data was just gathered from the connected integrations to answer the "
+    "user's question; the tool results are below and are NOT otherwise shown to "
+    "the user. Answer using the three-part response shape from the system "
+    "prompt: **I found:**, **Here's what that looks like:**, and **Want me to:** "
+    "with a specific next step. Cite concrete findings (issues, log lines, or "
+    "metrics). If the data does not contain the answer, say so plainly. You have "
+    "ALREADY queried the connected sources, so do NOT tell the user to paste an "
+    "alert or to run `opensre investigate`; instead report what each source "
+    "returned and, if you need more signal, ask for the specific detail (error "
+    "string, service, version, or time window) that would let you narrow it down "
+    "here."
+)
+
+_OFF_SCREEN_FRAMING_NO_WANT_ME_TO = (
+    "Live data was just gathered from the connected integrations to answer the "
+    "user's question; the tool results are below and are NOT otherwise shown to "
+    "the user. Answer using **I found:** and **Here's what that looks like:** — "
+    "omit **Want me to:** entirely (an outer session goal owns continuation). "
+    "Cite concrete findings (issues, log lines, or metrics). If the data does "
+    "not contain the answer, say so plainly. You have ALREADY queried the "
+    "connected sources, so do NOT tell the user to paste an alert or to run "
+    "`opensre investigate`; instead report what each source returned and, if "
+    "you need more signal, ask for the specific detail (error string, service, "
+    "version, or time window) that would let you narrow it down here."
+)
+
+_CLOSER_WANT_ME_TO = (
+    "Do NOT request, plan, or emit any further tool calls or actions in this "
+    "turn — phrase next steps only as prose in **Want me to:**."
+)
+
+_CLOSER_NO_WANT_ME_TO = (
+    "Do NOT request, plan, or emit any further tool calls or actions in this "
+    "turn. Do NOT close with **Want me to:** — phrase any next diagnostic step "
+    "as plain prose in the body if the goal asks for it."
+)
+
 
 def _guidance_for_handoff_tag(tag: str) -> str | None:
     """Resolve exact handoff tags, then known prefixes (e.g. ``database_query:``)."""
@@ -33,40 +89,25 @@ def build_handoff_guidance_block(handoff_contents: tuple[str, ...]) -> str:
     return "".join(blocks)
 
 
-def build_observation_block(tool_observation: str | None, *, on_screen: bool = True) -> str:
-    """Wrap freshly-gathered tool output so the assistant summarizes it directly."""
+def build_observation_block(
+    tool_observation: str | None,
+    *,
+    on_screen: bool = True,
+    omit_want_me_to: bool = False,
+) -> str:
+    """Wrap freshly-gathered tool output so the assistant summarizes it directly.
+
+    When ``omit_want_me_to`` is true (outer session goal active), skip the
+    Want-me-to closer — the host loop owns continuation.
+    """
     if not tool_observation or not tool_observation.strip():
         return ""
     if on_screen:
-        framing = (
-            "A read-only discovery command was just run to answer the user's question; "
-            "its output is below. Summarize it to answer the user's question directly, "
-            "citing the relevant status. The output is already on screen, so keep "
-            "**Here's what that looks like:** brief or omit it when it would repeat "
-            "what the user just saw. Still end with **Want me to:** and a specific "
-            "next step tied to the finding (for integration questions: connect another "
-            "integration, verify a failed service, or set up a missing one)."
-        )
+        framing = _ON_SCREEN_FRAMING_NO_WANT_ME_TO if omit_want_me_to else _ON_SCREEN_FRAMING
     else:
-        framing = (
-            "Live data was just gathered from the connected integrations to answer the "
-            "user's question; the tool results are below and are NOT otherwise shown to "
-            "the user. Answer using the three-part response shape from the system "
-            "prompt: **I found:**, **Here's what that looks like:**, and **Want me to:** "
-            "with a specific next step. Cite concrete findings (issues, log lines, or "
-            "metrics). If the data does not contain the answer, say so plainly. You have "
-            "ALREADY queried the connected sources, so do NOT tell the user to paste an "
-            "alert or to run `opensre investigate`; instead report what each source "
-            "returned and, if you need more signal, ask for the specific detail (error "
-            "string, service, version, or time window) that would let you narrow it down "
-            "here."
-        )
-    return (
-        f"{framing} Do NOT request, plan, or emit any further tool calls or "
-        "actions in this turn — phrase next steps only as prose in "
-        "**Want me to:**.\n\n"
-        f"--- tool_results ---\n{tool_observation}\n\n"
-    )
+        framing = _OFF_SCREEN_FRAMING_NO_WANT_ME_TO if omit_want_me_to else _OFF_SCREEN_FRAMING
+    closer = _CLOSER_NO_WANT_ME_TO if omit_want_me_to else _CLOSER_WANT_ME_TO
+    return f"{framing} {closer}\n\n--- tool_results ---\n{tool_observation}\n\n"
 
 
 # Legacy private name used by older tests.

core/agent_harness/prompts/assistant/text.py

@@ -103,7 +103,9 @@
         "(comma-separate multiple). When every item is done, include "
         "`session_goal:achieved`. Put those tags on their own at the end of "
         "the reply; the harness strips them before the user sees the text. "
-        "Do not ask whether to continue while the goal is active.\n\n"
+        "Do not ask whether to continue while the goal is active. "
+        "Do NOT close with **Want me to:** (no investigation offer, no "
+        "follow-up menu) — the outer loop owns continuation.\n\n"
     ),
     # Prefix key: ``build_handoff_guidance_block`` matches any
     # ``database_query:<topic>`` tag (mysql_active_connections, mariadb_dashboard, …).

core/agent_harness/prompts/assistant/turn.py

@@ -68,6 +68,10 @@ def log_diagnostics(self, reason: str) -> None:
         """Emit grounding-cache diagnostics for ``reason``."""
 
 
+def _handoff_has_session_goal(handoff_contents: tuple[str, ...]) -> bool:
+    return any(tag.startswith("session_goal:") for tag in handoff_contents)
+
+
 def _assistant_context_blocks(
     *,
     turn_snapshot: TurnSnapshot,
@@ -76,11 +80,16 @@ def _assistant_context_blocks(
     tool_observation_on_screen: bool,
     suggested_prompt: str = SUGGESTED_PROMPT_AFTER_FAILED_SYNTHETIC_TEST,
 ) -> str:
+    omit_want_me_to = _handoff_has_session_goal(handoff_contents)
     return "".join(
         (
             _build_integration_guard(turn_snapshot),
             build_handoff_guidance_block(handoff_contents),
-            build_observation_block(tool_observation, on_screen=tool_observation_on_screen),
+            build_observation_block(
+                tool_observation,
+                on_screen=tool_observation_on_screen,
+                omit_want_me_to=omit_want_me_to,
+            ),
             synthetic_failure.build_block(turn_snapshot, suggested_prompt=suggested_prompt),
         )
     )