fix: eliminate thinking-block poisoning + no-op navigation trap
ROOT CAUSE: qwen3.5 (reasoning model) returns response='' with thinking block containing all reasoning. llm_provider.py line 352 silently substituted the thinking block as the response via: content = raw_response or raw_thinking or '' The Brain then extracted random actions from the reasoning text. FIXES: 1. llm_provider.py: Conditional thinking isolation - format_json=True (SAE/perception): thinking fallback preserved - format_json=False (Brain): thinking NEVER substituted - Added think=false for Ollama free-text calls to force direct response 2. planner.py: No-Op Guard strips tab actions that navigate to the current screen (e.g. 'tap profile tab' on OWN_PROFILE) 3. test_brain_live.py: Stochastic testing (5 runs, 60% min valid) to handle non-deterministic LLM behavior reliably 4. tests/integration/test_llm_provider_pipeline.py: NEW test layer mocking at HTTP level (requests.post) to exercise the FULL llm_provider → Brain pipeline. This would have caught the thinking substitution bug from day one. Suite: 168 passed, 0 failed
This commit is contained in:
@@ -207,3 +207,130 @@ class TestUIChangedFidelity:
|
||||
|
||||
|
||||
from GramAddict.core.perception.screen_identity import ScreenType # noqa: E402
|
||||
|
||||
|
||||
class TestBrainEmptyResponse:
|
||||
"""Contract: When the LLM returns response='', the Brain must NOT
|
||||
extract actions from the thinking block. The thinking block is
|
||||
REASONING, not decisions."""
|
||||
|
||||
def test_empty_response_returns_none_not_thinking_extraction(self, monkeypatch):
|
||||
"""REGRESSION: In the 2026-04-28 23:51 run, the LLM returned response=''
|
||||
with a thinking block mentioning 'tap profile tab'. The Brain extracted
|
||||
'tap profile tab' which was a no-op on OWN_PROFILE."""
|
||||
import GramAddict.core.navigation.brain
|
||||
|
||||
def mock_llm(**kwargs):
|
||||
# The EXACT production failure: response is empty, thinking has actions
|
||||
return {"response": ""}
|
||||
|
||||
monkeypatch.setattr(GramAddict.core.navigation.brain, "query_llm", mock_llm)
|
||||
|
||||
result = ask_brain_for_action(
|
||||
goal="nurture community",
|
||||
screen_type="OWN_PROFILE",
|
||||
available_actions=["tap message button", "scroll down", "press back", "tap profile tab"],
|
||||
explored_actions=set(),
|
||||
)
|
||||
# When the LLM gives NO response, the Brain must return None
|
||||
# to force the planner's structural fallback
|
||||
assert result is None, (
|
||||
f"Brain returned '{result}' from an empty LLM response! "
|
||||
f"It must return None so the planner can use HD Map fallback."
|
||||
)
|
||||
|
||||
def test_whitespace_only_response_treated_as_empty(self, monkeypatch):
|
||||
"""Response with only whitespace/newlines is effectively empty."""
|
||||
import GramAddict.core.navigation.brain
|
||||
|
||||
def mock_llm(**kwargs):
|
||||
return {"response": " \n \n "}
|
||||
|
||||
monkeypatch.setattr(GramAddict.core.navigation.brain, "query_llm", mock_llm)
|
||||
|
||||
result = ask_brain_for_action(
|
||||
goal="open explore",
|
||||
screen_type="HOME_FEED",
|
||||
available_actions=["tap explore tab", "scroll down"],
|
||||
explored_actions=set(),
|
||||
)
|
||||
assert result is None, (
|
||||
f"Brain returned '{result}' from a whitespace-only response! "
|
||||
f"Must return None."
|
||||
)
|
||||
|
||||
|
||||
class TestPlannerNoOpGuard:
|
||||
"""Contract: The planner must NEVER ask the Brain to execute a tab action
|
||||
that would navigate to the screen we're already on."""
|
||||
|
||||
def test_planner_strips_current_screen_tab_before_brain(self, monkeypatch):
|
||||
"""On OWN_PROFILE, 'tap profile tab' is a no-op. The planner must
|
||||
strip it from available_actions before asking the Brain."""
|
||||
import GramAddict.core.navigation.brain
|
||||
from GramAddict.core.navigation.planner import GoalPlanner
|
||||
|
||||
captured_prompts = []
|
||||
|
||||
def spy_query_llm(**kwargs):
|
||||
captured_prompts.append(kwargs.get("system", ""))
|
||||
return {"response": "scroll down"}
|
||||
|
||||
monkeypatch.setattr(GramAddict.core.navigation.brain, "query_llm", spy_query_llm)
|
||||
|
||||
planner = GoalPlanner("test_user")
|
||||
screen = {
|
||||
"screen_type": ScreenType.OWN_PROFILE,
|
||||
"available_actions": ["tap profile tab", "tap home tab", "scroll down", "press back"],
|
||||
"context": {},
|
||||
}
|
||||
|
||||
planner.plan_next_step("nurture community", screen)
|
||||
|
||||
assert len(captured_prompts) == 1, "Brain was not called"
|
||||
prompt = captured_prompts[0]
|
||||
|
||||
for line in prompt.splitlines():
|
||||
if "available to you right now" in line:
|
||||
assert "tap profile tab" not in line, (
|
||||
f"Planner passed no-op action 'tap profile tab' to Brain on OWN_PROFILE!\n"
|
||||
f"Line: {line}"
|
||||
)
|
||||
break
|
||||
else:
|
||||
pytest.fail("Could not find 'available to you right now' in Brain prompt")
|
||||
|
||||
def test_planner_strips_home_tab_on_home_feed(self, monkeypatch):
|
||||
"""On HOME_FEED, 'tap home tab' is a no-op."""
|
||||
import GramAddict.core.navigation.brain
|
||||
from GramAddict.core.navigation.planner import GoalPlanner
|
||||
|
||||
captured_prompts = []
|
||||
|
||||
def spy_query_llm(**kwargs):
|
||||
captured_prompts.append(kwargs.get("system", ""))
|
||||
return {"response": "scroll down"}
|
||||
|
||||
monkeypatch.setattr(GramAddict.core.navigation.brain, "query_llm", spy_query_llm)
|
||||
|
||||
planner = GoalPlanner("test_user")
|
||||
screen = {
|
||||
"screen_type": ScreenType.HOME_FEED,
|
||||
"available_actions": ["tap home tab", "tap explore tab", "scroll down"],
|
||||
"context": {},
|
||||
}
|
||||
|
||||
planner.plan_next_step("nurture community", screen)
|
||||
|
||||
assert len(captured_prompts) == 1, "Brain was not called"
|
||||
prompt = captured_prompts[0]
|
||||
|
||||
for line in prompt.splitlines():
|
||||
if "available to you right now" in line:
|
||||
assert "tap home tab" not in line, (
|
||||
f"Planner passed no-op 'tap home tab' to Brain on HOME_FEED!\n"
|
||||
f"Line: {line}"
|
||||
)
|
||||
break
|
||||
else:
|
||||
pytest.fail("Could not find 'available to you right now' in Brain prompt")
|
||||
|
||||
Reference in New Issue
Block a user