fix: eliminate thinking-block poisoning + no-op navigation trap

ROOT CAUSE: qwen3.5 (reasoning model) returns response='' with thinking
block containing all reasoning. llm_provider.py line 352 silently
substituted the thinking block as the response via:
  content = raw_response or raw_thinking or ''
The Brain then extracted random actions from the reasoning text.

FIXES:
1. llm_provider.py: Conditional thinking isolation
   - format_json=True (SAE/perception): thinking fallback preserved
   - format_json=False (Brain): thinking NEVER substituted
   - Added think=false for Ollama free-text calls to force direct response

2. planner.py: No-Op Guard strips tab actions that navigate to
   the current screen (e.g. 'tap profile tab' on OWN_PROFILE)

3. test_brain_live.py: Stochastic testing (5 runs, 60% min valid)
   to handle non-deterministic LLM behavior reliably

4. tests/integration/test_llm_provider_pipeline.py: NEW test layer
   mocking at HTTP level (requests.post) to exercise the FULL
   llm_provider → Brain pipeline. This would have caught the
   thinking substitution bug from day one.

Suite: 168 passed, 0 failed
This commit is contained in:
2026-04-29 00:06:23 +02:00
parent ad012b4cd4
commit ac5d5351a6
5 changed files with 397 additions and 18 deletions

View File

@@ -207,3 +207,130 @@ class TestUIChangedFidelity:
from GramAddict.core.perception.screen_identity import ScreenType # noqa: E402
class TestBrainEmptyResponse:
"""Contract: When the LLM returns response='', the Brain must NOT
extract actions from the thinking block. The thinking block is
REASONING, not decisions."""
def test_empty_response_returns_none_not_thinking_extraction(self, monkeypatch):
"""REGRESSION: In the 2026-04-28 23:51 run, the LLM returned response=''
with a thinking block mentioning 'tap profile tab'. The Brain extracted
'tap profile tab' which was a no-op on OWN_PROFILE."""
import GramAddict.core.navigation.brain
def mock_llm(**kwargs):
# The EXACT production failure: response is empty, thinking has actions
return {"response": ""}
monkeypatch.setattr(GramAddict.core.navigation.brain, "query_llm", mock_llm)
result = ask_brain_for_action(
goal="nurture community",
screen_type="OWN_PROFILE",
available_actions=["tap message button", "scroll down", "press back", "tap profile tab"],
explored_actions=set(),
)
# When the LLM gives NO response, the Brain must return None
# to force the planner's structural fallback
assert result is None, (
f"Brain returned '{result}' from an empty LLM response! "
f"It must return None so the planner can use HD Map fallback."
)
def test_whitespace_only_response_treated_as_empty(self, monkeypatch):
"""Response with only whitespace/newlines is effectively empty."""
import GramAddict.core.navigation.brain
def mock_llm(**kwargs):
return {"response": " \n \n "}
monkeypatch.setattr(GramAddict.core.navigation.brain, "query_llm", mock_llm)
result = ask_brain_for_action(
goal="open explore",
screen_type="HOME_FEED",
available_actions=["tap explore tab", "scroll down"],
explored_actions=set(),
)
assert result is None, (
f"Brain returned '{result}' from a whitespace-only response! "
f"Must return None."
)
class TestPlannerNoOpGuard:
"""Contract: The planner must NEVER ask the Brain to execute a tab action
that would navigate to the screen we're already on."""
def test_planner_strips_current_screen_tab_before_brain(self, monkeypatch):
"""On OWN_PROFILE, 'tap profile tab' is a no-op. The planner must
strip it from available_actions before asking the Brain."""
import GramAddict.core.navigation.brain
from GramAddict.core.navigation.planner import GoalPlanner
captured_prompts = []
def spy_query_llm(**kwargs):
captured_prompts.append(kwargs.get("system", ""))
return {"response": "scroll down"}
monkeypatch.setattr(GramAddict.core.navigation.brain, "query_llm", spy_query_llm)
planner = GoalPlanner("test_user")
screen = {
"screen_type": ScreenType.OWN_PROFILE,
"available_actions": ["tap profile tab", "tap home tab", "scroll down", "press back"],
"context": {},
}
planner.plan_next_step("nurture community", screen)
assert len(captured_prompts) == 1, "Brain was not called"
prompt = captured_prompts[0]
for line in prompt.splitlines():
if "available to you right now" in line:
assert "tap profile tab" not in line, (
f"Planner passed no-op action 'tap profile tab' to Brain on OWN_PROFILE!\n"
f"Line: {line}"
)
break
else:
pytest.fail("Could not find 'available to you right now' in Brain prompt")
def test_planner_strips_home_tab_on_home_feed(self, monkeypatch):
"""On HOME_FEED, 'tap home tab' is a no-op."""
import GramAddict.core.navigation.brain
from GramAddict.core.navigation.planner import GoalPlanner
captured_prompts = []
def spy_query_llm(**kwargs):
captured_prompts.append(kwargs.get("system", ""))
return {"response": "scroll down"}
monkeypatch.setattr(GramAddict.core.navigation.brain, "query_llm", spy_query_llm)
planner = GoalPlanner("test_user")
screen = {
"screen_type": ScreenType.HOME_FEED,
"available_actions": ["tap home tab", "tap explore tab", "scroll down"],
"context": {},
}
planner.plan_next_step("nurture community", screen)
assert len(captured_prompts) == 1, "Brain was not called"
prompt = captured_prompts[0]
for line in prompt.splitlines():
if "available to you right now" in line:
assert "tap home tab" not in line, (
f"Planner passed no-op 'tap home tab' to Brain on HOME_FEED!\n"
f"Line: {line}"
)
break
else:
pytest.fail("Could not find 'available to you right now' in Brain prompt")