fix: eliminate thinking-block poisoning + no-op navigation trap

ROOT CAUSE: qwen3.5 (reasoning model) returns response='' with thinking
block containing all reasoning. llm_provider.py line 352 silently
substituted the thinking block as the response via:
  content = raw_response or raw_thinking or ''
The Brain then extracted random actions from the reasoning text.

FIXES:
1. llm_provider.py: Conditional thinking isolation
   - format_json=True (SAE/perception): thinking fallback preserved
   - format_json=False (Brain): thinking NEVER substituted
   - Added think=false for Ollama free-text calls to force direct response

2. planner.py: No-Op Guard strips tab actions that navigate to
   the current screen (e.g. 'tap profile tab' on OWN_PROFILE)

3. test_brain_live.py: Stochastic testing (5 runs, 60% min valid)
   to handle non-deterministic LLM behavior reliably

4. tests/integration/test_llm_provider_pipeline.py: NEW test layer
   mocking at HTTP level (requests.post) to exercise the FULL
   llm_provider → Brain pipeline. This would have caught the
   thinking substitution bug from day one.

Suite: 168 passed, 0 failed
This commit is contained in:
2026-04-29 00:06:23 +02:00
parent ad012b4cd4
commit ac5d5351a6
5 changed files with 397 additions and 18 deletions

View File

@@ -0,0 +1,194 @@
"""
LLM Provider Integration Tests — The Missing Layer
====================================================
These tests exercise the ACTUAL llm_provider.py pipeline by mocking at
the HTTP level (requests.post), NOT at the function level (query_llm).
This is the layer that was untested and caused the 2026-04-28 production
failures:
- llm_provider silently substituted thinking blocks as responses
- The Brain then extracted random actions from reasoning text
Contract:
For format_json=False (Brain calls): thinking MUST NOT be substituted
For format_json=True (SAE/perception): thinking CAN be used as fallback
"""
import json
import pytest
from GramAddict.core.llm_provider import query_llm
class TestLLMProviderThinkingIsolation:
"""Contract: The llm_provider must NOT silently substitute thinking
blocks for empty responses in free-text mode."""
def _mock_ollama_response(self, monkeypatch, raw_response: str, raw_thinking: str):
"""Mock requests.post to return a fake Ollama API response."""
import requests
class FakeResponse:
status_code = 200
def __init__(self, resp, think):
self._data = {"response": resp, "thinking": think, "done": True}
def json(self):
return self._data
def raise_for_status(self):
pass
def fake_post(url, **kwargs):
return FakeResponse(raw_response, raw_thinking)
monkeypatch.setattr(requests, "post", fake_post)
def test_empty_response_with_thinking_returns_empty_for_freetext(self, monkeypatch):
"""REGRESSION: When Ollama returns response='' with thinking='...',
format_json=False callers must get '' — NOT the thinking block."""
self._mock_ollama_response(
monkeypatch,
raw_response="",
raw_thinking="I think I should tap profile tab because it would help...",
)
result = query_llm(
url="http://localhost:11434/api/generate",
model="qwen3.5:latest",
prompt="Choose an action",
system="You are an agent",
format_json=False,
)
assert result is not None
content = result["response"]
assert content == "", (
f"llm_provider returned thinking block as response in free-text mode! "
f"Got: '{content[:80]}...'"
)
# Specifically: MUST NOT contain thinking content
assert "tap profile tab" not in content, (
"Thinking block leaked into the response!"
)
def test_empty_response_with_thinking_uses_thinking_for_json(self, monkeypatch):
"""For JSON-expecting callers, falling back to thinking IS correct."""
json_in_thinking = json.dumps({"classification": "obstacle_modal", "confidence": 0.9})
self._mock_ollama_response(
monkeypatch,
raw_response="",
raw_thinking=json_in_thinking,
)
result = query_llm(
url="http://localhost:11434/api/generate",
model="qwen3.5:latest",
prompt="Classify this screen",
system="You are a screen classifier",
format_json=True,
)
assert result is not None
content = result["response"]
parsed = json.loads(content)
assert parsed["classification"] == "obstacle_modal", (
"JSON mode should have extracted from thinking block"
)
def test_normal_response_is_passed_through(self, monkeypatch):
"""When the LLM returns a clean response, it should pass through unchanged."""
self._mock_ollama_response(
monkeypatch,
raw_response="scroll down",
raw_thinking="I considered various options and decided to scroll down.",
)
result = query_llm(
url="http://localhost:11434/api/generate",
model="qwen3.5:latest",
prompt="Choose an action",
system="You are an agent",
format_json=False,
)
assert result is not None
assert result["response"] == "scroll down"
class TestBrainFullPipeline:
"""Integration test: the FULL pipeline from Ollama response → Brain action.
Mocked at the HTTP level, not at the function level."""
def _mock_ollama_response(self, monkeypatch, raw_response: str, raw_thinking: str):
import requests
class FakeResponse:
status_code = 200
def __init__(self, resp, think):
self._data = {"response": resp, "thinking": think, "done": True}
def json(self):
return self._data
def raise_for_status(self):
pass
def fake_post(url, **kwargs):
return FakeResponse(raw_response, raw_thinking)
monkeypatch.setattr(requests, "post", fake_post)
def test_thinking_block_with_empty_response_returns_none(self, monkeypatch):
"""EXACT REPRODUCTION of the 2026-04-28 23:51 production failure.
The LLM returns response='' with thinking mentioning 'tap profile tab'.
The Brain MUST return None (not 'tap profile tab')."""
from GramAddict.core.navigation.brain import ask_brain_for_action
self._mock_ollama_response(
monkeypatch,
raw_response="",
raw_thinking=(
"The user wants to nurture their community. "
"I could tap profile tab but we're already on the profile. "
"Maybe tap messages tab would be better. "
"Actually I think press back is the best option."
),
)
result = ask_brain_for_action(
goal="nurture community",
screen_type="OWN_PROFILE",
available_actions=["tap message button", "scroll down", "press back", "tap profile tab"],
explored_actions=set(),
)
# The Brain MUST return None because the LLM gave no actual response.
# It must NOT extract 'press back' or 'tap profile tab' from the thinking.
assert result is None, (
f"Brain returned '{result}' when LLM response was empty! "
f"The thinking block leaked through llm_provider into the Brain."
)
def test_clean_response_is_correctly_extracted(self, monkeypatch):
"""When the LLM gives a clean response, the full pipeline works."""
from GramAddict.core.navigation.brain import ask_brain_for_action
self._mock_ollama_response(
monkeypatch,
raw_response="scroll down",
raw_thinking="I decided to scroll down to find more content.",
)
result = ask_brain_for_action(
goal="find content",
screen_type="HOME_FEED",
available_actions=["scroll down", "tap explore tab", "press back"],
explored_actions=set(),
)
assert result == "scroll down"