fix: eliminate thinking-block poisoning + no-op navigation trap
ROOT CAUSE: qwen3.5 (reasoning model) returns response='' with thinking block containing all reasoning. llm_provider.py line 352 silently substituted the thinking block as the response via: content = raw_response or raw_thinking or '' The Brain then extracted random actions from the reasoning text. FIXES: 1. llm_provider.py: Conditional thinking isolation - format_json=True (SAE/perception): thinking fallback preserved - format_json=False (Brain): thinking NEVER substituted - Added think=false for Ollama free-text calls to force direct response 2. planner.py: No-Op Guard strips tab actions that navigate to the current screen (e.g. 'tap profile tab' on OWN_PROFILE) 3. test_brain_live.py: Stochastic testing (5 runs, 60% min valid) to handle non-deterministic LLM behavior reliably 4. tests/integration/test_llm_provider_pipeline.py: NEW test layer mocking at HTTP level (requests.post) to exercise the FULL llm_provider → Brain pipeline. This would have caught the thinking substitution bug from day one. Suite: 168 passed, 0 failed
This commit is contained in:
194
tests/integration/test_llm_provider_pipeline.py
Normal file
194
tests/integration/test_llm_provider_pipeline.py
Normal file
@@ -0,0 +1,194 @@
|
||||
"""
|
||||
LLM Provider Integration Tests — The Missing Layer
|
||||
====================================================
|
||||
|
||||
These tests exercise the ACTUAL llm_provider.py pipeline by mocking at
|
||||
the HTTP level (requests.post), NOT at the function level (query_llm).
|
||||
|
||||
This is the layer that was untested and caused the 2026-04-28 production
|
||||
failures:
|
||||
- llm_provider silently substituted thinking blocks as responses
|
||||
- The Brain then extracted random actions from reasoning text
|
||||
|
||||
Contract:
|
||||
For format_json=False (Brain calls): thinking MUST NOT be substituted
|
||||
For format_json=True (SAE/perception): thinking CAN be used as fallback
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from GramAddict.core.llm_provider import query_llm
|
||||
|
||||
|
||||
class TestLLMProviderThinkingIsolation:
|
||||
"""Contract: The llm_provider must NOT silently substitute thinking
|
||||
blocks for empty responses in free-text mode."""
|
||||
|
||||
def _mock_ollama_response(self, monkeypatch, raw_response: str, raw_thinking: str):
|
||||
"""Mock requests.post to return a fake Ollama API response."""
|
||||
import requests
|
||||
|
||||
class FakeResponse:
|
||||
status_code = 200
|
||||
|
||||
def __init__(self, resp, think):
|
||||
self._data = {"response": resp, "thinking": think, "done": True}
|
||||
|
||||
def json(self):
|
||||
return self._data
|
||||
|
||||
def raise_for_status(self):
|
||||
pass
|
||||
|
||||
def fake_post(url, **kwargs):
|
||||
return FakeResponse(raw_response, raw_thinking)
|
||||
|
||||
monkeypatch.setattr(requests, "post", fake_post)
|
||||
|
||||
def test_empty_response_with_thinking_returns_empty_for_freetext(self, monkeypatch):
|
||||
"""REGRESSION: When Ollama returns response='' with thinking='...',
|
||||
format_json=False callers must get '' — NOT the thinking block."""
|
||||
self._mock_ollama_response(
|
||||
monkeypatch,
|
||||
raw_response="",
|
||||
raw_thinking="I think I should tap profile tab because it would help...",
|
||||
)
|
||||
|
||||
result = query_llm(
|
||||
url="http://localhost:11434/api/generate",
|
||||
model="qwen3.5:latest",
|
||||
prompt="Choose an action",
|
||||
system="You are an agent",
|
||||
format_json=False,
|
||||
)
|
||||
|
||||
assert result is not None
|
||||
content = result["response"]
|
||||
assert content == "", (
|
||||
f"llm_provider returned thinking block as response in free-text mode! "
|
||||
f"Got: '{content[:80]}...'"
|
||||
)
|
||||
# Specifically: MUST NOT contain thinking content
|
||||
assert "tap profile tab" not in content, (
|
||||
"Thinking block leaked into the response!"
|
||||
)
|
||||
|
||||
def test_empty_response_with_thinking_uses_thinking_for_json(self, monkeypatch):
|
||||
"""For JSON-expecting callers, falling back to thinking IS correct."""
|
||||
json_in_thinking = json.dumps({"classification": "obstacle_modal", "confidence": 0.9})
|
||||
self._mock_ollama_response(
|
||||
monkeypatch,
|
||||
raw_response="",
|
||||
raw_thinking=json_in_thinking,
|
||||
)
|
||||
|
||||
result = query_llm(
|
||||
url="http://localhost:11434/api/generate",
|
||||
model="qwen3.5:latest",
|
||||
prompt="Classify this screen",
|
||||
system="You are a screen classifier",
|
||||
format_json=True,
|
||||
)
|
||||
|
||||
assert result is not None
|
||||
content = result["response"]
|
||||
parsed = json.loads(content)
|
||||
assert parsed["classification"] == "obstacle_modal", (
|
||||
"JSON mode should have extracted from thinking block"
|
||||
)
|
||||
|
||||
def test_normal_response_is_passed_through(self, monkeypatch):
|
||||
"""When the LLM returns a clean response, it should pass through unchanged."""
|
||||
self._mock_ollama_response(
|
||||
monkeypatch,
|
||||
raw_response="scroll down",
|
||||
raw_thinking="I considered various options and decided to scroll down.",
|
||||
)
|
||||
|
||||
result = query_llm(
|
||||
url="http://localhost:11434/api/generate",
|
||||
model="qwen3.5:latest",
|
||||
prompt="Choose an action",
|
||||
system="You are an agent",
|
||||
format_json=False,
|
||||
)
|
||||
|
||||
assert result is not None
|
||||
assert result["response"] == "scroll down"
|
||||
|
||||
|
||||
class TestBrainFullPipeline:
|
||||
"""Integration test: the FULL pipeline from Ollama response → Brain action.
|
||||
Mocked at the HTTP level, not at the function level."""
|
||||
|
||||
def _mock_ollama_response(self, monkeypatch, raw_response: str, raw_thinking: str):
|
||||
import requests
|
||||
|
||||
class FakeResponse:
|
||||
status_code = 200
|
||||
|
||||
def __init__(self, resp, think):
|
||||
self._data = {"response": resp, "thinking": think, "done": True}
|
||||
|
||||
def json(self):
|
||||
return self._data
|
||||
|
||||
def raise_for_status(self):
|
||||
pass
|
||||
|
||||
def fake_post(url, **kwargs):
|
||||
return FakeResponse(raw_response, raw_thinking)
|
||||
|
||||
monkeypatch.setattr(requests, "post", fake_post)
|
||||
|
||||
def test_thinking_block_with_empty_response_returns_none(self, monkeypatch):
|
||||
"""EXACT REPRODUCTION of the 2026-04-28 23:51 production failure.
|
||||
The LLM returns response='' with thinking mentioning 'tap profile tab'.
|
||||
The Brain MUST return None (not 'tap profile tab')."""
|
||||
from GramAddict.core.navigation.brain import ask_brain_for_action
|
||||
|
||||
self._mock_ollama_response(
|
||||
monkeypatch,
|
||||
raw_response="",
|
||||
raw_thinking=(
|
||||
"The user wants to nurture their community. "
|
||||
"I could tap profile tab but we're already on the profile. "
|
||||
"Maybe tap messages tab would be better. "
|
||||
"Actually I think press back is the best option."
|
||||
),
|
||||
)
|
||||
|
||||
result = ask_brain_for_action(
|
||||
goal="nurture community",
|
||||
screen_type="OWN_PROFILE",
|
||||
available_actions=["tap message button", "scroll down", "press back", "tap profile tab"],
|
||||
explored_actions=set(),
|
||||
)
|
||||
|
||||
# The Brain MUST return None because the LLM gave no actual response.
|
||||
# It must NOT extract 'press back' or 'tap profile tab' from the thinking.
|
||||
assert result is None, (
|
||||
f"Brain returned '{result}' when LLM response was empty! "
|
||||
f"The thinking block leaked through llm_provider into the Brain."
|
||||
)
|
||||
|
||||
def test_clean_response_is_correctly_extracted(self, monkeypatch):
|
||||
"""When the LLM gives a clean response, the full pipeline works."""
|
||||
from GramAddict.core.navigation.brain import ask_brain_for_action
|
||||
|
||||
self._mock_ollama_response(
|
||||
monkeypatch,
|
||||
raw_response="scroll down",
|
||||
raw_thinking="I decided to scroll down to find more content.",
|
||||
)
|
||||
|
||||
result = ask_brain_for_action(
|
||||
goal="find content",
|
||||
screen_type="HOME_FEED",
|
||||
available_actions=["scroll down", "tap explore tab", "press back"],
|
||||
explored_actions=set(),
|
||||
)
|
||||
|
||||
assert result == "scroll down"
|
||||
Reference in New Issue
Block a user