test: add live LLM test for brain to prevent hallucination regressions
This commit is contained in:
34
tests/e2e/test_brain_live.py
Normal file
34
tests/e2e/test_brain_live.py
Normal file
@@ -0,0 +1,34 @@
|
||||
import pytest
|
||||
from GramAddict.core.navigation.brain import ask_brain_for_action
|
||||
from GramAddict.core.perception.screen_identity import ScreenType
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@pytest.mark.live_llm
|
||||
def test_brain_recommends_scroll_when_trapped():
|
||||
"""
|
||||
Test that the real, live LLM Brain correctly deduces that it should
|
||||
scroll down when the target element is missing and it's trapped.
|
||||
"""
|
||||
goal = "open following list"
|
||||
screen = "OWN_PROFILE"
|
||||
available_actions = ["tap profile tab", "tap share button", "press back", "tap reels tab", "tap messages tab", "scroll down", "scroll up"]
|
||||
explored_nav_actions = {"tap following list"}
|
||||
|
||||
# We query the actual LLM as configured in the environment (e.g. qwen3.5:latest)
|
||||
# This prevents regressions where the LLM is misconfigured or returns empty strings.
|
||||
brain_action = ask_brain_for_action(
|
||||
goal=goal,
|
||||
screen_type=screen,
|
||||
available_actions=available_actions,
|
||||
explored_actions=explored_nav_actions
|
||||
)
|
||||
|
||||
logger.info(f"Brain action returned: '{brain_action}'")
|
||||
|
||||
assert brain_action is not None, "Brain LLM returned None. Is the URL/Model configured correctly?"
|
||||
assert brain_action != "", "Brain LLM returned an empty string."
|
||||
|
||||
# The brain should reasonably choose 'scroll down' to find the missing following list
|
||||
assert brain_action == "scroll down", f"Expected Brain to choose 'scroll down', but got '{brain_action}'"
|
||||
Reference in New Issue
Block a user