From 097a5753f9c96cd9ec97b9bcb2a8b0ecff763885 Mon Sep 17 00:00:00 2001 From: Marc Mintel Date: Mon, 27 Apr 2026 22:44:55 +0200 Subject: [PATCH] test: add live LLM test for brain to prevent hallucination regressions --- tests/e2e/test_brain_live.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 tests/e2e/test_brain_live.py diff --git a/tests/e2e/test_brain_live.py b/tests/e2e/test_brain_live.py new file mode 100644 index 0000000..a9de297 --- /dev/null +++ b/tests/e2e/test_brain_live.py @@ -0,0 +1,34 @@ +import pytest +from GramAddict.core.navigation.brain import ask_brain_for_action +from GramAddict.core.perception.screen_identity import ScreenType +import logging + +logger = logging.getLogger(__name__) + +@pytest.mark.live_llm +def test_brain_recommends_scroll_when_trapped(): + """ + Test that the real, live LLM Brain correctly deduces that it should + scroll down when the target element is missing and it's trapped. + """ + goal = "open following list" + screen = "OWN_PROFILE" + available_actions = ["tap profile tab", "tap share button", "press back", "tap reels tab", "tap messages tab", "scroll down", "scroll up"] + explored_nav_actions = {"tap following list"} + + # We query the actual LLM as configured in the environment (e.g. qwen3.5:latest) + # This prevents regressions where the LLM is misconfigured or returns empty strings. + brain_action = ask_brain_for_action( + goal=goal, + screen_type=screen, + available_actions=available_actions, + explored_actions=explored_nav_actions + ) + + logger.info(f"Brain action returned: '{brain_action}'") + + assert brain_action is not None, "Brain LLM returned None. Is the URL/Model configured correctly?" + assert brain_action != "", "Brain LLM returned an empty string." + + # The brain should reasonably choose 'scroll down' to find the missing following list + assert brain_action == "scroll down", f"Expected Brain to choose 'scroll down', but got '{brain_action}'"