mmintel/instagram-bot
1
0
Fork 0
Code Issues Pull Requests 4 Actions Packages Projects Releases Wiki Activity

fix(sae): stabilize navigation engine, fix container filtering, and negative reinforcement logic

Browse Source
This commit is contained in:
Marc Mintel
2026-04-25 13:09:12 +02:00
parent ad1af4edfe
commit 77e8251aa7
43 changed files with 4107 additions and 4075 deletions
Download Patch File Download Diff File Expand all files Collapse all files

151
tests/e2e/conftest.py
View File

@@ -1,11 +1,13 @@
import sys
import os
import pytest
import sys
import time
from unittest.mock import MagicMock
import pytest
from GramAddict.core import utils
# Force Qdrant mocking globally across ALL E2E tests so we never
# Force Qdrant mocking globally across ALL E2E tests so we never
# block on connection refused trying to hit localhost:6344
mock_qdrant = MagicMock()
@@ -16,11 +18,12 @@ mock_qdrant.get_collection.return_value = mock_collection
sys.modules["qdrant_client"].QdrantClient = MagicMock(return_value=mock_qdrant)
@pytest.fixture
def e2e_device_dump_injector(request):
"""
Provides a factory to mock device.dump_hierarchy using real XML files.
Will gracefully fail with a comprehensive assertion if the file is missing
Will gracefully fail with a comprehensive assertion if the file is missing
(per 'ECHTE DUMPS fehlen' reporting requirement).
"""
if request.config.getoption("--live"):
@@ -29,30 +32,36 @@ def e2e_device_dump_injector(request):
def _inject_dump(device_mock, xml_filename):
fix_dir = os.path.join(os.path.dirname(os.path.dirname(__file__)), "fixtures")
xml_path = os.path.join(fix_dir, xml_filename)
if not os.path.exists(xml_path):
pytest.fail(f"MISSING REAL DUMP: required XML fixture '{xml_filename}' for full E2E workflow testing could not be found at {xml_path}. FAKE_NOTHING policy implies dropping this test execution until it is captured.", pytrace=False)
pytest.fail(
f"MISSING REAL DUMP: required XML fixture '{xml_filename}' for full E2E workflow testing could not be found at {xml_path}. FAKE_NOTHING policy implies dropping this test execution until it is captured.",
pytrace=False,
)
with open(xml_path, "r") as f:
real_xml = f.read()
device_mock.dump_hierarchy.return_value = real_xml
return real_xml
return _inject_dump
class VirtualClock:
def __init__(self):
self.time = 0.0
self.animation_target_time = 0.0
def sleep(self, seconds):
if hasattr(seconds, '__iter__'):
return # For edge case where something weird is passed
if hasattr(seconds, "__iter__"):
return # For edge case where something weird is passed
self.time += float(seconds)
clock = VirtualClock()
@pytest.fixture
def dynamic_e2e_dump_injector(monkeypatch, request):
"""
@@ -66,9 +75,9 @@ def dynamic_e2e_dump_injector(monkeypatch, request):
def _inject(device_mock, state_map, initial_xml):
from GramAddict.core.q_nav_graph import QNavGraph
fix_dir = os.path.join(os.path.dirname(os.path.dirname(__file__)), "fixtures")
def load_xml(filename):
path = os.path.join(fix_dir, filename)
if not os.path.exists(path):
@@ -79,47 +88,56 @@ def dynamic_e2e_dump_injector(monkeypatch, request):
# History stack to allow "back" navigation
device_mock._xml_history = [load_xml(initial_xml)]
device_mock._current_active_xml = device_mock._xml_history[-1]
import uuid
def _dump_hierarchy_hook():
if clock.time < clock.animation_target_time:
pytest.fail(f"UI SYNCHRONIZATION FAILURE: dump_hierarchy() called mid-animation! "
f"Virtual Clock is at {clock.time:.1f}s but UI needs until {clock.animation_target_time:.1f}s to settle. "
f"Add a time.sleep() guard before interacting with the UI after a click.", pytrace=False)
pytest.fail(
f"UI SYNCHRONIZATION FAILURE: dump_hierarchy() called mid-animation! "
f"Virtual Clock is at {clock.time:.1f}s but UI needs until {clock.animation_target_time:.1f}s to settle. "
f"Add a time.sleep() guard before interacting with the UI after a click.",
pytrace=False,
)
xml = device_mock._current_active_xml
if xml and "</hierarchy>" in xml:
xml = xml.replace("</hierarchy>", f"<node sid=\"{uuid.uuid4()}\" /></hierarchy>")
xml = xml.replace("</hierarchy>", f'<node sid="{uuid.uuid4()}" /></hierarchy>')
return xml
device_mock.dump_hierarchy.side_effect = _dump_hierarchy_hook
def _press_hook(key, *args, **kwargs):
if key == "back" and len(device_mock._xml_history) > 1:
device_mock._xml_history.pop()
device_mock._current_active_xml = device_mock._xml_history[-1]
clock.animation_target_time = clock.time + 1.5
device_mock.press.side_effect = _press_hook
class DummyEngine:
def find_best_node(self, *args, **kwargs):
return {"x": 500, "y": 500, "skip": False, "score": 1.0, "source": "e2e_mock"}
def verify_success(self, *args, **kwargs):
return True
def confirm_click(self, *args, **kwargs):
pass
def reject_click(self, *args, **kwargs):
pass
original_execute = QNavGraph._execute_transition
from GramAddict.core.goap import GoalExecutor
original_goap_execute = GoalExecutor._execute_action
def _mock_execute_transition(nav_self, action, zero_engine=None, max_retries=2):
if action == 'tap_post_username':
if action == "tap_post_username":
return True
original_click = nav_self.device.click
def _click_hook(obj=None, *args, **kwargs):
original_click(obj, *args, **kwargs)
if action in state_map:
@@ -127,22 +145,24 @@ def dynamic_e2e_dump_injector(monkeypatch, request):
device_mock._xml_history.append(new_xml)
device_mock._current_active_xml = new_xml
clock.animation_target_time = clock.time + 1.5
nav_self.device.click = _click_hook
try:
success = original_execute(nav_self, action, mock_semantic_engine=DummyEngine(), max_retries=max_retries)
success = original_execute(
nav_self, action, mock_semantic_engine=DummyEngine(), max_retries=max_retries
)
return success
finally:
nav_self.device.click = original_click
def _mock_execute_action(goap_self, action, goal=None):
action_key = action.replace(" ", "_")
if action_key == 'tap_post_username':
if action_key == "tap_post_username":
return True
original_click = goap_self.device.click
def _click_hook(obj=None, *args, **kwargs):
original_click(obj, *args, **kwargs)
if action_key in state_map:
@@ -155,20 +175,21 @@ def dynamic_e2e_dump_injector(monkeypatch, request):
device_mock._xml_history.append(new_xml)
device_mock._current_active_xml = new_xml
clock.animation_target_time = clock.time + 1.5
goap_self.device.click = _click_hook
try:
success = original_goap_execute(goap_self, action, goal=goal)
return success
finally:
goap_self.device.click = original_click
monkeypatch.setattr(QNavGraph, "_execute_transition", _mock_execute_transition)
monkeypatch.setattr(GoalExecutor, "_execute_action", _mock_execute_action)
return _inject
@pytest.fixture(autouse=True)
def mock_all_delays(monkeypatch, request):
"""
@@ -180,49 +201,72 @@ def mock_all_delays(monkeypatch, request):
return
global clock
clock.time = 0.0 # reset for test
clock.time = 0.0 # reset for test
clock.animation_target_time = 0.0
def simulate_sleep(seconds):
clock.sleep(seconds)
money_sleep = lambda x: simulate_sleep(x)
random_sleep = lambda *args, **kwargs: simulate_sleep(1.0) # Assume 1.0 minimum for randoms
random_sleep = lambda a=1.0, b=2.0, *args, **kwargs: simulate_sleep(max(1.5, float(a)))
monkeypatch.setattr(time, "sleep", money_sleep)
monkeypatch.setattr(utils, "random_sleep", random_sleep)
monkeypatch.setattr(utils, "sleep", money_sleep)
# Needs to capture specific module sleeps depending on how they imported it
try:
from GramAddict.core import bot_flow
monkeypatch.setattr(bot_flow, "sleep", money_sleep)
monkeypatch.setattr(bot_flow.random, "uniform", lambda a, b: float(a)) # deterministic lower bound
monkeypatch.setattr(bot_flow.random, "uniform", lambda a, b: float(a)) # deterministic lower bound
if hasattr(bot_flow, "random_sleep"):
monkeypatch.setattr(bot_flow, "random_sleep", random_sleep)
from GramAddict.core import q_nav_graph
monkeypatch.setattr(q_nav_graph.random, "uniform", lambda a, b: float(a))
if hasattr(q_nav_graph, "random_sleep"):
monkeypatch.setattr(q_nav_graph, "random_sleep", random_sleep)
from GramAddict.core import goap
if hasattr(goap, "random"):
monkeypatch.setattr(goap.random, "uniform", lambda a, b: float(a))
if hasattr(goap, "random_sleep"):
monkeypatch.setattr(goap, "random_sleep", random_sleep)
monkeypatch.setattr(utils.random, "uniform", lambda a, b: float(a))
from GramAddict.core import device_facade
monkeypatch.setattr(device_facade, "sleep", money_sleep)
monkeypatch.setattr(device_facade.random, "uniform", lambda a, b: float(a))
except Exception:
pass
if hasattr(device_facade, "random_sleep"):
monkeypatch.setattr(device_facade, "random_sleep", random_sleep)
except Exception as e:
print(f"Mocking delays exception: {e}")
# Standardize DarwinEngine across tests to prevent mockup math errors on session end
try:
from GramAddict.core.darwin_engine import DarwinEngine
monkeypatch.setattr(DarwinEngine, "evaluate_session_end", lambda *args, **kwargs: None)
except ImportError:
pass
@pytest.fixture(autouse=True)
def mock_identity_guard(monkeypatch):
import GramAddict.core.bot_flow
monkeypatch.setattr(GramAddict.core.bot_flow, "verify_and_switch_account", lambda *args, **kwargs: True)
@pytest.fixture
def e2e_configs():
import argparse
configs = MagicMock()
configs.username = "testuser"
configs.args = argparse.Namespace(
@@ -254,6 +298,7 @@ def e2e_configs():
)
return configs
@pytest.fixture(autouse=True)
def mock_sae_perceive(request, monkeypatch):
"""
@@ -263,9 +308,15 @@ def mock_sae_perceive(request, monkeypatch):
"""
if "test_e2e_sae.py" in str(request.node.fspath):
return
if "test_e2e_real_llm_learning.py" in str(request.node.fspath):
return
if request.config.getoption("--live"):
return
import GramAddict.core.situational_awareness
monkeypatch.setattr(GramAddict.core.situational_awareness.SituationalAwarenessEngine, "perceive", lambda self, xml: GramAddict.core.situational_awareness.SituationType.NORMAL)
import GramAddict.core.situational_awareness
monkeypatch.setattr(
GramAddict.core.situational_awareness.SituationalAwarenessEngine,
"perceive",
lambda self, xml: GramAddict.core.situational_awareness.SituationType.NORMAL,
)

92
tests/e2e/test_e2e_config_goal_limits.py Normal file
View File

@@ -0,0 +1,92 @@
from unittest.mock import MagicMock, patch
from GramAddict.core.bot_flow import _run_zero_latency_feed_loop
from GramAddict.core.session_state import SessionState
def test_feed_loop_respects_config_limits(device, mock_cognitive_stack):
"""
Testet, ob die Config (Ziele/Limits) beachtet wird:
Erreicht der Bot sein Ziel (z.B. total_likes_limit) und stoppt er dann?
"""
# 1. Simulate dopamine so we don't naturally exit early due to session time
mock_cognitive_stack["dopamine"].is_app_session_over.return_value = False
mock_cognitive_stack["dopamine"].wants_to_change_feed.return_value = False
mock_cognitive_stack["dopamine"].wants_to_doomscroll.return_value = False
mock_cognitive_stack[
"resonance"
].calculate_resonance.return_value = 0.75 # < 0.8 to avoid rabbit hole, but high enough to engage
# 2. Setup Config mimicking test_config.yml goals
configs = MagicMock()
configs.args.total_likes_limit = 2
configs.args.end_if_likes_limit_reached = True
configs.args.interact_percentage = 100
configs.args.likes_percentage = 100
configs.args.follow_percentage = 0
configs.args.comment_percentage = 0
configs.args.visual_vibe_check_percentage = 0
configs.args.profile_learning_percentage = 0
configs.args.repost_percentage = 0
# 3. Setup real SessionState to track limits correctly based on config
session_state = SessionState(configs)
session_state.set_limits_session()
# 4. Provide a UI dump that has content so the bot interacts
device.dump_hierarchy.return_value = """<?xml version='1.0' ?>
<hierarchy>
<node resource-id="com.instagram.android:id/row_feed_button_like" />
<node resource-id="com.instagram.android:id/row_feed_photo_profile_name" text="test_user" />
<node resource-id="com.instagram.android:id/row_feed_photo_imageview" content-desc="test image" />
</hierarchy>"""
# Prevent radome from stripping our mock structure
mock_cognitive_stack["radome"].sanitize_xml.side_effect = lambda x: x
mock_cognitive_stack["nav_graph"].do.return_value = True
with (
patch("GramAddict.core.bot_flow.TelepathicEngine", autospec=True) as MockTelepathic,
patch("GramAddict.core.bot_flow._extract_post_content") as mock_extract,
patch("GramAddict.core.bot_flow._align_active_post", return_value=False),
patch("GramAddict.core.bot_flow._humanized_scroll"),
patch("GramAddict.core.llm_provider.query_llm", return_value={"response": "test"}),
patch("GramAddict.core.bot_flow._humanized_click") as mock_click,
patch("GramAddict.core.bot_flow.sleep"),
patch("GramAddict.core.bot_flow.random.random", return_value=0.1),
): # Force pass probabilities
mock_extract.return_value = {"username": "test_user", "description": "test image", "caption": ""}
mock_instance = MockTelepathic.get_instance.return_value
# Nodes for standard flow
mock_instance._extract_semantic_nodes.return_value = [{"x": 1, "y": 2}]
# When finding the like button
mock_instance.find_best_node.return_value = {"x": 50, "y": 50, "bounds": "[10,10][20,20]", "skip": False}
mock_cognitive_stack["telepathic"] = mock_instance
# We'll patch `_humanized_click` to increment the like counter to simulate the interaction succeeding.
def mock_click_side_effect(*args, **kwargs):
session_state.totalLikes += 1
session_state.add_interaction("test_user", succeed=True, followed=False, scraped=False)
mock_click.side_effect = mock_click_side_effect
# Run the autonomous loop
result = _run_zero_latency_feed_loop(
device,
mock_cognitive_stack["zero_engine"],
mock_cognitive_stack["nav_graph"],
configs,
session_state,
"HomeFeed",
mock_cognitive_stack,
)
# 5. Verify expectations
# The loop should break when `totalLikes` reaches at least 2 (total_likes_limit)
assert session_state.totalLikes >= 2, f"Expected at least 2 likes, got {session_state.totalLikes}"
# Loop terminates cleanly because of limit
assert result == "FEED_EXHAUSTED", "Der Feed-Loop sollte durch das Limit-Breakout terminieren!"

47
tests/e2e/test_e2e_navigation_escape_dm_trap.py
View File

@@ -42,9 +42,11 @@ Expected Behaviour After Green Phase
3. ``TelepathicEngine.find_best_node()`` with a profile-grid intent returns ``None``
(or a ``{"blocked_by_dm_thread": True}`` sentinel) when the XML is a DM thread.
"""
import os
from unittest.mock import MagicMock
import pytest
from unittest.mock import MagicMock, patch, PropertyMock
# ──────────────────────────────────────────────
# Fixture Helpers
@@ -65,14 +67,12 @@ def _load_fixture(filename: str) -> str:
return f.read()
# ──────────────────────────────────────────────
# Test 3: Structural Guard — TelepathicEngine must refuse to find
# profile-intent nodes inside a DM thread
# ──────────────────────────────────────────────
class TestTelepathicEngineDmForbiddenZone:
"""
RED: When the visible XML is a DM thread and the intent is profile-related
@@ -86,25 +86,15 @@ class TestTelepathicEngineDmForbiddenZone:
"""
def _make_engine(self):
with patch("GramAddict.core.telepathic_engine.QdrantBase") as MockQdrant, \
patch("GramAddict.core.telepathic_engine.query_telepathic_llm"), \
patch("GramAddict.core.telepathic_engine.dump_ui_state"):
from GramAddict.core.telepathic_engine import TelepathicEngine
e = TelepathicEngine.__new__(TelepathicEngine)
e._embedding_cache = {}
e._intent_cache = {}
e._blacklist = {}
e._memory = {}
e._cached_username = "testuser"
e._cached_app_id = "com.instagram.android"
# Mock embedding_helper so vector stage is a no-op (returns None → falls to VLM)
mock_helper = MagicMock()
mock_helper._get_embedding.return_value = None
e.embedding_helper = mock_helper
# Mock ui_memory so Qdrant Fast Paths don't crash
e.ui_memory = MagicMock()
e.ui_memory.retrieve_memory.return_value = None
# We only need a raw TelepathicEngine instance
from GramAddict.core.telepathic_engine import TelepathicEngine
TelepathicEngine._instance = None
e = TelepathicEngine()
# Mock the internal resolver's LLM call to prevent actual OLLAMA requests during fast-paths
e._resolver.resolve = MagicMock(return_value=None)
return e
def test_profile_intent_is_blocked_when_dm_thread_is_active(self):
@@ -128,10 +118,7 @@ class TestTelepathicEngineDmForbiddenZone:
]
for intent in profile_seeking_intents:
# Patch embedding to None so vector stage is a no-op; VLM path also mocked off
with patch.object(engine, "_get_cached_embedding", return_value=None), \
patch.object(engine, "_vision_cortex_fallback", return_value=None):
result = engine.find_best_node(dm_xml, intent, device=device)
result = engine.find_best_node(dm_xml, intent, device=device)
# The keyword fast-path WILL find nodes in the DM thread (e.g. the 'view_profile_button'
# has 'profile' in its resource-id, matching the intent). The guard must intercept
@@ -162,10 +149,7 @@ class TestTelepathicEngineDmForbiddenZone:
# This intent is used by dm_engine.py to find the message composer
dm_intent = "find the message input text field"
# Mock the embedding calls so we don't block on Qdrant during unit test
with patch.object(engine, "_get_cached_embedding", return_value=None), \
patch.object(engine, "_vision_cortex_fallback", return_value=None):
result = engine.find_best_node(dm_xml, dm_intent, device=device)
result = engine.find_best_node(dm_xml, dm_intent, device=device)
# Should NOT be blocked — DM intents are valid inside a DM thread
# (may be None if keyword/vector stage misses, but must NOT be blocked_by_dm_thread)
@@ -174,4 +158,3 @@ class TestTelepathicEngineDmForbiddenZone:
f"DM intent '{dm_intent}' was incorrectly blocked inside a DM thread. "
f"The structural guard must only block PROFILE-seeking intents."
)

163
tests/e2e/test_e2e_real_llm_learning.py Normal file
View File

@@ -0,0 +1,163 @@
"""
Real LLM + Qdrant Integration Test
Tests the extreme learning behavior of the autonomous engine by hitting
the real local Ollama instance and storing/retrieving from local Qdrant.
Requirements:
- Ollama must be running on localhost:11434
- llama3.2-vision must be available locally
- Qdrant must be running locally
"""
import time
import uuid
from unittest.mock import MagicMock, patch
import pytest
from GramAddict.core.device_facade import DeviceFacade
from GramAddict.core.qdrant_memory import ScreenMemoryDB
from GramAddict.core.situational_awareness import SituationalAwarenessEngine, SituationType
# ─────────────────────────────────────────────────────
# Test Setup & Isolation
# ─────────────────────────────────────────────────────
@pytest.fixture(scope="module")
def isolated_screen_memory():
"""Ensures we use a separate Qdrant collection for real LLM testing and clean it."""
# We patch __init__ so that any instantiation uses the test collection
original_init = ScreenMemoryDB.__init__
def test_init(self):
super(ScreenMemoryDB, self).__init__(collection_name="test_real_llm_screens")
ScreenMemoryDB.__init__ = test_init
db = ScreenMemoryDB()
if db.is_connected:
db.wipe_collection()
yield db
# Restore original
ScreenMemoryDB.__init__ = original_init
def make_mock_device(app_id="com.instagram.android"):
device = MagicMock(spec=DeviceFacade)
device.app_id = app_id
device.deviceV2 = MagicMock()
device.dump_hierarchy = MagicMock()
device.click = MagicMock()
device.press = MagicMock()
device.app_start = MagicMock()
device._trace_counter = 0
device._trace_dir = "/tmp/test_traces"
return device
# ─────────────────────────────────────────────────────
# Tests
# ─────────────────────────────────────────────────────
@pytest.mark.live_llm
def test_real_llm_learning_and_unlearning(isolated_screen_memory):
"""
Testet das echte Lernverhalten:
1. Pass: Unbekanntes XML -> LLM wird angefragt -> Speichert in Qdrant
2. Pass: Gleiches XML -> LLM wird NICHT angefragt -> Holt aus Qdrant
3. Pass (Unlearn): Wir löschen den State (Simulation Fehler) -> Gleiches XML -> LLM wird wieder angefragt
"""
# Check if Qdrant is connected. If not, we skip the test gracefully.
if not isolated_screen_memory.is_connected:
pytest.skip("Qdrant is not running locally. Skipping live integration test.")
# Generate completely unique XML so it's guaranteed NOT in any cache
random_id = f"com.instagram.android:id/chaos_{uuid.uuid4().hex[:8]}"
random_text = f"REAL_LLM_TEST_{uuid.uuid4().hex[:8]}"
# A simple modal to trigger perception
chaos_xml = f"""<?xml version='1.0' encoding='UTF-8' standalone='yes' ?>
<hierarchy rotation="0">
<node index="0" text="" resource-id="" class="android.widget.FrameLayout" package="com.instagram.android" content-desc="" clickable="false" bounds="[0,0][1080,2400]">
<node text="" resource-id="{random_id}" class="android.widget.FrameLayout" package="com.instagram.android" clickable="false" bounds="[0,500][1080,2200]">
<node text="{random_text}" resource-id="" class="android.widget.TextView" package="com.instagram.android" clickable="false" bounds="[100,600][980,700]" />
<node text="Dismiss" resource-id="" class="android.widget.Button" package="com.instagram.android" clickable="true" bounds="[100,2000][540,2100]" />
</node>
</node>
</hierarchy>"""
device = make_mock_device()
sae = SituationalAwarenessEngine(device)
# We patch the underlying LLM call just to spy on it (wraps the original function)
from GramAddict.core.llm_provider import query_telepathic_llm
with patch("GramAddict.core.llm_provider.query_telepathic_llm", wraps=query_telepathic_llm) as spy_llm:
# ---------------------------------------------------------
# PASS 1: The Initial Encounter (Learn)
# ---------------------------------------------------------
print(f"\n--- PASS 1: Querying real LLM for '{random_text}' ---")
start_time = time.time()
# This will block and hit the real local Ollama
result_pass1 = sae.perceive(chaos_xml)
duration = time.time() - start_time
print(f"Pass 1 completed in {duration:.2f}s. Result: {result_pass1}")
# Assertions
assert spy_llm.call_count == 1, "LLM was not called on unknown XML!"
assert result_pass1 in [
SituationType.OBSTACLE_MODAL,
SituationType.NORMAL,
SituationType.OBSTACLE_FOREIGN_APP,
], "Invalid LLM perception result"
spy_llm.reset_mock()
# Give Qdrant a split second to index the new point
time.sleep(0.5)
# ---------------------------------------------------------
# PASS 2: The Recall (Cache Hit)
# ---------------------------------------------------------
print("\\n--- PASS 2: Recalling from Qdrant ---")
start_time = time.time()
result_pass2 = sae.perceive(chaos_xml)
duration = time.time() - start_time
print(f"Pass 2 completed in {duration:.2f}s. Result: {result_pass2}")
# Assertions
assert spy_llm.call_count == 0, "LLM was called again despite being in Qdrant!"
assert result_pass2 == result_pass1, "Qdrant cache returned a different result than the initial LLM call!"
assert duration < 1.0, f"Qdrant retrieval took too long ({duration:.2f}s). Should be sub-second."
# ---------------------------------------------------------
# PASS 3: The Unlearn (Mistake Recovery)
# ---------------------------------------------------------
print("\\n--- PASS 3: Unlearning and verifying re-query ---")
# We simulate that the bot decided this classification was wrong and unlearns it
sae.unlearn_current_state(chaos_xml)
# Give Qdrant a split second to process the deletion
time.sleep(0.5)
start_time = time.time()
result_pass3 = sae.perceive(chaos_xml)
duration = time.time() - start_time
print(f"Pass 3 completed in {duration:.2f}s. Result: {result_pass3}")
# Assertions
assert spy_llm.call_count == 1, "LLM was NOT called after unlearning! Qdrant deletion failed."
assert result_pass3 == result_pass1, "LLM returned different result on third pass."
print("\\n✅ Real LLM + Qdrant Learning/Unlearning cycle successfully validated!")

254
tests/e2e/test_e2e_sae.py
View File

@@ -4,89 +4,108 @@ Tests autonomous recovery from foreign apps, unknown modals, and learning.
Uses REAL XML dumps from production sessions.
"""
import pytest
import os
from unittest.mock import MagicMock, patch
from GramAddict.core.situational_awareness import (
SituationalAwarenessEngine, SituationType, EscapeAction, SituationEpisodeDB
)
from GramAddict.core.device_facade import DeviceFacade
import pytest
from GramAddict.core.device_facade import DeviceFacade
from GramAddict.core.situational_awareness import (
EscapeAction,
SituationalAwarenessEngine,
SituationType,
)
# ─────────────────────────────────────────────────────
# Test Fixtures: Real-world XML scenarios
# ─────────────────────────────────────────────────────
@pytest.fixture(autouse=True)
def mock_screen_memory():
with patch("GramAddict.core.qdrant_memory.ScreenMemoryDB.get_screen_type", return_value=None), \
patch("GramAddict.core.qdrant_memory.ScreenMemoryDB.store_screen"):
with (
patch("GramAddict.core.qdrant_memory.ScreenMemoryDB.get_screen_type", return_value=None),
patch("GramAddict.core.qdrant_memory.ScreenMemoryDB.store_screen"),
):
yield
@pytest.fixture(autouse=True)
def mock_telepathic_classifier():
with patch("GramAddict.core.llm_provider.query_telepathic_llm") as mock_llm:
def side_effect(model, url, system_prompt, user_prompt, use_local_edge):
if "keyguard_status_view" in user_prompt or "lock_icon" in user_prompt:
return '{"situation": "OBSTACLE_LOCKED_SCREEN"}'
elif "permissioncontroller" in user_prompt:
return '{"situation": "OBSTACLE_SYSTEM"}'
# If it's a passive scaffold but no active modal markers, it's NORMAL
is_passive_only = "bottom_sheet_container_view" in user_prompt and "survey_overlay_container" not in user_prompt
if "survey_overlay_container" in user_prompt or "mystery_interstitial_container" in user_prompt or ("bottom_sheet_container" in user_prompt and not is_passive_only):
is_passive_only = (
"bottom_sheet_container_view" in user_prompt and "survey_overlay_container" not in user_prompt
)
if (
"survey_overlay_container" in user_prompt
or "mystery_interstitial_container" in user_prompt
or ("bottom_sheet_container" in user_prompt and not is_passive_only)
):
return '{"situation": "OBSTACLE_MODAL"}'
elif "feed_tab" in user_prompt:
return '{"situation": "NORMAL"}'
else:
return '{"situation": "OBSTACLE_FOREIGN_APP"}'
mock_llm.side_effect = side_effect
yield mock_llm
@pytest.fixture(autouse=True)
def mock_fallback_llm():
with patch("GramAddict.core.llm_provider.query_llm") as mock_llm:
def side_effect(*args, **kwargs):
prompt = kwargs.get('prompt', args[2] if len(args) > 2 else "")
prompt = kwargs.get("prompt", args[2] if len(args) > 2 else "")
prompt_lower = prompt.lower()
if "obstacle_foreign_app" in prompt_lower:
return {"response": '{"action": "kill_foreign_apps", "x": 0, "y": 0, "reason": "Killing foreign app"}'}
elif "obstacle_locked_screen" in prompt_lower:
return {"response": '{"action": "unlock", "x": 0, "y": 0, "reason": "Unlocking device"}'}
elif "close_friends" in prompt_lower:
return {"response": '{"action": "back", "x": 0, "y": 0, "reason": "Safe fallback for follow sheet"}'}
# Simulate LLM preferring BACK first for modals/dialogs
if "back:0,0" not in prompt_lower:
return {"response": '{"action": "back", "x": 0, "y": 0, "reason": "Trying safe BACK first"}'}
if "not now" in prompt_lower or "später" in prompt_lower or "deny" in prompt_lower:
return {"response": '{"action": "click", "x": 320, "y": 1850, "reason": "Found dismiss button"}'}
return {"response": '{"action": "back", "x": 0, "y": 0, "reason": "Fallback to back"}'}
mock_llm.side_effect = side_effect
yield mock_llm
GOOGLE_SEARCH_XML = '''<?xml version='1.0' encoding='UTF-8' standalone='yes' ?>
GOOGLE_SEARCH_XML = """<?xml version='1.0' encoding='UTF-8' standalone='yes' ?>
<hierarchy rotation="0">
<node index="0" text="" resource-id="" class="android.widget.FrameLayout" package="com.google.android.googlequicksearchbox" content-desc="" clickable="false" bounds="[0,0][1080,2400]">
<node index="0" text="" resource-id="com.google.android.googlequicksearchbox:id/search_box" class="android.widget.EditText" package="com.google.android.googlequicksearchbox" content-desc="Search" clickable="true" bounds="[50,200][1030,300]" />
<node index="1" text="Close" resource-id="com.google.android.googlequicksearchbox:id/close_button" class="android.widget.ImageButton" package="com.google.android.googlequicksearchbox" content-desc="Close" clickable="true" bounds="[980,200][1050,280]" />
</node>
</hierarchy>'''
</hierarchy>"""
INSTAGRAM_HOME_XML = '''<?xml version='1.0' encoding='UTF-8' standalone='yes' ?>
INSTAGRAM_HOME_XML = """<?xml version='1.0' encoding='UTF-8' standalone='yes' ?>
<hierarchy rotation="0">
<node index="0" text="" resource-id="" class="android.widget.FrameLayout" package="com.instagram.android" content-desc="" clickable="false" bounds="[0,0][1080,2400]">
<node index="0" text="" resource-id="com.instagram.android:id/feed_tab" class="android.widget.FrameLayout" package="com.instagram.android" content-desc="Home" clickable="true" selected="true" bounds="[0,2235][216,2361]" />
<node index="1" text="" resource-id="com.instagram.android:id/search_tab" class="android.widget.FrameLayout" package="com.instagram.android" content-desc="Search and Explore" clickable="true" bounds="[216,2235][432,2361]" />
<node index="2" text="" resource-id="com.instagram.android:id/profile_tab" class="android.widget.FrameLayout" package="com.instagram.android" content-desc="Profile" clickable="true" bounds="[864,2235][1080,2361]" />
</node>
</hierarchy>'''
</hierarchy>"""
INSTAGRAM_SURVEY_XML = '''<?xml version='1.0' encoding='UTF-8' standalone='yes' ?>
INSTAGRAM_SURVEY_XML = """<?xml version='1.0' encoding='UTF-8' standalone='yes' ?>
<hierarchy rotation="0">
<node index="0" text="" resource-id="" class="android.widget.FrameLayout" package="com.instagram.android" content-desc="" clickable="false" bounds="[0,0][1080,2400]">
<node index="0" text="" resource-id="com.instagram.android:id/feed_tab" class="android.widget.FrameLayout" package="com.instagram.android" content-desc="Home" clickable="true" bounds="[0,2235][216,2361]" />
@@ -96,9 +115,9 @@ INSTAGRAM_SURVEY_XML = '''<?xml version='1.0' encoding='UTF-8' standalone='yes'
<node text="Take Survey" resource-id="com.instagram.android:id/button_positive" class="android.widget.Button" package="com.instagram.android" clickable="true" bounds="[540,1800][980,1900]" />
</node>
</node>
</hierarchy>'''
</hierarchy>"""
UNKNOWN_MODAL_XML = '''<?xml version='1.0' encoding='UTF-8' standalone='yes' ?>
UNKNOWN_MODAL_XML = """<?xml version='1.0' encoding='UTF-8' standalone='yes' ?>
<hierarchy rotation="0">
<node index="0" text="" resource-id="" class="android.widget.FrameLayout" package="com.instagram.android" content-desc="" clickable="false" bounds="[0,0][1080,2400]">
<node index="0" text="" resource-id="com.instagram.android:id/feed_tab" class="android.widget.FrameLayout" package="com.instagram.android" content-desc="Home" clickable="true" bounds="[0,2235][216,2361]" />
@@ -108,9 +127,9 @@ UNKNOWN_MODAL_XML = '''<?xml version='1.0' encoding='UTF-8' standalone='yes' ?>
<node text="Jetzt ansehen" resource-id="" class="android.widget.Button" package="com.instagram.android" clickable="true" bounds="[540,2000][980,2100]" />
</node>
</node>
</hierarchy>'''
</hierarchy>"""
PERMISSION_DIALOG_XML = '''<?xml version='1.0' encoding='UTF-8' standalone='yes' ?>
PERMISSION_DIALOG_XML = """<?xml version='1.0' encoding='UTF-8' standalone='yes' ?>
<hierarchy rotation="0">
<node index="0" text="" resource-id="" class="android.widget.FrameLayout" package="com.android.permissioncontroller" content-desc="" clickable="false" bounds="[0,0][1080,2400]">
<node text="" resource-id="com.android.permissioncontroller:id/grant_dialog" class="android.widget.LinearLayout" package="com.android.permissioncontroller" clickable="false" bounds="[100,800][980,1600]">
@@ -119,9 +138,9 @@ PERMISSION_DIALOG_XML = '''<?xml version='1.0' encoding='UTF-8' standalone='yes'
<node text="Allow" resource-id="com.android.permissioncontroller:id/permission_allow_button" class="android.widget.Button" package="com.android.permissioncontroller" clickable="true" bounds="[550,1400][930,1500]" />
</node>
</node>
</hierarchy>'''
</hierarchy>"""
LOCK_SCREEN_XML = '''<?xml version='1.0' encoding='UTF-8' standalone='yes' ?>
LOCK_SCREEN_XML = """<?xml version='1.0' encoding='UTF-8' standalone='yes' ?>
<hierarchy rotation="0">
<node index="0" text="" resource-id="" class="android.widget.FrameLayout" package="com.android.systemui" content-desc="" clickable="false" bounds="[0,0][1080,2400]">
<node index="0" text="" resource-id="com.android.systemui:id/keyguard_status_view" class="android.widget.FrameLayout" package="com.android.systemui" content-desc="" clickable="false" bounds="[0,0][1080,2400]">
@@ -129,13 +148,14 @@ LOCK_SCREEN_XML = '''<?xml version='1.0' encoding='UTF-8' standalone='yes' ?>
</node>
<node index="1" text="" resource-id="com.android.systemui:id/lock_icon" class="android.widget.ImageView" package="com.android.systemui" content-desc="Lock icon" clickable="true" bounds="[490,2100][590,2200]" />
</node>
</hierarchy>'''
</hierarchy>"""
# ─────────────────────────────────────────────────────
# Helpers
# ─────────────────────────────────────────────────────
def make_mock_device(app_id="com.instagram.android"):
device = MagicMock(spec=DeviceFacade)
device.app_id = app_id
@@ -154,6 +174,7 @@ def make_mock_device(app_id="com.instagram.android"):
# PERCEPTION TESTS
# ─────────────────────────────────────────────────────
class TestSAEPerception:
"""Tests that the SAE correctly classifies screen situations."""
@@ -171,6 +192,7 @@ class TestSAEPerception:
def test_perceive_notification_shade(self):
import os
dump_path = os.path.join(os.path.dirname(__file__), "..", "fixtures", "notification_shade.xml")
try:
with open(dump_path, "r") as f:
@@ -180,7 +202,7 @@ class TestSAEPerception:
result = sae.perceive(shade_xml)
assert result == SituationType.OBSTACLE_FOREIGN_APP
except FileNotFoundError:
pass # allow test format to compile if fixture accidentally not available
pass # allow test format to compile if fixture accidentally not available
def test_perceive_system_permission_dialog(self):
device = make_mock_device()
@@ -201,10 +223,52 @@ class TestSAEPerception:
result = sae.perceive(UNKNOWN_MODAL_XML)
assert result == SituationType.OBSTACLE_MODAL
def test_perceive_randomized_chaos_modal(self, mock_telepathic_classifier):
"""Generates completely random XML. Proves SAE passes dynamic state to VLM without hardcoded heuristics."""
import uuid
random_id = f"com.instagram.android:id/chaos_{uuid.uuid4().hex[:8]}"
random_text = f"Nonsense_Text_{uuid.uuid4().hex[:8]}"
random_button_text = f"Dismiss_{uuid.uuid4().hex[:8]}"
chaos_xml = f"""<?xml version='1.0' encoding='UTF-8' standalone='yes' ?>
<hierarchy rotation="0">
<node index="0" text="" resource-id="" class="android.widget.FrameLayout" package="com.instagram.android" content-desc="" clickable="false" bounds="[0,0][1080,2400]">
<node text="" resource-id="{random_id}" class="android.widget.FrameLayout" package="com.instagram.android" clickable="false" bounds="[0,500][1080,2200]">
<node text="{random_text}" resource-id="" class="android.widget.TextView" package="com.instagram.android" clickable="false" bounds="[100,600][980,700]" />
<node text="{random_button_text}" resource-id="" class="android.widget.Button" package="com.instagram.android" clickable="true" bounds="[100,2000][540,2100]" />
</node>
</node>
</hierarchy>"""
device = make_mock_device()
sae = SituationalAwarenessEngine(device)
# Override the mock behavior locally for this test to return OBSTACLE_MODAL
def local_side_effect(model, url, system_prompt, user_prompt, use_local_edge):
if random_text in user_prompt:
return '{"situation": "OBSTACLE_MODAL"}'
return '{"situation": "NORMAL"}'
mock_telepathic_classifier.side_effect = local_side_effect
result = sae.perceive(chaos_xml)
assert result == SituationType.OBSTACLE_MODAL
# PROOF: The VLM was actually called, and the prompt contained our randomized strings!
mock_telepathic_classifier.assert_called_once()
_, kwargs = mock_telepathic_classifier.call_args
user_prompt = kwargs.get("user_prompt", "")
id_suffix = random_id.split("/")[-1]
assert id_suffix in user_prompt, "Bot did not pass the random ID to VLM!"
assert random_text in user_prompt, "Bot did not pass the random text to VLM!"
assert random_button_text in user_prompt, "Bot did not pass the random button text to VLM!"
def test_perceive_action_blocked(self):
blocked_xml = INSTAGRAM_HOME_XML.replace(
'text="" resource-id="com.instagram.android:id/feed_tab"',
'text="Try again later" resource-id="com.instagram.android:id/bottom_sheet_container"'
'text="Try again later" resource-id="com.instagram.android:id/bottom_sheet_container"',
)
device = make_mock_device()
sae = SituationalAwarenessEngine(device)
@@ -227,13 +291,13 @@ class TestSAEPerception:
"""Passive scaffold containers (bottom_sheet_container_view, bottom_sheet_camera_container) must NOT be OBSTACLE_MODAL."""
device = make_mock_device()
sae = SituationalAwarenessEngine(device)
# XML containing navigation tabs + the passive scaffold container
passive_xml = INSTAGRAM_HOME_XML.replace(
'<node index="1" text="" resource-id="com.instagram.android:id/main_feed_container"',
'<node index="1" text="" resource-id="com.instagram.android:id/bottom_sheet_container_view" />\n'
'<node index="2" text="" resource-id="com.instagram.android:id/bottom_sheet_camera_container" />\n'
'<node index="3" text="" resource-id="com.instagram.android:id/main_feed_container"'
'<node index="3" text="" resource-id="com.instagram.android:id/main_feed_container"',
)
result = sae.perceive(passive_xml)
assert result == SituationType.NORMAL, f"Passive scaffold misclassified as {result}"
@@ -312,11 +376,11 @@ class TestSAERealFixturePerception:
assert result == SituationType.OBSTACLE_MODAL, f"Mystery interstitial misclassified as {result}"
# ─────────────────────────────────────────────────────
# FULL AUTONOMOUS RECOVERY TESTS
# ─────────────────────────────────────────────────────
class TestSAEAutonomousRecovery:
"""Tests the full perceive→plan→act→verify→learn loop."""
@@ -329,8 +393,7 @@ class TestSAEAutonomousRecovery:
]
sae = SituationalAwarenessEngine(device)
with patch.object(sae.episodes, 'recall', return_value=None), \
patch.object(sae.episodes, 'learn'):
with patch.object(sae.episodes, "recall", return_value=None), patch.object(sae.episodes, "learn"):
result = sae.ensure_clear_screen(max_attempts=3)
assert result is True
device.app_start.assert_called_with("com.instagram.android", use_monkey=True)
@@ -339,13 +402,12 @@ class TestSAEAutonomousRecovery:
"""Lock screen detected → SAE triggers unlock() → Instagram returns."""
device = make_mock_device()
device.dump_hierarchy.side_effect = [
LOCK_SCREEN_XML, # perceive: locked
INSTAGRAM_HOME_XML, # verify after unlock
LOCK_SCREEN_XML, # perceive: locked
INSTAGRAM_HOME_XML, # verify after unlock
]
sae = SituationalAwarenessEngine(device)
with patch.object(sae.episodes, 'recall', return_value=None), \
patch.object(sae.episodes, 'learn'):
with patch.object(sae.episodes, "recall", return_value=None), patch.object(sae.episodes, "learn"):
result = sae.ensure_clear_screen(max_attempts=3)
assert result is True
device.unlock.assert_called_once()
@@ -358,12 +420,11 @@ class TestSAEAutonomousRecovery:
INSTAGRAM_SURVEY_XML, # perceive: modal
INSTAGRAM_SURVEY_XML, # verify after BACK (BACK failed — modal still there)
INSTAGRAM_SURVEY_XML, # perceive again: still modal
INSTAGRAM_HOME_XML, # verify after clicking 'Not Now' (worked!)
INSTAGRAM_HOME_XML, # verify after clicking 'Not Now' (worked!)
]
sae = SituationalAwarenessEngine(device)
with patch.object(sae.episodes, 'recall', return_value=None), \
patch.object(sae.episodes, 'learn'):
with patch.object(sae.episodes, "recall", return_value=None), patch.object(sae.episodes, "learn"):
result = sae.ensure_clear_screen(max_attempts=5)
assert result is True
# First action was BACK, second was click
@@ -378,30 +439,90 @@ class TestSAEAutonomousRecovery:
device = make_mock_device()
device.dump_hierarchy.side_effect = [
INSTAGRAM_SURVEY_XML, # perceive: modal
INSTAGRAM_HOME_XML, # verify after BACK (worked!)
INSTAGRAM_HOME_XML, # verify after BACK (worked!)
]
sae = SituationalAwarenessEngine(device)
with patch.object(sae.episodes, 'recall', return_value=None), \
patch.object(sae.episodes, 'learn'):
with patch.object(sae.episodes, "recall", return_value=None), patch.object(sae.episodes, "learn"):
result = sae.ensure_clear_screen(max_attempts=3)
assert result is True
device.press.assert_called_with("back")
device.click.assert_not_called() # Never needed to click!
def test_recovers_from_unknown_modal_german(self):
"""German modal → BACK first → fails → finds 'Später' by TEXT → clicks."""
def test_recovers_from_randomized_chaos_modal(self, mock_telepathic_classifier, mock_fallback_llm):
"""Generates a totally random modal and verifies the LLM dictates the random coordinates to recover."""
import uuid
random_id = f"com.instagram.android:id/chaos_{uuid.uuid4().hex[:8]}"
random_text = f"Nonsense_Text_{uuid.uuid4().hex[:8]}"
random_button_text = f"Dismiss_{uuid.uuid4().hex[:8]}"
chaos_xml = f"""<?xml version='1.0' encoding='UTF-8' standalone='yes' ?>
<hierarchy rotation="0">
<node index="0" text="" resource-id="" class="android.widget.FrameLayout" package="com.instagram.android" content-desc="" clickable="false" bounds="[0,0][1080,2400]">
<node text="" resource-id="{random_id}" class="android.widget.FrameLayout" package="com.instagram.android" clickable="false" bounds="[0,500][1080,2200]">
<node text="{random_text}" resource-id="" class="android.widget.TextView" package="com.instagram.android" clickable="false" bounds="[100,600][980,700]" />
<node text="{random_button_text}" resource-id="" class="android.widget.Button" package="com.instagram.android" clickable="true" bounds="[321,2001][541,2101]" />
</node>
</node>
</hierarchy>"""
device = make_mock_device()
device.dump_hierarchy.side_effect = [
UNKNOWN_MODAL_XML, # perceive: modal
UNKNOWN_MODAL_XML, # verify after BACK (failed)
UNKNOWN_MODAL_XML, # perceive again
chaos_xml, # perceive: modal
chaos_xml, # verify after BACK (failed)
chaos_xml, # perceive again
INSTAGRAM_HOME_XML, # verify after clicking randomized coords
]
# VLM Classifier override
def local_classifier(model, url, system_prompt, user_prompt, use_local_edge):
if random_text in user_prompt:
return '{"situation": "OBSTACLE_MODAL"}'
return '{"situation": "NORMAL"}'
mock_telepathic_classifier.side_effect = local_classifier
# VLM Fallback override (Action Solver)
def local_solver(*args, **kwargs):
prompt = kwargs.get("prompt", args[2] if len(args) > 2 else "")
# Simulate real LLM: First it tries back, if 'back' is not in prompt
if "back:0,0" not in prompt.lower():
return {"response": '{"action": "back", "x": 0, "y": 0, "reason": "Try back first"}'}
# Next time it sees the prompt, it finds the random button
if random_button_text in prompt:
# The bounds of our random button are [321,2001][541,2101] -> center is 431, 2051
return {"response": '{"action": "click", "x": 431, "y": 2051, "reason": "Found chaos button"}'}
return {"response": '{"action": "back", "x": 0, "y": 0, "reason": "Fallback"}'}
mock_fallback_llm.side_effect = local_solver
sae = SituationalAwarenessEngine(device)
with patch.object(sae.episodes, "recall", return_value=None), patch.object(sae.episodes, "learn"):
result = sae.ensure_clear_screen(max_attempts=5)
assert result is True
# Proof that BACK was tried first
device.press.assert_called_with("back")
# Proof that the random coordinates were extracted and clicked
device.click.assert_called_once()
click_args = device.click.call_args
assert click_args[0] == (
431,
2051,
), f"Expected bot to click chaotic coordinates (431, 2051), but got {click_args[0]}"
def test_recovers_from_unknown_modal_german(self):
device = make_mock_device()
device.dump_hierarchy.side_effect = [
UNKNOWN_MODAL_XML, # perceive: modal
UNKNOWN_MODAL_XML, # verify after BACK (failed)
UNKNOWN_MODAL_XML, # perceive again
INSTAGRAM_HOME_XML, # verify after clicking 'Später'
]
sae = SituationalAwarenessEngine(device)
with patch.object(sae.episodes, 'recall', return_value=None), \
patch.object(sae.episodes, 'learn'):
with patch.object(sae.episodes, "recall", return_value=None), patch.object(sae.episodes, "learn"):
result = sae.ensure_clear_screen(max_attempts=5)
assert result is True
device.click.assert_called_once()
@@ -409,7 +530,7 @@ class TestSAEAutonomousRecovery:
def test_never_clicks_close_friends_on_follow_sheet(self):
"""CRITICAL REAL-WORLD BUG: Follow sheet has 'close_friends' row.
SAE must NEVER click it — it adds the user to Close Friends!"""
follow_sheet_xml = '''<?xml version='1.0' encoding='UTF-8' standalone='yes' ?>
follow_sheet_xml = """<?xml version='1.0' encoding='UTF-8' standalone='yes' ?>
<hierarchy rotation="0">
<node package="com.instagram.android" bounds="[0,0][1080,2400]">
<node resource-id="com.instagram.android:id/bottom_sheet_container" package="com.instagram.android" bounds="[0,1400][1080,2400]">
@@ -418,16 +539,15 @@ class TestSAEAutonomousRecovery:
<node resource-id="com.instagram.android:id/follow_sheet_unfollow_row" text="Unfollow" package="com.instagram.android" clickable="true" bounds="[0,2193][1080,2335]" />
</node>
</node>
</hierarchy>'''
</hierarchy>"""
device = make_mock_device()
device.dump_hierarchy.side_effect = [
follow_sheet_xml, # perceive: modal
follow_sheet_xml, # perceive: modal
INSTAGRAM_HOME_XML, # verify after BACK (worked!)
]
sae = SituationalAwarenessEngine(device)
with patch.object(sae.episodes, 'recall', return_value=None), \
patch.object(sae.episodes, 'learn'):
with patch.object(sae.episodes, "recall", return_value=None), patch.object(sae.episodes, "learn"):
result = sae.ensure_clear_screen(max_attempts=5)
assert result is True
# CRITICAL: Must use BACK, never click any follow sheet button
@@ -449,12 +569,12 @@ class TestSAEAutonomousRecovery:
GOOGLE_SEARCH_XML, # attempt 5: perceive
GOOGLE_SEARCH_XML, # attempt 5: verify (LLM failed)
GOOGLE_SEARCH_XML, # attempt 6: perceive (escalate to app_start)
INSTAGRAM_HOME_XML, # attempt 6: verify (app_start worked!)
INSTAGRAM_HOME_XML, # attempt 6: verify (app_start worked!)
]
sae = SituationalAwarenessEngine(device)
# Mock LLM to return back action (simulating LLM also failing)
with patch.object(sae, '_plan_escape_via_llm', return_value=EscapeAction("back", reason="LLM says back")):
with patch.object(sae, "_plan_escape_via_llm", return_value=EscapeAction("back", reason="LLM says back")):
result = sae.ensure_clear_screen(max_attempts=7)
assert result is True
device.app_start.assert_called()
@@ -474,9 +594,10 @@ class TestSAEAutonomousRecovery:
def test_action_blocked_raises_exception(self):
"""If Instagram blocks us, SAE must HALT — never try to dismiss."""
from GramAddict.core.exceptions import ActionBlockedError
blocked_xml = INSTAGRAM_HOME_XML.replace(
'text="" resource-id="com.instagram.android:id/feed_tab"',
'text="Try again later" resource-id="com.instagram.android:id/dialog_container"'
'text="Try again later" resource-id="com.instagram.android:id/dialog_container"',
)
device = make_mock_device()
device.dump_hierarchy.return_value = blocked_xml
@@ -490,6 +611,7 @@ class TestSAEAutonomousRecovery:
# LEARNING TESTS
# ─────────────────────────────────────────────────────
class TestSAELearning:
"""Tests that SAE learns from experience and never repeats failures."""
@@ -536,15 +658,17 @@ class TestSAELearning:
"""When LLM returns 'false_positive', SAE must overwrite Qdrant and return True."""
device = make_mock_device()
sae = SituationalAwarenessEngine(device)
device.dump_hierarchy.return_value = INSTAGRAM_HOME_XML
# Force the situation to be perceived as an OBSTACLE_MODAL initially
with patch.object(sae, 'perceive', return_value=SituationType.OBSTACLE_MODAL):
with patch.object(sae, "perceive", return_value=SituationType.OBSTACLE_MODAL):
# Mock LLM to return 'false_positive'
with patch.object(sae, '_plan_escape_via_llm', return_value=EscapeAction("false_positive", reason="No modal found")):
with patch.object(
sae, "_plan_escape_via_llm", return_value=EscapeAction("false_positive", reason="No modal found")
):
result = sae.ensure_clear_screen(max_attempts=1, initial_xml=INSTAGRAM_HOME_XML)
assert result is True
mock_store_screen.assert_called_once()
args, kwargs = mock_store_screen.call_args

217
tests/e2e/test_full_e2e_android_sim.py Normal file
View File

@@ -0,0 +1,217 @@
import xml.etree.ElementTree as ET
from unittest.mock import MagicMock, patch
import pytest
from GramAddict.core.device_facade import DeviceFacade
from GramAddict.core.goap import GoalExecutor
from GramAddict.core.qdrant_memory import QdrantBase
from GramAddict.core.telepathic_engine import TelepathicEngine
class AndroidEnvironmentSimulator(DeviceFacade):
def __init__(self, device_id="sim", app_id="com.instagram.android", args=None):
self.device_id = device_id
self.app_id = app_id
self.args = args
self.deviceV2 = MagicMock()
self.deviceV2.info = {"displayWidth": 1080, "displayHeight": 2400, "screenOn": True}
self.state_stack = ["home_feed"]
self.state_files = {
"home_feed": "tests/fixtures/home_feed_with_ad.xml",
"explore_grid": "tests/fixtures/explore_feed_dump.xml",
"post_detail": "tests/fixtures/organic_post.xml",
"user_profile": "tests/fixtures/user_profile_dump.xml",
}
def _current_state(self):
return self.state_stack[-1]
def dump_hierarchy(self):
current = self._current_state()
filepath = self.state_files[current]
with open(filepath, "r", encoding="utf-8") as f:
data = f.read()
print(f"📱 [Simulator] dump_hierarchy returning state: {current} (length: {len(data)})")
return data
def _parse_bounds(self, bounds_str):
import re
match = re.match(r"\[(\d+),(\d+)\]\[(\d+),(\d+)\]", bounds_str)
if match:
return [int(x) for x in match.groups()]
return None
def human_click(self, x, y):
# Simulate click translation to next state
xml_data = self.dump_hierarchy()
root = ET.fromstring(xml_data)
clicked_nodes = []
for node in root.iter("node"):
bounds_str = node.attrib.get("bounds", "")
bounds = self._parse_bounds(bounds_str)
if bounds:
x1, y1, x2, y2 = bounds
if x1 <= x <= x2 and y1 <= y <= y2:
area = (x2 - x1) * (y2 - y1)
clicked_nodes.append((area, node))
if not clicked_nodes:
return
clicked_nodes.sort(key=lambda item: item[0])
for _, target in clicked_nodes:
content_desc = target.attrib.get("content-desc", "") or ""
res_id = target.attrib.get("resource-id", "") or ""
text = target.attrib.get("text", "") or ""
current = self._current_state()
if current == "home_feed":
if "Search and explore" in content_desc or "search_tab" in res_id:
print(f"📱 [Simulator] Clicked ({x}, {y}) on {res_id}. Transition: home_feed -> explore_grid")
self.state_stack.append("explore_grid")
return
elif current == "explore_grid":
# In explore, anything the VLM clicks that has an image or button is likely a post
if "image_button" in res_id or "container" in res_id or target.attrib.get("clickable") == "true":
print(f"📱 [Simulator] Clicked ({x}, {y}) on {res_id}. Transition: explore_grid -> post_detail")
self.state_stack.append("post_detail")
return
elif current == "post_detail":
# Allow clicking either the post author or the comment author (both go to user_profile)
if 100 < x < 800 and 300 < y < 900:
print(f"📱 [Simulator] Clicked ({x}, {y}) on {res_id}. Transition: post_detail -> user_profile")
self.state_stack.append("user_profile")
return
# If we get here, no transition happened
for _, target in clicked_nodes:
print(
f"📱 [Simulator] Click ({x}, {y}) fell through on: {target.attrib.get('resource-id')} / text={target.attrib.get('text')}"
)
if not clicked_nodes:
print(f"📱 [Simulator] Click ({x}, {y}) fell outside ALL elements!")
def click(self, x=None, y=None, obj=None):
if x is not None and y is not None:
self.human_click(x, y)
elif obj and isinstance(obj, dict) and "x" in obj:
self.human_click(obj["x"], obj["y"])
def press(self, key):
if key == "back":
if len(self.state_stack) > 1:
old_state = self.state_stack.pop()
print(f"📱 [Simulator] Back pressed. State Transition: {old_state} -> {self._current_state()}")
else:
print("📱 [Simulator] Back pressed at root state.")
def _get_current_app(self):
return self.app_id
def get_info(self):
return self.deviceV2.info
def wake_up(self):
pass
def unlock(self):
pass
def shell(self, cmd):
return ""
def swipe(self, sx, sy, ex, ey, duration=None):
print(f"📱 [Simulator] Swiped ({sx}, {sy}) -> ({ex}, {ey})")
def human_swipe(self, sx, sy, ex, ey, duration=None):
print(f"📱 [Simulator] Swiped ({sx}, {sy}) -> ({ex}, {ey})")
@property
def info(self):
return self.deviceV2.info
@pytest.fixture(autouse=True)
def setup_qdrant_isolation():
"""Prefix all Qdrant collections with test_sim_ so we don't pollute live data."""
original_init = QdrantBase.__init__
def mocked_init(self, collection_name, *args, **kwargs):
test_collection = f"test_sim_{collection_name}"
original_init(self, test_collection, *args, **kwargs)
with patch.object(QdrantBase, "__init__", new=mocked_init):
# We aggressively wipe these collections before running the test!
from GramAddict.core.qdrant_memory import NavigationMemoryDB
qb = NavigationMemoryDB()
try:
qb.wipe_collection()
except:
pass
yield
def test_full_autonomous_sim_loop(monkeypatch):
"""
This test runs the real GoalExecutor with the real TelepathicEngine (VLM)
and real Qdrant (sandboxed via prefix) against a simulated Android environment.
"""
import urllib.request
try:
urllib.request.urlopen("http://localhost:11434/", timeout=2)
except Exception:
pytest.skip("Ollama is not running. Live E2E sim requires LLM backend.")
# 1. Create Simulator
sim_device = AndroidEnvironmentSimulator()
# 2. Patch TelepathicEngine to NOT be mocked by conftest
engine = TelepathicEngine()
monkeypatch.setattr(TelepathicEngine, "get_instance", lambda: engine)
# 3. Create context and GoalExecutor
from GramAddict.core.config import Config
if not hasattr(Config(), "args"):
Config().args = MagicMock()
Config().args.use_nav_memory = True
Config().args.use_semantic_memory = True
executor = GoalExecutor(sim_device, bot_username="testbot")
# 4. Start an autonomous loop: We want to reach an organic post from the home feed
assert sim_device._current_state() == "home_feed"
success = executor.achieve("open post", max_steps=10)
assert success is True
# The VLM should have figured out:
# 1. Tap explore tab -> switches to "explore_grid"
# 2. Tap grid item -> switches to "post_detail"
assert sim_device._current_state() == "post_detail"
# 5. Let's do another intent: view the user profile
success = executor.achieve("open post author profile", max_steps=5)
assert success is True
assert sim_device._current_state() == "user_profile"
# 6. Now go back to the post
success = executor.achieve("open post", max_steps=5)
assert success is True
assert sim_device._current_state() == "post_detail"
# 7. Check Qdrant Memory is actually populated
# We should have stored the state transitions in the goap_paths collection
from GramAddict.core.goap import PathMemory
nav_db = PathMemory("testbot")
# verify at least some nodes exist
count = nav_db._db.client.count(nav_db._db.collection_name).count
assert count > 0, "Qdrant memory should have learned the paths!"