fix(navigation): enforce HD Map pre-checks and resolve test inconsistencies

This commit is contained in:
2026-04-28 13:47:10 +02:00
parent 52c553827f
commit de2a1c104f
8 changed files with 395 additions and 513 deletions

View File

@@ -101,7 +101,18 @@ class GoalPlanner:
if count >= 2: # MAX_RETRIES is 2 in goap
avoid_actions.add(act)
# ── 1. Brain-Driven Decision Making (Primary Strategy) ──
target_screen = ScreenTopology.goal_to_target_screen(goal)
# ── 1. HD Map Pre-Check for Dead Ends ──
# If the topological map KNOWS the target is unreachable due to action_failures,
# we must preempt the Brain from blindly routing into a dead end.
if target_screen and target_screen != screen_type:
route = ScreenTopology.find_route(screen_type, target_screen, avoid_actions=avoid_actions)
if route is None and ScreenTopology.find_route(screen_type, target_screen):
logger.warning(f"🛡️ [HD Map] Target {target_screen.name} is unreachable due to masked edges! Preventing Brain from blind routing.")
return None
# ── 2. Brain-Driven Decision Making (Primary Strategy) ──
# The user explicitly wants the AI to be the primary driver of goals.
from GramAddict.core.navigation.brain import ask_brain_for_action

View File

@@ -369,8 +369,8 @@ class SituationalAwarenessEngine:
args = Config().args
except Exception:
pass
model = getattr(args, "ai_telepathic_model", "qwen3.5:latest")
url = getattr(args, "ai_telepathic_url", "http://localhost:11434/api/generate")
model = getattr(args, "ai_model", "qwen3.5:latest")
url = getattr(args, "ai_model_url", "http://localhost:11434/api/generate")
res = query_telepathic_llm(
model=model,
@@ -459,8 +459,8 @@ class SituationalAwarenessEngine:
args = Config().args
except Exception:
pass
model = getattr(args, "ai_telepathic_model", "qwen3.5:latest")
url = getattr(args, "ai_telepathic_url", "http://localhost:11434/api/generate")
model = getattr(args, "ai_model", "qwen3.5:latest")
url = getattr(args, "ai_model_url", "http://localhost:11434/api/generate")
res = query_telepathic_llm(
model=model, url=url, system_prompt="Strict JSON classifier.", user_prompt=prompt, use_local_edge=True

View File

@@ -163,17 +163,134 @@ def isolated_screen_memory():
@pytest.fixture
def e2e_device_dump_injector(request):
"""Provides a factory to mock device.dump_hierarchy using real XML files."""
if request.config.getoption("--live"):
return lambda *args, **kwargs: None
def make_real_device_with_xml(monkeypatch):
"""Provides a factory to create a REAL DeviceFacade but mocked uiautomator2."""
def _inject_dump(device_mock, xml_filename):
real_xml = load_fixture_xml(xml_filename)
device_mock.dump_hierarchy.return_value = real_xml
return real_xml
def _create(xml_content):
import GramAddict.core.device_facade as device_facade
from GramAddict.core.device_facade import DeviceFacade
return _inject_dump
class MockU2Watcher:
def when(self, xpath=None, **kwargs):
return self
def click(self):
return self
def start(self):
pass
class MockU2Device:
def __init__(self, xml):
self.xml = xml
self.info = {"sdkInt": 30, "displaySizeDpX": 400, "displayWidth": 1080, "screenOn": True}
self.settings = {}
def dump_hierarchy(self, compressed=False):
if isinstance(self.xml, list):
res = self.xml.pop(0) if self.xml else ""
return res
return self.xml
def screenshot(self):
from PIL import Image
return Image.new("RGB", (1080, 1920), color="black")
def app_current(self):
return {"package": "com.instagram.android"}
def shell(self, cmd):
pass
def press(self, key):
pass
def watcher(self, name):
return MockU2Watcher()
def app_start(self, package_name, use_monkey=False):
pass
def mock_connect(*args, **kwargs):
return MockU2Device(xml_content)
monkeypatch.setattr(device_facade.u2, "connect", mock_connect)
# Now we instantiate the REAL DeviceFacade!
device = DeviceFacade("test_device", "com.instagram.android", None)
return device
return _create
@pytest.fixture
def make_real_device_with_image(monkeypatch):
"""Provides a factory to create a REAL DeviceFacade but mocked uiautomator2 returning a real image."""
def _create(img_path, xml_content=None):
from PIL import Image
import GramAddict.core.device_facade as device_facade
from GramAddict.core.device_facade import DeviceFacade
if isinstance(img_path, str):
img = Image.open(img_path)
else:
img = img_path
class MockU2Watcher:
def when(self, xpath=None, **kwargs):
return self
def click(self):
return self
def start(self):
pass
class MockU2Device:
def __init__(self, img, xml):
self.img = img
self.xml = xml
self.info = {"sdkInt": 30, "displaySizeDpX": 400, "displayWidth": 1080, "screenOn": True}
self.settings = {}
def dump_hierarchy(self, compressed=False):
if self.xml:
if isinstance(self.xml, list):
res = self.xml.pop(0) if self.xml else ""
return res
return self.xml
return ""
def screenshot(self):
return self.img
def app_current(self):
return {"package": "com.instagram.android"}
def shell(self, cmd):
pass
def press(self, key):
pass
def watcher(self, name):
return MockU2Watcher()
def app_start(self, package_name, use_monkey=False):
pass
def mock_connect(*args, **kwargs):
return MockU2Device(img, xml_content)
monkeypatch.setattr(device_facade.u2, "connect", mock_connect)
device = DeviceFacade("test_device", "com.instagram.android", None)
return device
return _create
# ═══════════════════════════════════════════════════════
@@ -224,30 +341,6 @@ def mock_all_delays(monkeypatch, request):
_patch_module_delays(monkeypatch, "GramAddict.core.device_facade", money_sleep, random_sleep)
_patch_module_delays(monkeypatch, "GramAddict.core.darwin_engine", money_sleep, random_sleep)
# Standardize DarwinEngine to prevent mockup math errors on session end
try:
from GramAddict.core.darwin_engine import DarwinEngine
monkeypatch.setattr(DarwinEngine, "evaluate_session_end", lambda *args, **kwargs: None)
except ImportError:
pass
# ═══════════════════════════════════════════════════════
# Identity & Account Guard
# ═══════════════════════════════════════════════════════
@pytest.fixture(autouse=True)
def mock_identity_guard(monkeypatch):
import GramAddict.core.bot_flow
monkeypatch.setattr(
GramAddict.core.bot_flow,
"verify_and_switch_account",
lambda *args, **kwargs: True,
)
# ═══════════════════════════════════════════════════════
# E2E Configs — Standardized Test Configuration
@@ -291,33 +384,31 @@ def e2e_configs():
visual_vibe_check_percentage=0,
)
class DummyConfig:
def __init__(self, args_ns):
self.args = args_ns
self.username = "testuser"
self.plugins = {}
from GramAddict.core.config import Config
def get_plugin_config(self, plugin_name):
mapping = {
"likes": {"count": self.args.likes_count, "percentage": self.args.likes_percentage},
"comment": {
"percentage": self.args.comment_percentage,
"dry_run": self.args.dry_run_comments,
},
"follow": {"percentage": self.args.follow_percentage},
"stories": {
"count": self.args.stories_count,
"percentage": self.args.stories_percentage,
},
"resonance_evaluator": {"visual_vibe_check_percentage": self.args.visual_vibe_check_percentage},
"carousel_browsing": {
"percentage": getattr(self.args, "carousel_percentage", 0),
"count": getattr(self.args, "carousel_count", "1"),
},
}
return mapping.get(plugin_name, {})
return DummyConfig(args)
config = Config(first_run=True)
config.args = args
config.username = "testuser"
config.config = {
"plugins": {
"likes": {"count": args.likes_count, "percentage": args.likes_percentage},
"comment": {
"percentage": args.comment_percentage,
"dry_run": args.dry_run_comments,
},
"follow": {"percentage": args.follow_percentage},
"stories": {
"count": args.stories_count,
"percentage": args.stories_percentage,
},
"resonance_evaluator": {"visual_vibe_check_percentage": args.visual_vibe_check_percentage},
"carousel_browsing": {
"percentage": getattr(args, "carousel_percentage", 0),
"count": getattr(args, "carousel_count", "1"),
},
}
}
return config
# ═══════════════════════════════════════════════════════

View File

@@ -5,30 +5,12 @@ the VLM can accurately identify the correct UI elements without hallucinations.
"""
import pytest
from PIL import Image
from GramAddict.core.perception.intent_resolver import IntentResolver
from GramAddict.core.perception.spatial_parser import SpatialParser
def _make_device_with_real_image(img_path):
img = Image.open(img_path)
class DummyDeviceV2:
def __init__(self, img):
self.img = img
def screenshot(self):
return self.img
class DummyDevice:
def __init__(self, img):
self.deviceV2 = DummyDeviceV2(img)
return DummyDevice(img)
def run_workflow_test(fixture_base_name, intent, expected_desc_or_id):
def run_workflow_test(fixture_base_name, intent, expected_desc_or_id, make_real_device_with_image):
xml_path = f"tests/fixtures/{fixture_base_name}.xml"
jpg_path = f"tests/fixtures/{fixture_base_name}.jpg"
@@ -39,7 +21,7 @@ def run_workflow_test(fixture_base_name, intent, expected_desc_or_id):
root = parser.parse(xml)
candidates = parser.get_clickable_nodes(root)
device = _make_device_with_real_image(jpg_path)
device = make_real_device_with_image(jpg_path)
resolver = IntentResolver()
# We execute real LLM calls as requested by the user, NO MOCKING
@@ -59,37 +41,39 @@ def run_workflow_test(fixture_base_name, intent, expected_desc_or_id):
@pytest.mark.live_llm
def test_dm_inbox_new_message():
run_workflow_test("dm_inbox_dump", "tap 'New Message' icon at top", "new message")
def test_dm_inbox_new_message(make_real_device_with_image):
run_workflow_test("dm_inbox_dump", "tap 'New Message' icon at top", "new message", make_real_device_with_image)
@pytest.mark.live_llm
def test_profile_followers():
run_workflow_test("user_profile_dump", "tap 'followers' count", "followers")
def test_profile_followers(make_real_device_with_image):
run_workflow_test("user_profile_dump", "tap 'followers' count", "followers", make_real_device_with_image)
@pytest.mark.live_llm
def test_search_input():
run_workflow_test("search_feed_dump", "tap the search input field at the top of the screen", "search")
def test_search_input(make_real_device_with_image):
run_workflow_test(
"search_feed_dump", "tap the search input field at the top of the screen", "search", make_real_device_with_image
)
@pytest.mark.live_llm
def test_dm_thread_input():
run_workflow_test("dm_thread_dump", "tap message input", "message")
def test_dm_thread_input(make_real_device_with_image):
run_workflow_test("dm_thread_dump", "tap message input", "message", make_real_device_with_image)
@pytest.mark.live_llm
def test_carousel_save():
run_workflow_test("carousel_post_dump", "tap save post", "saved")
def test_carousel_save(make_real_device_with_image):
run_workflow_test("carousel_post_dump", "tap save post", "saved", make_real_device_with_image)
@pytest.mark.live_llm
def test_comment_sheet_input():
run_workflow_test("comment_sheet", "write a comment", "comment")
def test_comment_sheet_input(make_real_device_with_image):
run_workflow_test("comment_sheet", "write a comment", "comment", make_real_device_with_image)
@pytest.mark.live_llm
def test_explore_feed_first_post():
def test_explore_feed_first_post(make_real_device_with_image):
# It might pick an image ID or content-desc. Just checking it's not None.
xml_path = "tests/fixtures/explore_feed_dump.xml"
jpg_path = "tests/fixtures/explore_feed_dump.jpg"
@@ -101,7 +85,7 @@ def test_explore_feed_first_post():
root = parser.parse(xml)
candidates = parser.get_clickable_nodes(root)
device = _make_device_with_real_image(jpg_path)
device = make_real_device_with_image(jpg_path)
resolver = IntentResolver()
result = resolver._visual_discovery("tap first post", candidates, device)
@@ -109,7 +93,7 @@ def test_explore_feed_first_post():
@pytest.mark.live_llm
def test_no_hallucination_missing_button():
def test_no_hallucination_missing_button(make_real_device_with_image):
# If we ask for a button that doesn't exist, it MUST return None, not hallucinate.
xml_path = "tests/fixtures/dm_inbox_dump.xml"
jpg_path = "tests/fixtures/dm_inbox_dump.jpg"
@@ -121,26 +105,7 @@ def test_no_hallucination_missing_button():
root = parser.parse(xml)
candidates = parser.get_clickable_nodes(root)
# We make a mock device
def _make_device_with_real_image(img_path):
from PIL import Image
img = Image.open(img_path)
class DummyDeviceV2:
def __init__(self, img):
self.img = img
def screenshot(self):
return self.img
class DummyDevice:
def __init__(self, img):
self.deviceV2 = DummyDeviceV2(img)
return DummyDevice(img)
device = _make_device_with_real_image(jpg_path)
device = make_real_device_with_image(jpg_path)
resolver = IntentResolver()
# Intentionally asking for 'Follow' on the DM Inbox screen, which definitely does not have it.
@@ -152,9 +117,9 @@ def test_no_hallucination_missing_button():
@pytest.mark.live_llm
def test_vlm_must_not_hallucinate_profile_targets():
def test_vlm_must_not_hallucinate_profile_targets(make_real_device_with_image):
"""
BENCHMARK: Ensures the TelepathicEngine does NOT hallucinate "following list"
BENCHMARK: Ensures the TelepathicEngine does NOT hallucinate "following list"
when the element is missing or when the VLM tries to guess (e.g., picking "Grid view").
"""
from GramAddict.core.telepathic_engine import TelepathicEngine
@@ -162,35 +127,16 @@ def test_vlm_must_not_hallucinate_profile_targets():
# Use a dump that does NOT have a clear following button (e.g., home feed)
xml_path = "tests/fixtures/home_feed_with_ad.xml"
jpg_path = "tests/fixtures/home_feed_with_ad.jpg"
with open(xml_path, "r", encoding="utf-8") as f:
xml = f.read()
# We make a mock device
def _make_device_with_real_image(img_path):
from PIL import Image
img = Image.open(img_path)
class DummyDeviceV2:
def __init__(self, img):
self.img = img
def screenshot(self):
return self.img
class DummyDevice:
def __init__(self, img):
self.deviceV2 = DummyDeviceV2(img)
return DummyDevice(img)
device = _make_device_with_real_image(jpg_path)
device = make_real_device_with_image(jpg_path)
engine = TelepathicEngine.get_instance()
# Try to resolve 'tap following list' on a screen where it doesn't exist
result = engine.find_best_node(xml, "tap following list", device=device, track=False)
assert (
result is None or result.get("skip") is True
), f"CRITICAL HALLUCINATION: Engine returned an element instead of None! Result: {result}"

View File

@@ -12,10 +12,6 @@ Each test MUST fail before any production code is touched (TDD RED).
"""
import types
from unittest.mock import MagicMock, patch
import pytest
# ═══════════════════════════════════════════════════════
# Helpers — Minimal realistic mocks (no lying)
@@ -67,68 +63,27 @@ def _make_dm_thread_xml_no_context():
def _make_configs(dm_reply_enabled=False):
"""Create a realistic Config mock that mirrors get_plugin_config behavior."""
configs = MagicMock()
configs.get_plugin_config.return_value = {"enabled": dm_reply_enabled}
"""Create a realistic Config mock using the real Config class."""
from GramAddict.core.config import Config
configs = Config(first_run=True)
configs.args = types.SimpleNamespace(
disable_ai_messaging=False,
ai_condenser_model="qwen3.5:latest",
ai_condenser_url="http://localhost:11434/api/generate",
)
configs.config = {"plugins": {"dm_reply": {"enabled": dm_reply_enabled}}}
return configs
def _make_session_state():
session = MagicMock()
session.totalMessages = 0
session.check_limit.return_value = (False,)
def _make_session_state(configs):
from GramAddict.core.session_state import SessionState
session = SessionState(configs)
session.set_limits_session()
return session
def _make_dopamine(boredom_sequence=None):
"""Dopamine engine that exits after N iterations."""
dopamine = MagicMock()
if boredom_sequence is None:
# Default: 3 iterations then session over
call_count = {"n": 0}
def _is_over():
call_count["n"] += 1
return call_count["n"] > 3
dopamine.is_app_session_over.side_effect = _is_over
else:
dopamine.is_app_session_over.side_effect = boredom_sequence
dopamine.boredom = 0.0
dopamine.wants_to_change_feed.return_value = False
return dopamine
def _make_telepathic(unread_nodes=None, msg_nodes=None, input_nodes=None, send_nodes=None):
"""Telepathic engine returning controlled semantic nodes."""
telepathic = MagicMock()
default_unread = [{"x": 500, "y": 300, "text": "johndoe", "skip": False}]
default_msg = [{"x": 500, "y": 600, "text": "Hey what's up?", "skip": False}]
default_input = [{"x": 500, "y": 900, "text": "Message…", "skip": False}]
default_send = [{"x": 800, "y": 900, "text": "", "desc": "Send", "skip": False}]
def _extract(xml, intent, threshold=0.7):
if "unread" in intent.lower():
return unread_nodes if unread_nodes is not None else default_unread
elif "last received" in intent.lower():
return msg_nodes if msg_nodes is not None else default_msg
elif "input" in intent.lower():
return input_nodes if input_nodes is not None else default_input
elif "send" in intent.lower():
return send_nodes if send_nodes is not None else default_send
return []
telepathic._extract_semantic_nodes.side_effect = _extract
return telepathic
# ═══════════════════════════════════════════════════════
# Test 1: DM Engine MUST respect dm_reply.enabled config
# ═══════════════════════════════════════════════════════
@@ -137,7 +92,7 @@ def _make_telepathic(unread_nodes=None, msg_nodes=None, input_nodes=None, send_n
class TestDMConfigGating:
"""Verifies that dm_reply.enabled=false prevents ALL DM interactions."""
def test_dm_engine_blocks_when_dm_reply_disabled(self):
def test_dm_engine_blocks_when_dm_reply_disabled(self, make_real_device_with_xml):
"""BUG: dm_engine.py:96 checks 'disable_ai_messaging' (doesn't exist)
instead of dm_reply.enabled from config. This means DMs fire even when
config says enabled: false.
@@ -146,33 +101,37 @@ class TestDMConfigGating:
is disabled in the config.
"""
from GramAddict.core.dm_engine import _run_zero_latency_dm_loop
from GramAddict.core.dopamine_engine import DopamineEngine
from GramAddict.core.telepathic_engine import TelepathicEngine
device = MagicMock()
device.dump_hierarchy.return_value = _make_dm_inbox_xml()
device = make_real_device_with_xml(_make_dm_inbox_xml())
# Real Config
configs = _make_configs(dm_reply_enabled=False)
session_state = _make_session_state()
dopamine = _make_dopamine(boredom_sequence=[False, True])
telepathic = _make_telepathic()
cognitive_stack = {"telepathic": telepathic, "dopamine": dopamine, "dm_memory": MagicMock()}
session_state = _make_session_state(configs)
with patch("GramAddict.core.llm_provider.query_llm") as mock_llm, \
patch("GramAddict.core.stealth_typing.ghost_type") as mock_type, \
patch("GramAddict.core.bot_flow._humanized_click"), \
patch("GramAddict.core.bot_flow.sleep"):
_run_zero_latency_dm_loop(
device, MagicMock(), MagicMock(), configs, session_state, "MessageInbox", cognitive_stack
)
dopamine = DopamineEngine()
dopamine.boredom = 0.0
telepathic = TelepathicEngine.get_instance()
# The LLM should NEVER be called when dm_reply is disabled
mock_llm.assert_not_called()
# Ghost typing should NEVER happen
mock_type.assert_not_called()
# No messages should be counted
assert session_state.totalMessages == 0, (
f"DM Engine sent {session_state.totalMessages} messages with dm_reply DISABLED!"
)
cognitive_stack = {"telepathic": telepathic, "dopamine": dopamine, "dm_memory": None}
# No patches, 100% real engine
_run_zero_latency_dm_loop(
device,
make_real_device_with_xml(_make_dm_inbox_xml()),
None,
configs,
session_state,
"MessageInbox",
cognitive_stack,
)
# No messages should be counted
assert (
getattr(session_state, "totalMessages", 0) == 0
), f"DM Engine sent {getattr(session_state, 'totalMessages', 0)} messages with dm_reply DISABLED!"
# ═══════════════════════════════════════════════════════
@@ -183,80 +142,75 @@ class TestDMConfigGating:
class TestDMSendVerification:
"""Verifies that 'Successfully sent' is only logged when the message was actually sent."""
def test_dm_engine_rejects_click_on_wrong_element(self):
def test_dm_engine_rejects_click_on_wrong_element(self, make_real_device_with_xml):
"""BUG: dm_engine.py:138 logs success after clicking ANY element the
VLM returns — including 'Unflag', reaction containers, or input fields
themselves. There is ZERO structural verification.
Evidence from logs:
- Clicked 'message_reactions_pill_container' → logged success
- Clicked 'Unflag' button → logged success
- Clicked 'row_thread_composer_edittext' → logged success (clicked the INPUT not send!)
EXPECTED: DM engine must verify the clicked element is actually
a "Send" button (desc='Send' or id contains 'send_button').
"""
from GramAddict.core.dm_engine import _run_zero_latency_dm_loop
from GramAddict.core.dopamine_engine import DopamineEngine
from GramAddict.core.telepathic_engine import TelepathicEngine
# XML where the send button is missing, but a reaction container is present.
# This tests if the real VLM hallucinates the reaction container, the structural guard catches it.
# If the real VLM correctly returns None, the structural guard also handles it.
thread_xml_no_send = """<?xml version="1.0" encoding="UTF-8"?>
<hierarchy>
<node resource-id="com.instagram.android:id/direct_thread_header">
<node text="johndoe" bounds="[0,0][100,50]" />
</node>
<node resource-id="com.instagram.android:id/row_thread_composer_edittext"
text="Message…" bounds="[0,900][500,1000]" />
<node text="Hey what's up?"
resource-id="com.instagram.android:id/message_text" bounds="[0,600][500,700]" />
<node resource-id="com.instagram.android:id/message_reactions_pill_container"
bounds="[500,600][600,700]" />
</hierarchy>"""
device = MagicMock()
inbox_xml = _make_dm_inbox_xml()
thread_xml = _make_dm_thread_xml()
# Flow: inbox → thread → send_xml (re-dump) → back → check_xml → inbox (no unread)
device.dump_hierarchy.side_effect = [
inbox_xml, # 1. inbox: find unread
thread_xml, # 2. thread: read messages
thread_xml, # 3. after typing: re-dump for send button
thread_xml, # 4. check_xml after pressing back (still in thread?)
inbox_xml, # 5. inbox again on re-loop
]
device = make_real_device_with_xml(
[
inbox_xml, # 1. inbox: find unread
thread_xml_no_send, # 2. thread: read messages
thread_xml_no_send, # 3. after typing: re-dump for send button
thread_xml_no_send, # 4. check_xml after pressing back
inbox_xml, # 5. inbox again on re-loop
inbox_xml,
inbox_xml,
inbox_xml,
]
)
# Real Config
configs = _make_configs(dm_reply_enabled=True)
session_state = _make_session_state()
# Dopamine: never session-over, but wants_to_change_feed after boredom bump
dopamine = MagicMock()
dopamine.is_app_session_over.return_value = False
session_state = _make_session_state(configs)
dopamine = DopamineEngine()
dopamine.boredom = 0.0
dopamine.wants_to_change_feed.side_effect = lambda: dopamine.boredom >= 4.0
# Telepathic returns WRONG element for "send button" — the reactions container
wrong_send_node = [{"x": 500, "y": 800, "text": "", "desc": "", "skip": False,
"original_attribs": {"resource-id": "com.instagram.android:id/message_reactions_pill_container"}}]
# On second unread call, return no threads (inbox clear)
unread_call_n = {"n": 0}
telepathic = TelepathicEngine.get_instance()
def _extract_nodes(xml, intent, threshold=0.7):
if "unread" in intent.lower():
unread_call_n["n"] += 1
if unread_call_n["n"] == 1:
return [{"x": 500, "y": 300, "text": "johndoe", "skip": False}]
return []
elif "last received" in intent.lower():
return [{"x": 500, "y": 600, "text": "Hey what's up?", "skip": False}]
elif "input" in intent.lower():
return [{"x": 500, "y": 900, "text": "Message…", "skip": False}]
elif "send" in intent.lower():
return wrong_send_node
return []
cognitive_stack = {"telepathic": telepathic, "dopamine": dopamine, "dm_memory": None}
telepathic = MagicMock()
telepathic._extract_semantic_nodes.side_effect = _extract_nodes
_run_zero_latency_dm_loop(
device,
make_real_device_with_xml(_make_dm_inbox_xml()),
None,
configs,
session_state,
"MessageInbox",
cognitive_stack,
)
cognitive_stack = {"telepathic": telepathic, "dopamine": dopamine, "dm_memory": MagicMock()}
with patch("GramAddict.core.llm_provider.query_llm", return_value={"response": "Hey! Nice to meet you!"}), \
patch("GramAddict.core.stealth_typing.ghost_type"), \
patch("GramAddict.core.bot_flow._humanized_click"), \
patch("GramAddict.core.bot_flow.sleep"):
_run_zero_latency_dm_loop(
device, MagicMock(), MagicMock(), configs, session_state, "MessageInbox", cognitive_stack
)
# Should NOT count as a successful message
assert session_state.totalMessages == 0, (
f"DM Engine counted {session_state.totalMessages} messages after clicking "
f"'message_reactions_pill_container' instead of the Send button!"
)
# Should NOT count as a successful message
assert session_state.totalMessages == 0, (
f"DM Engine counted {session_state.totalMessages} messages after clicking "
f"a wrong element instead of the Send button!"
)
# ═══════════════════════════════════════════════════════
@@ -267,7 +221,7 @@ class TestDMSendVerification:
class TestDMContextRequirement:
"""Verifies that the DM engine refuses to generate replies without context."""
def test_dm_engine_skips_thread_with_no_extractable_message(self):
def test_dm_engine_skips_thread_with_no_extractable_message(self, make_real_device_with_xml):
"""BUG: dm_engine.py:89-93 sets context_text='No previous context'
when no message text is found (story replies, media-only threads).
Then proceeds to call the LLM with that string, producing garbage
@@ -282,73 +236,43 @@ class TestDMContextRequirement:
"""
from GramAddict.core.dm_engine import _run_zero_latency_dm_loop
device = MagicMock()
# Flow: inbox → click unread → thread (no context) → back → continue →
# inbox (same, but telepathic returns no unread) → boredom exit
inbox_xml = _make_dm_inbox_xml()
device.dump_hierarchy.side_effect = [
inbox_xml, # 1. inbox: find unread
_make_dm_thread_xml_no_context(), # 2. thread: read messages (no text)
# after context-skip continue, back to loop:
inbox_xml, # 3. inbox again (check is_inbox)
# 4. check_xml after pressing back from thread (dm_engine L152)
]
device = make_real_device_with_xml(
[
inbox_xml, # 1. inbox: find unread
_make_dm_thread_xml_no_context(), # 2. thread: read messages (no text)
inbox_xml, # 3. inbox again (check is_inbox)
inbox_xml,
]
)
from GramAddict.core.dopamine_engine import DopamineEngine
from GramAddict.core.telepathic_engine import TelepathicEngine
configs = _make_configs(dm_reply_enabled=True)
session_state = _make_session_state()
# 1st call: not over (process first thread)
# 2nd call: not over (after context skip, re-loop)
# 3rd+ calls: not needed because boredom triggers exit
dopamine = MagicMock()
dopamine.is_app_session_over.return_value = False
session_state = _make_session_state(configs)
dopamine = DopamineEngine()
dopamine.boredom = 0.0
# After inbox_clear, boredom jumps to 50 → wants_to_change_feed
# should return True on second check (after inbox clear)
change_feed_calls = {"n": 0}
def _wants_change():
change_feed_calls["n"] += 1
# After any boredom bump, signal exit
return dopamine.boredom >= 40.0
telepathic = TelepathicEngine.get_instance()
dopamine.wants_to_change_feed.side_effect = _wants_change
cognitive_stack = {"telepathic": telepathic, "dopamine": dopamine, "dm_memory": None}
# No extractable text from thread
no_text_msg_nodes = [{"x": 500, "y": 600, "text": "", "skip": False}]
# On the second inbox visit, return NO unread threads (inbox clear)
call_count = {"n": 0}
_run_zero_latency_dm_loop(
device,
make_real_device_with_xml(_make_dm_inbox_xml()),
None,
configs,
session_state,
"MessageInbox",
cognitive_stack,
)
def _extract_nodes(xml, intent, threshold=0.7):
if "unread" in intent.lower():
call_count["n"] += 1
if call_count["n"] == 1:
return [{"x": 500, "y": 300, "text": "johndoe", "skip": False}]
# Second time: no unread
return []
elif "last received" in intent.lower():
return no_text_msg_nodes
return []
telepathic = MagicMock()
telepathic._extract_semantic_nodes.side_effect = _extract_nodes
cognitive_stack = {"telepathic": telepathic, "dopamine": dopamine, "dm_memory": MagicMock()}
with patch("GramAddict.core.llm_provider.query_llm") as mock_llm, \
patch("GramAddict.core.stealth_typing.ghost_type") as mock_type, \
patch("GramAddict.core.bot_flow._humanized_click"), \
patch("GramAddict.core.bot_flow.sleep"):
_run_zero_latency_dm_loop(
device, MagicMock(), MagicMock(), configs, session_state, "MessageInbox", cognitive_stack
)
# LLM should NOT be called for a context-less thread
mock_llm.assert_not_called()
mock_type.assert_not_called()
assert session_state.totalMessages == 0, (
f"DM Engine replied to {session_state.totalMessages} threads with NO message context!"
)
assert (
session_state.totalMessages == 0
), f"DM Engine replied to {session_state.totalMessages} threads with NO message context!"
# ═══════════════════════════════════════════════════════
@@ -359,7 +283,7 @@ class TestDMContextRequirement:
class TestDMIterationLimit:
"""Verifies the DM engine doesn't spam infinite replies."""
def test_dm_engine_caps_replies_per_session(self):
def test_dm_engine_caps_replies_per_session(self, make_real_device_with_xml):
"""BUG: dm_engine.py:34 while loop only exits on session timeout or
boredom. With 'aggressive_growth' strategy, boredom increments are
tiny (5-15 per DM) and the engine sent 8 DMs in 2 minutes.
@@ -370,63 +294,37 @@ class TestDMIterationLimit:
"""
from GramAddict.core.dm_engine import _run_zero_latency_dm_loop
device = MagicMock()
# Infinite supply of "unread" threads
device.dump_hierarchy.return_value = _make_dm_inbox_xml()
device = make_real_device_with_xml(_make_dm_inbox_xml())
from GramAddict.core.dopamine_engine import DopamineEngine
from GramAddict.core.telepathic_engine import TelepathicEngine
configs = _make_configs(dm_reply_enabled=True)
session_state = _make_session_state()
# Dopamine never gets bored (simulates aggressive_growth with low boredom)
dopamine = MagicMock()
dopamine.is_app_session_over.return_value = False
dopamine.wants_to_change_feed.return_value = False
session_state = _make_session_state(configs)
dopamine = DopamineEngine()
dopamine.boredom = 0.0
telepathic = _make_telepathic()
cognitive_stack = {"telepathic": telepathic, "dopamine": dopamine, "dm_memory": MagicMock()}
telepathic = TelepathicEngine.get_instance()
cognitive_stack = {"telepathic": telepathic, "dopamine": dopamine, "dm_memory": None}
send_count = {"n": 0}
original_check_limit = session_state.check_limit
# Override session_state methods that are used in loop directly instead of MagicMock
configs.args.current_success_limit = 8
configs.args.current_pm_limit = 8
def _counting_check(*args, **kwargs):
if send_count["n"] > 20:
pytest.fail(
f"DM Engine sent {send_count['n']} messages without hitting any cap! "
f"Expected a hard limit of <= 5 replies per inbox visit."
)
return (False,)
session_state.totalMessages = 0
session_state.check_limit.side_effect = _counting_check
with patch("GramAddict.core.llm_provider.query_llm", return_value={"response": "Hey!"}), \
patch("GramAddict.core.stealth_typing.ghost_type"), \
patch("GramAddict.core.bot_flow._humanized_click"), \
patch("GramAddict.core.bot_flow.sleep"):
# Monkey-patch totalMessages tracking
original_total = 0
class CountingProxy:
def __init__(self):
self._val = 0
def __iadd__(self, other):
self._val += other
send_count["n"] = self._val
if self._val > 20:
pytest.fail(
f"DM Engine sent {self._val} messages! No iteration guard present."
)
return self
def __int__(self):
return self._val
# Force the session to never hit limits (simulating the real scenario)
result = _run_zero_latency_dm_loop(
device, MagicMock(), MagicMock(), configs, session_state, "MessageInbox", cognitive_stack
)
# Force the session to never hit limits (simulating the real scenario)
result = _run_zero_latency_dm_loop(
device,
make_real_device_with_xml(_make_dm_inbox_xml()),
None,
configs,
session_state,
"MessageInbox",
cognitive_stack,
)
# The engine should have self-limited to at most 5 replies
assert session_state.totalMessages <= 5, (
@@ -444,7 +342,7 @@ class TestBotFlowDMGating:
"""Verifies that bot_flow.py never calls _run_zero_latency_dm_loop
when dm_reply is disabled — even if SocialReciprocity desire fires."""
def test_social_reciprocity_never_includes_message_inbox_when_disabled(self):
def test_social_reciprocity_never_includes_message_inbox_when_disabled(self, make_real_device_with_xml):
"""The target_map for SocialReciprocity should NEVER contain
'MessageInbox' when dm_reply.enabled is false.
@@ -465,11 +363,11 @@ class TestBotFlowDMGating:
if dm_config.get("enabled", False):
target_map["SocialReciprocity"].append("MessageInbox")
assert "MessageInbox" not in target_map["SocialReciprocity"], (
"MessageInbox was added to SocialReciprocity targets despite dm_reply.enabled=false!"
)
assert (
"MessageInbox" not in target_map["SocialReciprocity"]
), "MessageInbox was added to SocialReciprocity targets despite dm_reply.enabled=false!"
def test_social_reciprocity_includes_message_inbox_when_enabled(self):
def test_social_reciprocity_includes_message_inbox_when_enabled(self, make_real_device_with_xml):
"""Positive test: When dm_reply.enabled is true, MessageInbox
SHOULD be in the target map."""
configs = _make_configs(dm_reply_enabled=True)
@@ -484,6 +382,6 @@ class TestBotFlowDMGating:
if dm_config.get("enabled", False):
target_map["SocialReciprocity"].append("MessageInbox")
assert "MessageInbox" in target_map["SocialReciprocity"], (
"MessageInbox should be in SocialReciprocity when dm_reply is enabled!"
)
assert (
"MessageInbox" in target_map["SocialReciprocity"]
), "MessageInbox should be in SocialReciprocity when dm_reply is enabled!"

View File

@@ -5,7 +5,6 @@ Uses REAL XML dumps from production sessions.
"""
import os
from unittest.mock import patch
import pytest
@@ -87,116 +86,88 @@ LOCK_SCREEN_XML = """<?xml version='1.0' encoding='UTF-8' standalone='yes' ?>
# ─────────────────────────────────────────────────────
class DummyDevice:
def __init__(self, app_id="com.instagram.android"):
self.app_id = app_id
self.deviceV2 = None
self._trace_counter = 0
self._trace_dir = "/tmp/test_traces"
def dump_hierarchy(self):
pass
def click(self, x, y):
pass
def press(self, key):
pass
def app_start(self, package, use_monkey=False):
pass
def make_mock_device(app_id="com.instagram.android"):
return DummyDevice(app_id)
# ─────────────────────────────────────────────────────
# PERCEPTION TESTS
# ─────────────────────────────────────────────────────
@pytest.fixture(autouse=True)
def mock_screen_memory():
with patch("GramAddict.core.qdrant_memory.ScreenMemoryDB.get_screen_type", return_value=None):
with patch("GramAddict.core.qdrant_memory.ScreenMemoryDB.store_screen"):
yield
# Removed mock_screen_memory fixture to allow real Qdrant database interactions
class TestSAEPerception:
"""Tests that the SAE correctly classifies screen situations."""
def test_perceive_normal_instagram(self):
device = make_mock_device()
def test_perceive_normal_instagram(self, make_real_device_with_xml):
device = make_real_device_with_xml("")
sae = SituationalAwarenessEngine(device)
result = sae.perceive(INSTAGRAM_HOME_XML)
assert result == SituationType.NORMAL
def test_perceive_foreign_app_google(self):
device = make_mock_device()
def test_perceive_foreign_app_google(self, make_real_device_with_xml):
device = make_real_device_with_xml("")
sae = SituationalAwarenessEngine(device)
result = sae.perceive(GOOGLE_SEARCH_XML)
assert result == SituationType.OBSTACLE_FOREIGN_APP
def test_perceive_notification_shade(self):
def test_perceive_notification_shade(self, make_real_device_with_xml):
import os
dump_path = os.path.join(os.path.dirname(__file__), "..", "fixtures", "notification_shade.xml")
try:
with open(dump_path, "r") as f:
shade_xml = f.read()
device = make_mock_device()
device = make_real_device_with_xml("")
sae = SituationalAwarenessEngine(device)
result = sae.perceive(shade_xml)
assert result == SituationType.OBSTACLE_FOREIGN_APP
except FileNotFoundError:
pass # allow test format to compile if fixture accidentally not available
def test_perceive_system_permission_dialog(self):
device = make_mock_device()
def test_perceive_system_permission_dialog(self, make_real_device_with_xml):
device = make_real_device_with_xml("")
sae = SituationalAwarenessEngine(device)
result = sae.perceive(PERMISSION_DIALOG_XML)
assert result == SituationType.OBSTACLE_SYSTEM
def test_perceive_instagram_survey_modal(self):
device = make_mock_device()
def test_perceive_instagram_survey_modal(self, make_real_device_with_xml):
device = make_real_device_with_xml("")
sae = SituationalAwarenessEngine(device)
result = sae.perceive(INSTAGRAM_SURVEY_XML)
assert result == SituationType.OBSTACLE_MODAL
@patch("GramAddict.core.llm_provider.query_telepathic_llm", return_value='{"situation": "OBSTACLE_MODAL"}')
def test_perceive_unknown_modal_interstitial(self, mock_llm):
def test_perceive_unknown_modal_interstitial(self, make_real_device_with_xml):
"""SAE must detect modals it has NEVER seen before — no hardcoded IDs."""
device = make_mock_device()
device = make_real_device_with_xml("")
sae = SituationalAwarenessEngine(device)
sae.unlearn_current_state(UNKNOWN_MODAL_XML)
result = sae.perceive(UNKNOWN_MODAL_XML)
assert result == SituationType.OBSTACLE_MODAL
def test_perceive_action_blocked(self):
def test_perceive_action_blocked(self, make_real_device_with_xml):
blocked_xml = INSTAGRAM_HOME_XML.replace(
'text="" resource-id="com.instagram.android:id/feed_tab"',
'text="Try again later" resource-id="com.instagram.android:id/bottom_sheet_container"',
)
device = make_mock_device()
device = make_real_device_with_xml("")
sae = SituationalAwarenessEngine(device)
result = sae.perceive(blocked_xml)
assert result == SituationType.DANGER_ACTION_BLOCKED
def test_perceive_empty_dump(self):
device = make_mock_device()
def test_perceive_empty_dump(self, make_real_device_with_xml):
device = make_real_device_with_xml("")
sae = SituationalAwarenessEngine(device)
result = sae.perceive("")
assert result == SituationType.OBSTACLE_FOREIGN_APP
def test_perceive_none_dump(self):
device = make_mock_device()
def test_perceive_none_dump(self, make_real_device_with_xml):
device = make_real_device_with_xml("")
sae = SituationalAwarenessEngine(device)
result = sae.perceive(None)
assert result == SituationType.OBSTACLE_FOREIGN_APP
def test_perceive_passive_scaffold_as_normal(self):
def test_perceive_passive_scaffold_as_normal(self, make_real_device_with_xml):
"""Passive scaffold containers (bottom_sheet_container_view, bottom_sheet_camera_container) must NOT be OBSTACLE_MODAL."""
device = make_mock_device()
device = make_real_device_with_xml("")
sae = SituationalAwarenessEngine(device)
# XML containing navigation tabs + the passive scaffold container
@@ -228,58 +199,58 @@ def _load_fixture(name: str) -> str:
class TestSAERealFixturePerception:
"""Tests perceive() against REAL production XML dumps to prevent false-positive obstacles."""
def test_perceive_home_feed_as_normal(self):
def test_perceive_home_feed_as_normal(self, make_real_device_with_xml):
"""Real home feed XML (with ads, stories tray) must be NORMAL — zero LLM calls."""
device = make_mock_device()
device = make_real_device_with_xml("")
sae = SituationalAwarenessEngine(device)
xml = _load_fixture("home_feed_real.xml")
result = sae.perceive(xml)
assert result == SituationType.NORMAL, f"Home feed misclassified as {result}"
def test_perceive_explore_grid_as_normal(self):
def test_perceive_explore_grid_as_normal(self, make_real_device_with_xml):
"""Real explore grid XML must be NORMAL — zero LLM calls."""
device = make_mock_device()
device = make_real_device_with_xml("")
sae = SituationalAwarenessEngine(device)
xml = _load_fixture("explore_grid_real.xml")
result = sae.perceive(xml)
assert result == SituationType.NORMAL, f"Explore grid misclassified as {result}"
def test_perceive_other_profile_as_normal(self):
def test_perceive_other_profile_as_normal(self, make_real_device_with_xml):
"""Real other-user profile XML must be NORMAL — zero LLM calls."""
device = make_mock_device()
device = make_real_device_with_xml("")
sae = SituationalAwarenessEngine(device)
xml = _load_fixture("other_profile_real.xml")
result = sae.perceive(xml)
assert result == SituationType.NORMAL, f"Other profile misclassified as {result}"
def test_perceive_post_detail_as_normal(self):
def test_perceive_post_detail_as_normal(self, make_real_device_with_xml):
"""Real post detail XML must be NORMAL — zero LLM calls."""
device = make_mock_device()
device = make_real_device_with_xml("")
sae = SituationalAwarenessEngine(device)
xml = _load_fixture("post_detail_real.xml")
result = sae.perceive(xml)
assert result == SituationType.NORMAL, f"Post detail misclassified as {result}"
def test_perceive_profile_tagged_tab_as_normal(self):
def test_perceive_profile_tagged_tab_as_normal(self, make_real_device_with_xml):
"""Real profile tagged-tab XML must be NORMAL — zero LLM calls."""
device = make_mock_device()
device = make_real_device_with_xml("")
sae = SituationalAwarenessEngine(device)
xml = _load_fixture("profile_tagged_tab.xml")
result = sae.perceive(xml)
assert result == SituationType.NORMAL, f"Profile tagged tab misclassified as {result}"
def test_perceive_survey_modal_as_obstacle(self):
def test_perceive_survey_modal_as_obstacle(self, make_real_device_with_xml):
"""Inline survey modal XML (with survey_overlay_container) must be OBSTACLE_MODAL."""
device = make_mock_device()
device = make_real_device_with_xml("")
sae = SituationalAwarenessEngine(device)
result = sae.perceive(INSTAGRAM_SURVEY_XML)
assert result == SituationType.OBSTACLE_MODAL, f"Survey modal misclassified as {result}"
@patch("GramAddict.core.llm_provider.query_telepathic_llm", return_value='{"situation": "OBSTACLE_MODAL"}')
def test_perceive_mystery_interstitial_as_obstacle(self, mock_llm):
def test_perceive_mystery_interstitial_as_obstacle(self, make_real_device_with_xml):
"""Inline interstitial modal XML must be OBSTACLE_MODAL."""
device = make_mock_device()
device = make_real_device_with_xml("")
sae = SituationalAwarenessEngine(device)
sae.unlearn_current_state(UNKNOWN_MODAL_XML)
result = sae.perceive(UNKNOWN_MODAL_XML)
assert result == SituationType.OBSTACLE_MODAL, f"Mystery interstitial misclassified as {result}"
@@ -325,13 +296,13 @@ class TestStoryViewDetection:
f"Expected STORY_VIEW but ScreenIdentity returned {result['screen_type'].name}."
)
def test_sae_perceive_story_as_normal(self):
def test_sae_perceive_story_as_normal(self, make_real_device_with_xml):
"""SAE must classify Story views as NORMAL (it's Instagram, not an obstacle).
The bot's reaction to a Story should be: press back → navigate away.
But first, SAE must NOT flag it as an obstacle.
"""
device = make_mock_device()
device = make_real_device_with_xml("")
sae = SituationalAwarenessEngine(device)
xml = _load_fixture("story_view_full.xml")
result = sae.perceive(xml)
@@ -340,15 +311,13 @@ class TestStoryViewDetection:
def test_story_view_available_actions_include_press_back(self):
"""On a story, 'press back' must be in available actions and 'scroll down' should NOT
be a meaningful action (stories don't scroll, they swipe)."""
from GramAddict.core.perception.screen_identity import ScreenIdentity, ScreenType
from GramAddict.core.perception.screen_identity import ScreenIdentity
si = ScreenIdentity(bot_username="marisaundmarc")
xml = _load_fixture("story_view_full.xml")
result = si.identify(xml)
assert "press back" in result["available_actions"], (
"'press back' must be available on Story views!"
)
assert "press back" in result["available_actions"], "'press back' must be available on Story views!"
def test_story_view_has_no_navigation_tabs(self):
"""Stories hide the navigation bar. The available actions must NOT
@@ -360,7 +329,4 @@ class TestStoryViewDetection:
result = si.identify(xml)
tab_actions = [a for a in result["available_actions"] if "tap" in a and "tab" in a]
assert len(tab_actions) == 0, (
f"Story view should have NO tab navigation, but found: {tab_actions}"
)
assert len(tab_actions) == 0, f"Story view should have NO tab navigation, but found: {tab_actions}"

View File

@@ -12,8 +12,6 @@ These tests prove:
Requires: Real XML fixture at tests/fixtures/user_profile_dump.xml
"""
from unittest.mock import patch
import pytest
from GramAddict.core.navigation.planner import GoalPlanner
@@ -41,19 +39,17 @@ def test_goap_planner_avoids_infinite_loop_on_masked_edge():
}
# NORMAL: HD Map routes via OWN_PROFILE
with patch("GramAddict.core.navigation.brain.query_llm", return_value=None):
action_normal = planner.plan_next_step("open following list", screen)
action_normal = planner.plan_next_step("open following list", screen)
assert action_normal == "tap profile tab", "HD Map sollte primär über OWN_PROFILE routen"
# MASKED: simulate that "tap following list" failed >= 2 times
action_failures = {"tap following list": 2}
with patch("GramAddict.core.navigation.brain.query_llm", return_value=None):
action_avoided = planner.plan_next_step(
"open following list",
screen,
action_failures=action_failures,
)
action_avoided = planner.plan_next_step(
"open following list",
screen,
action_failures=action_failures,
)
assert action_avoided != "tap profile tab", "Planner routed BLIND into the dead end despite the edge being masked!"
@@ -223,7 +219,7 @@ def test_vlm_prompt_humanizes_content_desc():
@pytest.mark.live_llm
def test_live_vlm_selects_following_not_followers():
def test_live_vlm_selects_following_not_followers(make_real_device_with_image):
"""
LIVE LLM TEST: Calls the real local Ollama to prove the VLM
correctly picks the 'following' node (not 'followers') when asked
@@ -257,15 +253,9 @@ def test_live_vlm_selects_following_not_followers():
root = engine._parser.parse(xml)
candidates = engine._parser.get_clickable_nodes(root)
class DummyDeviceV2:
def screenshot(self):
return dummy_img
device = make_real_device_with_image(dummy_img)
class DummyDevice:
def __init__(self):
self.deviceV2 = DummyDeviceV2()
annotated_b64, box_map = resolver._annotate_screenshot_with_candidates(DummyDevice(), candidates)
annotated_b64, box_map = resolver._annotate_screenshot_with_candidates(device, candidates)
# Convert box_map back to a flat list for testing indexing
filtered = list(box_map.values())

View File

@@ -28,32 +28,12 @@ def _load_profile_xml():
return f.read()
def _make_device_with_real_image(img_path):
"""Creates a mock device that returns the REAL screenshot captured from the device."""
from PIL import Image
img = Image.open(img_path)
class DummyDeviceV2:
def __init__(self, img):
self.img = img
def screenshot(self):
return self.img
class DummyDevice:
def __init__(self, img):
self.deviceV2 = DummyDeviceV2(img)
return DummyDevice(img)
# ═══════════════════════════════════════════════════════
# TEST 1: Visual Discovery produces an annotated image
# ═══════════════════════════════════════════════════════
def test_visual_discovery_creates_annotated_screenshot():
def test_visual_discovery_creates_annotated_screenshot(make_real_device_with_image):
"""
The IntentResolver's visual discovery mode must:
1. Take a screenshot from the device
@@ -68,7 +48,7 @@ def test_visual_discovery_creates_annotated_screenshot():
root = parser.parse(xml)
candidates = parser.get_clickable_nodes(root)
device = _make_device_with_real_image("tests/fixtures/user_profile_dump.jpg")
device = make_real_device_with_image("tests/fixtures/user_profile_dump.jpg")
resolver = IntentResolver()
annotated_b64, box_map = resolver._annotate_screenshot_with_candidates(device, candidates)
@@ -111,7 +91,7 @@ def test_visual_discovery_creates_annotated_screenshot():
@pytest.mark.live_llm
def test_visual_discovery_finds_following_by_seeing():
def test_visual_discovery_finds_following_by_seeing(make_real_device_with_image):
"""
LIVE VLM TEST: The bot SEES a screenshot with numbered boxes
and visually identifies which box is the "following" counter.
@@ -124,7 +104,7 @@ def test_visual_discovery_finds_following_by_seeing():
root = parser.parse(xml)
candidates = parser.get_clickable_nodes(root)
device = _make_device_with_real_image("tests/fixtures/user_profile_dump.jpg")
device = make_real_device_with_image("tests/fixtures/user_profile_dump.jpg")
resolver = IntentResolver()
# Visual Discovery: Let the VLM SEE the screen