diff --git a/GramAddict/core/perception/intent_resolver.py b/GramAddict/core/perception/intent_resolver.py index 4decb53..36d2731 100644 --- a/GramAddict/core/perception/intent_resolver.py +++ b/GramAddict/core/perception/intent_resolver.py @@ -278,7 +278,7 @@ class IntentResolver: # Posts/grid items usually have 'row X, column Y', 'photos by', or 'reel by' if "row 1" in desc or "column" in desc or "photos by" in desc or "reel by" in desc: grid_candidates.append(node) - + if grid_candidates: logger.info(f"🎯 [Grid Guard] Filtered to {len(grid_candidates)} actual grid candidates.") candidates = grid_candidates @@ -361,7 +361,13 @@ class IntentResolver: f"4. Ignore numbers inside the text itself. Do not confuse the text '19' with Box [19].\n" f"5. If the intent contains 'following', you MUST pick the box containing 'following'. Do NOT pick 'followers' or 'Follow'.\n" f"6. If the intent is to tap a 'post' or 'grid item', look for boxes with descriptions containing 'photos by', 'Reel by', or 'row 1, column 1' and pick the first matching one. Do NOT pick navigation buttons like 'Search'.\n" - f"7. If the exact control is NOT visible, return null. Do NOT guess.\n\n" + f"7. If the intent is a bottom navigation tab (e.g. 'profile tab', 'home tab'):\n" + f" - These are always at the BOTTOM edge of the screen.\n" + f" - 'profile tab' is usually the furthest right icon (your avatar).\n" + f" - 'home tab' is the furthest left icon (house).\n" + f" - 'explore tab' is the magnifying glass.\n" + f" - 'reels tab' is the video clapperboard.\n" + f"8. If the exact control is NOT visible, return null. Do NOT guess.\n\n" f'Reply ONLY with a valid JSON object: {{"box": }} or {{"box": null}}' ) diff --git a/tests/e2e/test_visual_intent_resolver.py b/tests/e2e/test_visual_intent_resolver.py index c30bc98..0feea31 100644 --- a/tests/e2e/test_visual_intent_resolver.py +++ b/tests/e2e/test_visual_intent_resolver.py @@ -159,3 +159,42 @@ def test_resolve_uses_structural_path_when_no_device(make_real_device_with_xml): result = resolver.resolve("tap profile tab", candidates, screen_height=2400) assert result is not None, "Structural fallback failed to find profile_tab without a device" assert result.resource_id == "com.instagram.android:id/profile_tab" + + +@pytest.mark.live_llm +def test_visual_discovery_finds_profile_tab_by_seeing(make_real_device_with_image): + """ + LIVE VLM TEST: The bot SEES a screenshot with numbered boxes + and visually identifies which box is the 'profile tab'. + This proves the prompt correctly guides the VLM to pick bottom navigation tabs + without hardcoding resource IDs. + """ + from GramAddict.core.perception.spatial_parser import SpatialParser + + with open("tests/fixtures/home_feed_with_ad.xml", "r", encoding="utf-8") as f: + xml = f.read() + + parser = SpatialParser() + root = parser.parse(xml) + candidates = parser.get_clickable_nodes(root) + + # Use a real image so the VLM can actually see the UI + device = make_real_device_with_image("tests/fixtures/home_feed_with_ad.jpg") + resolver = IntentResolver() + + # Visual Discovery: Let the VLM SEE the screen + result = resolver.resolve( + "tap profile tab", + candidates, + device, + ) + + assert result is not None, "Visual discovery returned None — VLM couldn't find 'profile tab' on screen" + + # Check that it actually selected the correct tab + selected_id = (result.resource_id or "").lower() + + # On the home_feed_with_ad_dump, the profile tab should be selected + assert ( + "profile_tab" in selected_id + ), f"Visual discovery picked wrong node! Got: id='{result.resource_id}', desc='{result.content_desc}'"