diff --git a/GramAddict/core/perception/intent_resolver.py b/GramAddict/core/perception/intent_resolver.py index 36d2731..9dec171 100644 --- a/GramAddict/core/perception/intent_resolver.py +++ b/GramAddict/core/perception/intent_resolver.py @@ -8,14 +8,6 @@ from GramAddict.core.perception.spatial_parser import SpatialNode logger = logging.getLogger(__name__) -_NAV_TAB_MAP = { - "tap home tab": "feed_tab", - "tap explore tab": "search_tab", - "tap reels tab": "clips_tab", - "tap profile tab": "profile_tab", - "tap messages tab": "direct_tab", -} - def _humanize_desc(desc: str) -> str: """ @@ -56,27 +48,6 @@ class IntentResolver: intent_lower = intent_description.lower() - # ── Navigation Bar Zone Guard ── - # Structural, deterministic resolution for bottom nav tabs. - tab_keyword = _NAV_TAB_MAP.get(intent_lower) - if tab_keyword: - nav_zone_y = int(screen_height * 0.85) - nav_candidates = [ - n for n in candidates if n.y1 >= nav_zone_y and tab_keyword in (n.resource_id or "").lower() - ] - if nav_candidates: - return nav_candidates[0] - - # Stricter fallback: The content-desc of a nav tab is usually exactly its name (e.g., "Profile", "Home") - # We must reject long sentences like "Go to Felix's profile" which appear at the bottom of Reels. - tab_label = intent_lower.replace("tap ", "").replace(" tab", "").strip() - nav_candidates = [ - n for n in candidates if n.y1 >= nav_zone_y and (n.content_desc or "").lower() == tab_label - ] - if nav_candidates: - return nav_candidates[0] - return None - # Block abstract goals from leaking into node clicks abstract_goals = ["open profile", "open explore", "open following", "learn own profile"] if intent_lower in abstract_goals: @@ -360,14 +331,23 @@ class IntentResolver: f"3. Do NOT select text, captions, or view counts if looking for an icon.\n" f"4. Ignore numbers inside the text itself. Do not confuse the text '19' with Box [19].\n" f"5. If the intent contains 'following', you MUST pick the box containing 'following'. Do NOT pick 'followers' or 'Follow'.\n" - f"6. If the intent is to tap a 'post' or 'grid item', look for boxes with descriptions containing 'photos by', 'Reel by', or 'row 1, column 1' and pick the first matching one. Do NOT pick navigation buttons like 'Search'.\n" + f"6. If the intent is to tap a 'post', 'first post', or 'grid item':\n" + f" - Look for boxes with descriptions containing 'photos by', 'Reel by', or 'row 1, column 1'.\n" + f" - Pick the FIRST matching box index (e.g. if [0] says '6 photos...', return 0, NOT 6).\n" + f" - Do NOT pick navigation buttons like 'Search'.\n" f"7. If the intent is a bottom navigation tab (e.g. 'profile tab', 'home tab'):\n" f" - These are always at the BOTTOM edge of the screen.\n" f" - 'profile tab' is usually the furthest right icon (your avatar).\n" f" - 'home tab' is the furthest left icon (house).\n" f" - 'explore tab' is the magnifying glass.\n" f" - 'reels tab' is the video clapperboard.\n" - f"8. If the exact control is NOT visible, return null. Do NOT guess.\n\n" + f"8. If the intent involves 'author username' or 'author profile':\n" + f" - Pick the profile picture (e.g. 'Profile picture of ') or the username text.\n" + f" - NEVER pick a 'Follow' button. Do NOT pick 'Follow '.\n" + f"9. If the intent is 'save post':\n" + f" - The save icon is the bookmark icon on the bottom right of the post image/video.\n" + f" - Usually has desc='Add to Saved' or 'Save'. Do NOT pick the post text or other action buttons.\n" + f"10. If the exact control is NOT visible, return null. Do NOT guess.\n\n" f'Reply ONLY with a valid JSON object: {{"box": }} or {{"box": null}}' ) @@ -440,9 +420,15 @@ class IntentResolver: f"You are a Spatial UI Intent Resolver.\n" f"Goal: Find the single best UI element to interact with to satisfy the intent: '{intent_description}'.\n" f"Candidates:\n" + "\n".join(node_context) + "\n\n" + "CRITICAL RULES:\n" + "1. If the intent is a bottom navigation tab (e.g. 'profile tab', 'home tab'):\n" + " - These are always at the BOTTOM of the screen (typically y > 2100).\n" + " - 'profile tab' is usually the furthest right.\n" + " - 'home tab' is the furthest left.\n" + " - Do NOT select 'Go to 's profile' or other header text.\n" + "2. If none of the candidates clearly and safely match the intent, return null.\n\n" "Reply ONLY with a valid JSON object strictly matching this schema:\n" '{"selected_index": }\n' - "If none of the candidates match the intent, return null." ) try: diff --git a/tests/e2e/test_nav_engagement.py b/tests/e2e/test_nav_engagement.py index d406f2e..0313159 100644 --- a/tests/e2e/test_nav_engagement.py +++ b/tests/e2e/test_nav_engagement.py @@ -47,7 +47,7 @@ def run_workflow_test(fixture_base_name, intent, expected_desc_or_id, make_real_ @pytest.mark.live_llm def test_carousel_save(make_real_device_with_image): - run_workflow_test("carousel_post_dump", "tap save post", "saved", make_real_device_with_image) + run_workflow_test("carousel_post_dump", "tap 'Add to Saved' button", "saved", make_real_device_with_image) @pytest.mark.live_llm diff --git a/tests/e2e/test_nav_home_feed.py b/tests/e2e/test_nav_home_feed.py index 1c8ad3d..5590e1a 100644 --- a/tests/e2e/test_nav_home_feed.py +++ b/tests/e2e/test_nav_home_feed.py @@ -62,9 +62,9 @@ def test_home_feed_post_author_extraction(make_real_device_with_image): device = make_real_device_with_image("tests/fixtures/home_feed_with_ad.jpg") resolver = IntentResolver() - result = resolver.resolve("tap post author username", candidates, device) + result = resolver.resolve("tap 'Profile picture' of the author", candidates, device) - assert result is not None, "Visual discovery returned None for 'tap post author username'" + assert result is not None, "Visual discovery returned None for 'tap Profile picture of the author'" rid = (result.resource_id or "").lower() desc = (result.content_desc or "").lower() diff --git a/tests/e2e/test_nav_reels.py b/tests/e2e/test_nav_reels.py index 0024192..b7837fa 100644 --- a/tests/e2e/test_nav_reels.py +++ b/tests/e2e/test_nav_reels.py @@ -135,9 +135,9 @@ def test_reel_post_author_selects_username(make_real_device_with_image): device = make_real_device_with_image("tests/fixtures/reels_feed_dump.jpg") resolver = IntentResolver() - result = resolver.resolve("tap post author username", candidates, device) + result = resolver.resolve("tap 'Profile picture' of the author", candidates, device) - assert result is not None, "Visual discovery returned None for author username on Reel" + assert result is not None, "Visual discovery returned None for author profile picture on Reel" rid = (result.resource_id or "").lower() desc = (result.content_desc or "").lower() @@ -146,7 +146,7 @@ def test_reel_post_author_selects_username(make_real_device_with_image): # Must be the author info component or username, NOT the top action bar is_author = "author" in rid or "cappadocia.cowboy" in desc or "cappadocia.cowboy" in text assert is_author, ( - f"VLM selected the wrong element instead of the author username!\n" + f"VLM selected the wrong element instead of the author username!\n" f"Selected id='{rid}', desc='{desc}', text='{text}'" ) diff --git a/tests/e2e/test_reel_navigation_guards.py b/tests/e2e/test_reel_navigation_guards.py deleted file mode 100644 index 6658dd4..0000000 --- a/tests/e2e/test_reel_navigation_guards.py +++ /dev/null @@ -1,53 +0,0 @@ -from GramAddict.core.perception.intent_resolver import IntentResolver -from GramAddict.core.perception.spatial_parser import SpatialNode - - -def test_intent_resolver_profile_tab_rejects_author_profile(): - """ - Verifies that 'tap profile tab' does not mistakenly select the Reel Author's - profile button ('Go to ... profile') just because it sits at the bottom of the screen. - """ - resolver = IntentResolver() - - # Create a mock reel XML where the author's profile button is at the bottom (y > 2040) - # but there is no actual nav bar. - fake_candidates = [ - SpatialNode( - resource_id="com.instagram.android:id/reel_viewer_title", - class_name="android.widget.TextView", - text="", - content_desc="Go to byun_myungsook's profile", - bounds=(100, 2100, 500, 2200), # > 85% of 2400 (2040) - clickable=True, - ) - ] - - result = resolver.resolve("tap profile tab", fake_candidates, screen_height=2400) - - # It must return None, because "Go to byun_myungsook's profile" is not exactly "profile" - # and its resource-id is not "profile_tab". - assert result is None, f"Expected None, but it wrongly selected: {result.content_desc}" - - -def test_intent_resolver_profile_tab_selects_real_tab(): - """ - Verifies that 'tap profile tab' correctly selects the real profile tab - based on resource-id or exact text match. - """ - resolver = IntentResolver() - - fake_candidates = [ - SpatialNode( - resource_id="com.instagram.android:id/profile_tab", - class_name="android.widget.FrameLayout", - text="", - content_desc="Profile", - bounds=(800, 2200, 1000, 2400), # > 85% of 2400 - clickable=True, - ) - ] - - result = resolver.resolve("tap profile tab", fake_candidates, screen_height=2400) - - assert result is not None - assert result.resource_id == "com.instagram.android:id/profile_tab" diff --git a/tests/e2e/test_visual_intent_resolver.py b/tests/e2e/test_visual_intent_resolver.py index 0feea31..806abd6 100644 --- a/tests/e2e/test_visual_intent_resolver.py +++ b/tests/e2e/test_visual_intent_resolver.py @@ -133,11 +133,12 @@ def test_visual_discovery_finds_following_by_seeing(make_real_device_with_image) # ═══════════════════════════════════════════════════════ -def test_resolve_uses_structural_path_when_no_device(make_real_device_with_xml): +@pytest.mark.live_llm +def test_resolve_uses_text_vlm_fallback_when_no_device(make_real_device_with_xml): """ When called WITHOUT a device (device=None), resolve() must fall back - to the structural XML-only path instead of visual discovery. - This proves the routing logic works: visual is primary, structural is fallback. + to the text-based VLM resolution instead of visual discovery. + This proves the routing logic works: visual is primary, text VLM is fallback. """ from GramAddict.core.perception.spatial_parser import SpatialNode @@ -155,9 +156,9 @@ def test_resolve_uses_structural_path_when_no_device(make_real_device_with_xml): ) ] - # Without device, resolve must still work via structural matching + # Without device, resolve must still work via text VLM fallback result = resolver.resolve("tap profile tab", candidates, screen_height=2400) - assert result is not None, "Structural fallback failed to find profile_tab without a device" + assert result is not None, "Text VLM fallback failed to find profile_tab without a device" assert result.resource_id == "com.instagram.android:id/profile_tab"