diff --git a/GramAddict/core/perception/intent_resolver.py b/GramAddict/core/perception/intent_resolver.py index 4decb53..a3574fb 100644 --- a/GramAddict/core/perception/intent_resolver.py +++ b/GramAddict/core/perception/intent_resolver.py @@ -254,11 +254,27 @@ class IntentResolver: and "per cent" not in (n.content_desc or "").lower() ] + # --- Structural Navigation Guard --- + # For strictly defined navigation tabs, we bypass the VLM entirely. + tab_mapping = { + "tap home tab": "feed_tab", + "tap explore tab": "search_tab", + "tap reels tab": "clips_tab", + "tap profile tab": "profile_tab", + "tap messages tab": "direct_tab", + } + intent_lower = intent_description.lower() + if intent_lower in tab_mapping: + target_id = tab_mapping[intent_lower] + for node in candidates: + if target_id in (node.resource_id or ""): + logger.info(f"🎯 [Navigation Guard] Exact match found for '{intent_lower}' via resource-id. Skipping VLM.") + return node + # --- Strict Button Guard --- # If the intent specifically asks for a "button", "icon", or "tab", # filter out candidates that contain long text (e.g. captions, comments) # to prevent the VLM from hallucinating text nodes as interactive buttons. - intent_lower = intent_description.lower() if "button" in intent_lower or "icon" in intent_lower or "tab" in intent_lower: filtered_candidates = [] for node in candidates: diff --git a/tests/e2e/test_visual_intent_resolver.py b/tests/e2e/test_visual_intent_resolver.py index c30bc98..0850ebd 100644 --- a/tests/e2e/test_visual_intent_resolver.py +++ b/tests/e2e/test_visual_intent_resolver.py @@ -133,18 +133,26 @@ def test_visual_discovery_finds_following_by_seeing(make_real_device_with_image) # ═══════════════════════════════════════════════════════ -def test_resolve_uses_structural_path_when_no_device(make_real_device_with_xml): +def test_structural_navigation_guard_bypasses_vlm(make_real_device_with_xml): """ - When called WITHOUT a device (device=None), resolve() must fall back - to the structural XML-only path instead of visual discovery. - This proves the routing logic works: visual is primary, structural is fallback. + TDD PROOF: The Structural Navigation Guard must intercept bottom-nav actions + (like "tap profile tab") and directly return the matching candidate based on + resource-id, entirely bypassing the VLM and Set-of-Mark visual annotation. """ from GramAddict.core.perception.spatial_parser import SpatialNode resolver = IntentResolver() - # A single candidate with a clear profile_tab match + # Provide candidates including a trap (Profile picture) and the real tab candidates = [ + SpatialNode( + resource_id="com.instagram.android:id/row_feed_photo_profile_imageview", + class_name="android.widget.ImageView", + text="", + content_desc="Profile picture of some user", + bounds=(0, 0, 100, 100), + clickable=True, + ), SpatialNode( resource_id="com.instagram.android:id/profile_tab", class_name="android.widget.FrameLayout", @@ -155,7 +163,14 @@ def test_resolve_uses_structural_path_when_no_device(make_real_device_with_xml): ) ] - # Without device, resolve must still work via structural matching - result = resolver.resolve("tap profile tab", candidates, screen_height=2400) - assert result is not None, "Structural fallback failed to find profile_tab without a device" - assert result.resource_id == "com.instagram.android:id/profile_tab" + # Device IS provided. Without the structural guard, this would trigger + # visual discovery (which fails in tests without an image). + device = make_real_device_with_xml("tests/fixtures/home_feed_with_ad.xml") + + # We resolve the intent + result = resolver.resolve("tap profile tab", candidates, device=device, screen_height=2400) + + assert result is not None, "Structural Navigation Guard failed to find the profile tab" + assert result.resource_id == "com.instagram.android:id/profile_tab", ( + f"Guard picked the wrong node! Selected: {result.resource_id}" + ) diff --git a/tests/unit/perception/test_screen_identity.py b/tests/unit/perception/test_screen_identity.py new file mode 100644 index 0000000..3ad7092 --- /dev/null +++ b/tests/unit/perception/test_screen_identity.py @@ -0,0 +1,25 @@ +import pytest +from GramAddict.core.perception.screen_identity import ScreenIdentity, ScreenType + +def test_identify_explore_grid_without_selected_tab(): + """ + TDD Proof: Ensure ScreenIdentity classifies EXPLORE_GRID correctly + even if the tab bar fails to report selected="true". + """ + # XML snippet containing the search bar and the search tab, but NO selected tabs. + xml_dump = """ + + + + + + + """ + + identity = ScreenIdentity("testbot") + result = identity.identify(xml_dump) + + assert result["screen_type"] == ScreenType.EXPLORE_GRID, ( + f"Expected EXPLORE_GRID, but got {result['screen_type']}. " + f"Structural heuristic failed to recognize search_edit_text + search_tab!" + )