From b9c29a5a2d4c7ebaad7dce248087bd8f939faacc Mon Sep 17 00:00:00 2001 From: Marc Mintel Date: Wed, 29 Apr 2026 01:16:06 +0200 Subject: [PATCH] fix(navigation): eliminate VLM hallucination on bottom navigation tabs via structural guard Added a 'Structural Navigation Guard' in IntentResolver to map critical bottom navigation intents (e.g., 'tap profile tab') directly to their stable resource-ids. This bypasses the VLM entirely, guaranteeing 100% deterministic clicks and resolving the issue where the VLM failed to locate the profile tab, causing the edge to become masked and trapping the bot on the home feed. Included TDD proof. --- GramAddict/core/perception/intent_resolver.py | 18 +++++++++- tests/e2e/test_visual_intent_resolver.py | 33 ++++++++++++++----- tests/unit/perception/test_screen_identity.py | 25 ++++++++++++++ 3 files changed, 66 insertions(+), 10 deletions(-) create mode 100644 tests/unit/perception/test_screen_identity.py diff --git a/GramAddict/core/perception/intent_resolver.py b/GramAddict/core/perception/intent_resolver.py index 4decb53..a3574fb 100644 --- a/GramAddict/core/perception/intent_resolver.py +++ b/GramAddict/core/perception/intent_resolver.py @@ -254,11 +254,27 @@ class IntentResolver: and "per cent" not in (n.content_desc or "").lower() ] + # --- Structural Navigation Guard --- + # For strictly defined navigation tabs, we bypass the VLM entirely. + tab_mapping = { + "tap home tab": "feed_tab", + "tap explore tab": "search_tab", + "tap reels tab": "clips_tab", + "tap profile tab": "profile_tab", + "tap messages tab": "direct_tab", + } + intent_lower = intent_description.lower() + if intent_lower in tab_mapping: + target_id = tab_mapping[intent_lower] + for node in candidates: + if target_id in (node.resource_id or ""): + logger.info(f"🎯 [Navigation Guard] Exact match found for '{intent_lower}' via resource-id. Skipping VLM.") + return node + # --- Strict Button Guard --- # If the intent specifically asks for a "button", "icon", or "tab", # filter out candidates that contain long text (e.g. captions, comments) # to prevent the VLM from hallucinating text nodes as interactive buttons. - intent_lower = intent_description.lower() if "button" in intent_lower or "icon" in intent_lower or "tab" in intent_lower: filtered_candidates = [] for node in candidates: diff --git a/tests/e2e/test_visual_intent_resolver.py b/tests/e2e/test_visual_intent_resolver.py index c30bc98..0850ebd 100644 --- a/tests/e2e/test_visual_intent_resolver.py +++ b/tests/e2e/test_visual_intent_resolver.py @@ -133,18 +133,26 @@ def test_visual_discovery_finds_following_by_seeing(make_real_device_with_image) # ═══════════════════════════════════════════════════════ -def test_resolve_uses_structural_path_when_no_device(make_real_device_with_xml): +def test_structural_navigation_guard_bypasses_vlm(make_real_device_with_xml): """ - When called WITHOUT a device (device=None), resolve() must fall back - to the structural XML-only path instead of visual discovery. - This proves the routing logic works: visual is primary, structural is fallback. + TDD PROOF: The Structural Navigation Guard must intercept bottom-nav actions + (like "tap profile tab") and directly return the matching candidate based on + resource-id, entirely bypassing the VLM and Set-of-Mark visual annotation. """ from GramAddict.core.perception.spatial_parser import SpatialNode resolver = IntentResolver() - # A single candidate with a clear profile_tab match + # Provide candidates including a trap (Profile picture) and the real tab candidates = [ + SpatialNode( + resource_id="com.instagram.android:id/row_feed_photo_profile_imageview", + class_name="android.widget.ImageView", + text="", + content_desc="Profile picture of some user", + bounds=(0, 0, 100, 100), + clickable=True, + ), SpatialNode( resource_id="com.instagram.android:id/profile_tab", class_name="android.widget.FrameLayout", @@ -155,7 +163,14 @@ def test_resolve_uses_structural_path_when_no_device(make_real_device_with_xml): ) ] - # Without device, resolve must still work via structural matching - result = resolver.resolve("tap profile tab", candidates, screen_height=2400) - assert result is not None, "Structural fallback failed to find profile_tab without a device" - assert result.resource_id == "com.instagram.android:id/profile_tab" + # Device IS provided. Without the structural guard, this would trigger + # visual discovery (which fails in tests without an image). + device = make_real_device_with_xml("tests/fixtures/home_feed_with_ad.xml") + + # We resolve the intent + result = resolver.resolve("tap profile tab", candidates, device=device, screen_height=2400) + + assert result is not None, "Structural Navigation Guard failed to find the profile tab" + assert result.resource_id == "com.instagram.android:id/profile_tab", ( + f"Guard picked the wrong node! Selected: {result.resource_id}" + ) diff --git a/tests/unit/perception/test_screen_identity.py b/tests/unit/perception/test_screen_identity.py new file mode 100644 index 0000000..3ad7092 --- /dev/null +++ b/tests/unit/perception/test_screen_identity.py @@ -0,0 +1,25 @@ +import pytest +from GramAddict.core.perception.screen_identity import ScreenIdentity, ScreenType + +def test_identify_explore_grid_without_selected_tab(): + """ + TDD Proof: Ensure ScreenIdentity classifies EXPLORE_GRID correctly + even if the tab bar fails to report selected="true". + """ + # XML snippet containing the search bar and the search tab, but NO selected tabs. + xml_dump = """ + + + + + + + """ + + identity = ScreenIdentity("testbot") + result = identity.identify(xml_dump) + + assert result["screen_type"] == ScreenType.EXPLORE_GRID, ( + f"Expected EXPLORE_GRID, but got {result['screen_type']}. " + f"Structural heuristic failed to recognize search_edit_text + search_tab!" + )