fix(navigation): eliminate VLM hallucination on bottom navigation tabs via structural guard
Added a 'Structural Navigation Guard' in IntentResolver to map critical bottom navigation intents (e.g., 'tap profile tab') directly to their stable resource-ids. This bypasses the VLM entirely, guaranteeing 100% deterministic clicks and resolving the issue where the VLM failed to locate the profile tab, causing the edge to become masked and trapping the bot on the home feed. Included TDD proof.
This commit is contained in:
@@ -254,11 +254,27 @@ class IntentResolver:
|
||||
and "per cent" not in (n.content_desc or "").lower()
|
||||
]
|
||||
|
||||
# --- Structural Navigation Guard ---
|
||||
# For strictly defined navigation tabs, we bypass the VLM entirely.
|
||||
tab_mapping = {
|
||||
"tap home tab": "feed_tab",
|
||||
"tap explore tab": "search_tab",
|
||||
"tap reels tab": "clips_tab",
|
||||
"tap profile tab": "profile_tab",
|
||||
"tap messages tab": "direct_tab",
|
||||
}
|
||||
intent_lower = intent_description.lower()
|
||||
if intent_lower in tab_mapping:
|
||||
target_id = tab_mapping[intent_lower]
|
||||
for node in candidates:
|
||||
if target_id in (node.resource_id or ""):
|
||||
logger.info(f"🎯 [Navigation Guard] Exact match found for '{intent_lower}' via resource-id. Skipping VLM.")
|
||||
return node
|
||||
|
||||
# --- Strict Button Guard ---
|
||||
# If the intent specifically asks for a "button", "icon", or "tab",
|
||||
# filter out candidates that contain long text (e.g. captions, comments)
|
||||
# to prevent the VLM from hallucinating text nodes as interactive buttons.
|
||||
intent_lower = intent_description.lower()
|
||||
if "button" in intent_lower or "icon" in intent_lower or "tab" in intent_lower:
|
||||
filtered_candidates = []
|
||||
for node in candidates:
|
||||
|
||||
@@ -133,18 +133,26 @@ def test_visual_discovery_finds_following_by_seeing(make_real_device_with_image)
|
||||
# ═══════════════════════════════════════════════════════
|
||||
|
||||
|
||||
def test_resolve_uses_structural_path_when_no_device(make_real_device_with_xml):
|
||||
def test_structural_navigation_guard_bypasses_vlm(make_real_device_with_xml):
|
||||
"""
|
||||
When called WITHOUT a device (device=None), resolve() must fall back
|
||||
to the structural XML-only path instead of visual discovery.
|
||||
This proves the routing logic works: visual is primary, structural is fallback.
|
||||
TDD PROOF: The Structural Navigation Guard must intercept bottom-nav actions
|
||||
(like "tap profile tab") and directly return the matching candidate based on
|
||||
resource-id, entirely bypassing the VLM and Set-of-Mark visual annotation.
|
||||
"""
|
||||
from GramAddict.core.perception.spatial_parser import SpatialNode
|
||||
|
||||
resolver = IntentResolver()
|
||||
|
||||
# A single candidate with a clear profile_tab match
|
||||
# Provide candidates including a trap (Profile picture) and the real tab
|
||||
candidates = [
|
||||
SpatialNode(
|
||||
resource_id="com.instagram.android:id/row_feed_photo_profile_imageview",
|
||||
class_name="android.widget.ImageView",
|
||||
text="",
|
||||
content_desc="Profile picture of some user",
|
||||
bounds=(0, 0, 100, 100),
|
||||
clickable=True,
|
||||
),
|
||||
SpatialNode(
|
||||
resource_id="com.instagram.android:id/profile_tab",
|
||||
class_name="android.widget.FrameLayout",
|
||||
@@ -155,7 +163,14 @@ def test_resolve_uses_structural_path_when_no_device(make_real_device_with_xml):
|
||||
)
|
||||
]
|
||||
|
||||
# Without device, resolve must still work via structural matching
|
||||
result = resolver.resolve("tap profile tab", candidates, screen_height=2400)
|
||||
assert result is not None, "Structural fallback failed to find profile_tab without a device"
|
||||
assert result.resource_id == "com.instagram.android:id/profile_tab"
|
||||
# Device IS provided. Without the structural guard, this would trigger
|
||||
# visual discovery (which fails in tests without an image).
|
||||
device = make_real_device_with_xml("tests/fixtures/home_feed_with_ad.xml")
|
||||
|
||||
# We resolve the intent
|
||||
result = resolver.resolve("tap profile tab", candidates, device=device, screen_height=2400)
|
||||
|
||||
assert result is not None, "Structural Navigation Guard failed to find the profile tab"
|
||||
assert result.resource_id == "com.instagram.android:id/profile_tab", (
|
||||
f"Guard picked the wrong node! Selected: {result.resource_id}"
|
||||
)
|
||||
|
||||
25
tests/unit/perception/test_screen_identity.py
Normal file
25
tests/unit/perception/test_screen_identity.py
Normal file
@@ -0,0 +1,25 @@
|
||||
import pytest
|
||||
from GramAddict.core.perception.screen_identity import ScreenIdentity, ScreenType
|
||||
|
||||
def test_identify_explore_grid_without_selected_tab():
|
||||
"""
|
||||
TDD Proof: Ensure ScreenIdentity classifies EXPLORE_GRID correctly
|
||||
even if the tab bar fails to report selected="true".
|
||||
"""
|
||||
# XML snippet containing the search bar and the search tab, but NO selected tabs.
|
||||
xml_dump = """<?xml version='1.0' encoding='UTF-8' standalone='yes' ?>
|
||||
<hierarchy rotation="0">
|
||||
<node index="0" text="Search" resource-id="com.instagram.android:id/action_bar_search_edit_text" class="android.widget.EditText" package="com.instagram.android" content-desc="" clickable="true" bounds="[32,173][943,265]" />
|
||||
<node index="3" text="" resource-id="com.instagram.android:id/search_tab" class="android.widget.FrameLayout" package="com.instagram.android" content-desc="Search and explore" clickable="true" selected="false" bounds="[648,2235][864,2361]" />
|
||||
<!-- Some image grid content -->
|
||||
<node index="1" text="" resource-id="com.instagram.android:id/image_button" class="android.widget.ImageView" package="com.instagram.android" content-desc="Photo by User" clickable="true" bounds="[0,300][350,650]" />
|
||||
</hierarchy>
|
||||
"""
|
||||
|
||||
identity = ScreenIdentity("testbot")
|
||||
result = identity.identify(xml_dump)
|
||||
|
||||
assert result["screen_type"] == ScreenType.EXPLORE_GRID, (
|
||||
f"Expected EXPLORE_GRID, but got {result['screen_type']}. "
|
||||
f"Structural heuristic failed to recognize search_edit_text + search_tab!"
|
||||
)
|
||||
Reference in New Issue
Block a user