Revert "fix(navigation): eliminate VLM hallucination on bottom navigation tabs via structural guard"

This reverts commit b9c29a5a2d.
This commit is contained in:
2026-04-29 01:19:18 +02:00
parent b9c29a5a2d
commit 03105437b8
3 changed files with 10 additions and 66 deletions

View File

@@ -254,27 +254,11 @@ class IntentResolver:
and "per cent" not in (n.content_desc or "").lower()
]
# --- Structural Navigation Guard ---
# For strictly defined navigation tabs, we bypass the VLM entirely.
tab_mapping = {
"tap home tab": "feed_tab",
"tap explore tab": "search_tab",
"tap reels tab": "clips_tab",
"tap profile tab": "profile_tab",
"tap messages tab": "direct_tab",
}
intent_lower = intent_description.lower()
if intent_lower in tab_mapping:
target_id = tab_mapping[intent_lower]
for node in candidates:
if target_id in (node.resource_id or ""):
logger.info(f"🎯 [Navigation Guard] Exact match found for '{intent_lower}' via resource-id. Skipping VLM.")
return node
# --- Strict Button Guard ---
# If the intent specifically asks for a "button", "icon", or "tab",
# filter out candidates that contain long text (e.g. captions, comments)
# to prevent the VLM from hallucinating text nodes as interactive buttons.
intent_lower = intent_description.lower()
if "button" in intent_lower or "icon" in intent_lower or "tab" in intent_lower:
filtered_candidates = []
for node in candidates:

View File

@@ -133,26 +133,18 @@ def test_visual_discovery_finds_following_by_seeing(make_real_device_with_image)
# ═══════════════════════════════════════════════════════
def test_structural_navigation_guard_bypasses_vlm(make_real_device_with_xml):
def test_resolve_uses_structural_path_when_no_device(make_real_device_with_xml):
"""
TDD PROOF: The Structural Navigation Guard must intercept bottom-nav actions
(like "tap profile tab") and directly return the matching candidate based on
resource-id, entirely bypassing the VLM and Set-of-Mark visual annotation.
When called WITHOUT a device (device=None), resolve() must fall back
to the structural XML-only path instead of visual discovery.
This proves the routing logic works: visual is primary, structural is fallback.
"""
from GramAddict.core.perception.spatial_parser import SpatialNode
resolver = IntentResolver()
# Provide candidates including a trap (Profile picture) and the real tab
# A single candidate with a clear profile_tab match
candidates = [
SpatialNode(
resource_id="com.instagram.android:id/row_feed_photo_profile_imageview",
class_name="android.widget.ImageView",
text="",
content_desc="Profile picture of some user",
bounds=(0, 0, 100, 100),
clickable=True,
),
SpatialNode(
resource_id="com.instagram.android:id/profile_tab",
class_name="android.widget.FrameLayout",
@@ -163,14 +155,7 @@ def test_structural_navigation_guard_bypasses_vlm(make_real_device_with_xml):
)
]
# Device IS provided. Without the structural guard, this would trigger
# visual discovery (which fails in tests without an image).
device = make_real_device_with_xml("tests/fixtures/home_feed_with_ad.xml")
# We resolve the intent
result = resolver.resolve("tap profile tab", candidates, device=device, screen_height=2400)
assert result is not None, "Structural Navigation Guard failed to find the profile tab"
assert result.resource_id == "com.instagram.android:id/profile_tab", (
f"Guard picked the wrong node! Selected: {result.resource_id}"
)
# Without device, resolve must still work via structural matching
result = resolver.resolve("tap profile tab", candidates, screen_height=2400)
assert result is not None, "Structural fallback failed to find profile_tab without a device"
assert result.resource_id == "com.instagram.android:id/profile_tab"

View File

@@ -1,25 +0,0 @@
import pytest
from GramAddict.core.perception.screen_identity import ScreenIdentity, ScreenType
def test_identify_explore_grid_without_selected_tab():
"""
TDD Proof: Ensure ScreenIdentity classifies EXPLORE_GRID correctly
even if the tab bar fails to report selected="true".
"""
# XML snippet containing the search bar and the search tab, but NO selected tabs.
xml_dump = """<?xml version='1.0' encoding='UTF-8' standalone='yes' ?>
<hierarchy rotation="0">
<node index="0" text="Search" resource-id="com.instagram.android:id/action_bar_search_edit_text" class="android.widget.EditText" package="com.instagram.android" content-desc="" clickable="true" bounds="[32,173][943,265]" />
<node index="3" text="" resource-id="com.instagram.android:id/search_tab" class="android.widget.FrameLayout" package="com.instagram.android" content-desc="Search and explore" clickable="true" selected="false" bounds="[648,2235][864,2361]" />
<!-- Some image grid content -->
<node index="1" text="" resource-id="com.instagram.android:id/image_button" class="android.widget.ImageView" package="com.instagram.android" content-desc="Photo by User" clickable="true" bounds="[0,300][350,650]" />
</hierarchy>
"""
identity = ScreenIdentity("testbot")
result = identity.identify(xml_dump)
assert result["screen_type"] == ScreenType.EXPLORE_GRID, (
f"Expected EXPLORE_GRID, but got {result['screen_type']}. "
f"Structural heuristic failed to recognize search_edit_text + search_tab!"
)