Revert "fix(navigation): eliminate VLM hallucination on bottom navigation tabs via structural guard"
This reverts commit b9c29a5a2d.
This commit is contained in:
@@ -254,27 +254,11 @@ class IntentResolver:
|
||||
and "per cent" not in (n.content_desc or "").lower()
|
||||
]
|
||||
|
||||
# --- Structural Navigation Guard ---
|
||||
# For strictly defined navigation tabs, we bypass the VLM entirely.
|
||||
tab_mapping = {
|
||||
"tap home tab": "feed_tab",
|
||||
"tap explore tab": "search_tab",
|
||||
"tap reels tab": "clips_tab",
|
||||
"tap profile tab": "profile_tab",
|
||||
"tap messages tab": "direct_tab",
|
||||
}
|
||||
intent_lower = intent_description.lower()
|
||||
if intent_lower in tab_mapping:
|
||||
target_id = tab_mapping[intent_lower]
|
||||
for node in candidates:
|
||||
if target_id in (node.resource_id or ""):
|
||||
logger.info(f"🎯 [Navigation Guard] Exact match found for '{intent_lower}' via resource-id. Skipping VLM.")
|
||||
return node
|
||||
|
||||
# --- Strict Button Guard ---
|
||||
# If the intent specifically asks for a "button", "icon", or "tab",
|
||||
# filter out candidates that contain long text (e.g. captions, comments)
|
||||
# to prevent the VLM from hallucinating text nodes as interactive buttons.
|
||||
intent_lower = intent_description.lower()
|
||||
if "button" in intent_lower or "icon" in intent_lower or "tab" in intent_lower:
|
||||
filtered_candidates = []
|
||||
for node in candidates:
|
||||
|
||||
@@ -133,26 +133,18 @@ def test_visual_discovery_finds_following_by_seeing(make_real_device_with_image)
|
||||
# ═══════════════════════════════════════════════════════
|
||||
|
||||
|
||||
def test_structural_navigation_guard_bypasses_vlm(make_real_device_with_xml):
|
||||
def test_resolve_uses_structural_path_when_no_device(make_real_device_with_xml):
|
||||
"""
|
||||
TDD PROOF: The Structural Navigation Guard must intercept bottom-nav actions
|
||||
(like "tap profile tab") and directly return the matching candidate based on
|
||||
resource-id, entirely bypassing the VLM and Set-of-Mark visual annotation.
|
||||
When called WITHOUT a device (device=None), resolve() must fall back
|
||||
to the structural XML-only path instead of visual discovery.
|
||||
This proves the routing logic works: visual is primary, structural is fallback.
|
||||
"""
|
||||
from GramAddict.core.perception.spatial_parser import SpatialNode
|
||||
|
||||
resolver = IntentResolver()
|
||||
|
||||
# Provide candidates including a trap (Profile picture) and the real tab
|
||||
# A single candidate with a clear profile_tab match
|
||||
candidates = [
|
||||
SpatialNode(
|
||||
resource_id="com.instagram.android:id/row_feed_photo_profile_imageview",
|
||||
class_name="android.widget.ImageView",
|
||||
text="",
|
||||
content_desc="Profile picture of some user",
|
||||
bounds=(0, 0, 100, 100),
|
||||
clickable=True,
|
||||
),
|
||||
SpatialNode(
|
||||
resource_id="com.instagram.android:id/profile_tab",
|
||||
class_name="android.widget.FrameLayout",
|
||||
@@ -163,14 +155,7 @@ def test_structural_navigation_guard_bypasses_vlm(make_real_device_with_xml):
|
||||
)
|
||||
]
|
||||
|
||||
# Device IS provided. Without the structural guard, this would trigger
|
||||
# visual discovery (which fails in tests without an image).
|
||||
device = make_real_device_with_xml("tests/fixtures/home_feed_with_ad.xml")
|
||||
|
||||
# We resolve the intent
|
||||
result = resolver.resolve("tap profile tab", candidates, device=device, screen_height=2400)
|
||||
|
||||
assert result is not None, "Structural Navigation Guard failed to find the profile tab"
|
||||
assert result.resource_id == "com.instagram.android:id/profile_tab", (
|
||||
f"Guard picked the wrong node! Selected: {result.resource_id}"
|
||||
)
|
||||
# Without device, resolve must still work via structural matching
|
||||
result = resolver.resolve("tap profile tab", candidates, screen_height=2400)
|
||||
assert result is not None, "Structural fallback failed to find profile_tab without a device"
|
||||
assert result.resource_id == "com.instagram.android:id/profile_tab"
|
||||
|
||||
@@ -1,25 +0,0 @@
|
||||
import pytest
|
||||
from GramAddict.core.perception.screen_identity import ScreenIdentity, ScreenType
|
||||
|
||||
def test_identify_explore_grid_without_selected_tab():
|
||||
"""
|
||||
TDD Proof: Ensure ScreenIdentity classifies EXPLORE_GRID correctly
|
||||
even if the tab bar fails to report selected="true".
|
||||
"""
|
||||
# XML snippet containing the search bar and the search tab, but NO selected tabs.
|
||||
xml_dump = """<?xml version='1.0' encoding='UTF-8' standalone='yes' ?>
|
||||
<hierarchy rotation="0">
|
||||
<node index="0" text="Search" resource-id="com.instagram.android:id/action_bar_search_edit_text" class="android.widget.EditText" package="com.instagram.android" content-desc="" clickable="true" bounds="[32,173][943,265]" />
|
||||
<node index="3" text="" resource-id="com.instagram.android:id/search_tab" class="android.widget.FrameLayout" package="com.instagram.android" content-desc="Search and explore" clickable="true" selected="false" bounds="[648,2235][864,2361]" />
|
||||
<!-- Some image grid content -->
|
||||
<node index="1" text="" resource-id="com.instagram.android:id/image_button" class="android.widget.ImageView" package="com.instagram.android" content-desc="Photo by User" clickable="true" bounds="[0,300][350,650]" />
|
||||
</hierarchy>
|
||||
"""
|
||||
|
||||
identity = ScreenIdentity("testbot")
|
||||
result = identity.identify(xml_dump)
|
||||
|
||||
assert result["screen_type"] == ScreenType.EXPLORE_GRID, (
|
||||
f"Expected EXPLORE_GRID, but got {result['screen_type']}. "
|
||||
f"Structural heuristic failed to recognize search_edit_text + search_tab!"
|
||||
)
|
||||
Reference in New Issue
Block a user