fix(navigation): eliminate VLM hallucination on bottom navigation tabs via structural guard
Added a 'Structural Navigation Guard' in IntentResolver to map critical bottom navigation intents (e.g., 'tap profile tab') directly to their stable resource-ids. This bypasses the VLM entirely, guaranteeing 100% deterministic clicks and resolving the issue where the VLM failed to locate the profile tab, causing the edge to become masked and trapping the bot on the home feed. Included TDD proof.
This commit is contained in:
@@ -133,18 +133,26 @@ def test_visual_discovery_finds_following_by_seeing(make_real_device_with_image)
|
||||
# ═══════════════════════════════════════════════════════
|
||||
|
||||
|
||||
def test_resolve_uses_structural_path_when_no_device(make_real_device_with_xml):
|
||||
def test_structural_navigation_guard_bypasses_vlm(make_real_device_with_xml):
|
||||
"""
|
||||
When called WITHOUT a device (device=None), resolve() must fall back
|
||||
to the structural XML-only path instead of visual discovery.
|
||||
This proves the routing logic works: visual is primary, structural is fallback.
|
||||
TDD PROOF: The Structural Navigation Guard must intercept bottom-nav actions
|
||||
(like "tap profile tab") and directly return the matching candidate based on
|
||||
resource-id, entirely bypassing the VLM and Set-of-Mark visual annotation.
|
||||
"""
|
||||
from GramAddict.core.perception.spatial_parser import SpatialNode
|
||||
|
||||
resolver = IntentResolver()
|
||||
|
||||
# A single candidate with a clear profile_tab match
|
||||
# Provide candidates including a trap (Profile picture) and the real tab
|
||||
candidates = [
|
||||
SpatialNode(
|
||||
resource_id="com.instagram.android:id/row_feed_photo_profile_imageview",
|
||||
class_name="android.widget.ImageView",
|
||||
text="",
|
||||
content_desc="Profile picture of some user",
|
||||
bounds=(0, 0, 100, 100),
|
||||
clickable=True,
|
||||
),
|
||||
SpatialNode(
|
||||
resource_id="com.instagram.android:id/profile_tab",
|
||||
class_name="android.widget.FrameLayout",
|
||||
@@ -155,7 +163,14 @@ def test_resolve_uses_structural_path_when_no_device(make_real_device_with_xml):
|
||||
)
|
||||
]
|
||||
|
||||
# Without device, resolve must still work via structural matching
|
||||
result = resolver.resolve("tap profile tab", candidates, screen_height=2400)
|
||||
assert result is not None, "Structural fallback failed to find profile_tab without a device"
|
||||
assert result.resource_id == "com.instagram.android:id/profile_tab"
|
||||
# Device IS provided. Without the structural guard, this would trigger
|
||||
# visual discovery (which fails in tests without an image).
|
||||
device = make_real_device_with_xml("tests/fixtures/home_feed_with_ad.xml")
|
||||
|
||||
# We resolve the intent
|
||||
result = resolver.resolve("tap profile tab", candidates, device=device, screen_height=2400)
|
||||
|
||||
assert result is not None, "Structural Navigation Guard failed to find the profile tab"
|
||||
assert result.resource_id == "com.instagram.android:id/profile_tab", (
|
||||
f"Guard picked the wrong node! Selected: {result.resource_id}"
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user