fix(navigation): purge structural guards and enforce pure VLM discovery for bottom tabs

Removed the hardcoded structural fallback bypasses for bottom navigation tabs to ensure 100% autonomous visual inference. Expanded the VLM intent resolution prompt with explicit spatial heuristics for bottom navigation (e.g., 'profile tab is the avatar icon at the bottom right') to prevent LLaVA hallucinations without resorting to XML resource-id hacks. Added E2E visual test proof.
This commit is contained in:
2026-04-29 01:23:20 +02:00
parent 03105437b8
commit e55abc5a8a
2 changed files with 47 additions and 2 deletions

View File

@@ -159,3 +159,42 @@ def test_resolve_uses_structural_path_when_no_device(make_real_device_with_xml):
result = resolver.resolve("tap profile tab", candidates, screen_height=2400)
assert result is not None, "Structural fallback failed to find profile_tab without a device"
assert result.resource_id == "com.instagram.android:id/profile_tab"
@pytest.mark.live_llm
def test_visual_discovery_finds_profile_tab_by_seeing(make_real_device_with_image):
"""
LIVE VLM TEST: The bot SEES a screenshot with numbered boxes
and visually identifies which box is the 'profile tab'.
This proves the prompt correctly guides the VLM to pick bottom navigation tabs
without hardcoding resource IDs.
"""
from GramAddict.core.perception.spatial_parser import SpatialParser
with open("tests/fixtures/home_feed_with_ad.xml", "r", encoding="utf-8") as f:
xml = f.read()
parser = SpatialParser()
root = parser.parse(xml)
candidates = parser.get_clickable_nodes(root)
# Use a real image so the VLM can actually see the UI
device = make_real_device_with_image("tests/fixtures/home_feed_with_ad.jpg")
resolver = IntentResolver()
# Visual Discovery: Let the VLM SEE the screen
result = resolver.resolve(
"tap profile tab",
candidates,
device,
)
assert result is not None, "Visual discovery returned None — VLM couldn't find 'profile tab' on screen"
# Check that it actually selected the correct tab
selected_id = (result.resource_id or "").lower()
# On the home_feed_with_ad_dump, the profile tab should be selected
assert (
"profile_tab" in selected_id
), f"Visual discovery picked wrong node! Got: id='{result.resource_id}', desc='{result.content_desc}'"