fix(navigation): purge structural guards and enforce pure VLM discovery for bottom tabs
Removed the hardcoded structural fallback bypasses for bottom navigation tabs to ensure 100% autonomous visual inference. Expanded the VLM intent resolution prompt with explicit spatial heuristics for bottom navigation (e.g., 'profile tab is the avatar icon at the bottom right') to prevent LLaVA hallucinations without resorting to XML resource-id hacks. Added E2E visual test proof.
This commit is contained in:
@@ -159,3 +159,42 @@ def test_resolve_uses_structural_path_when_no_device(make_real_device_with_xml):
|
||||
result = resolver.resolve("tap profile tab", candidates, screen_height=2400)
|
||||
assert result is not None, "Structural fallback failed to find profile_tab without a device"
|
||||
assert result.resource_id == "com.instagram.android:id/profile_tab"
|
||||
|
||||
|
||||
@pytest.mark.live_llm
|
||||
def test_visual_discovery_finds_profile_tab_by_seeing(make_real_device_with_image):
|
||||
"""
|
||||
LIVE VLM TEST: The bot SEES a screenshot with numbered boxes
|
||||
and visually identifies which box is the 'profile tab'.
|
||||
This proves the prompt correctly guides the VLM to pick bottom navigation tabs
|
||||
without hardcoding resource IDs.
|
||||
"""
|
||||
from GramAddict.core.perception.spatial_parser import SpatialParser
|
||||
|
||||
with open("tests/fixtures/home_feed_with_ad.xml", "r", encoding="utf-8") as f:
|
||||
xml = f.read()
|
||||
|
||||
parser = SpatialParser()
|
||||
root = parser.parse(xml)
|
||||
candidates = parser.get_clickable_nodes(root)
|
||||
|
||||
# Use a real image so the VLM can actually see the UI
|
||||
device = make_real_device_with_image("tests/fixtures/home_feed_with_ad.jpg")
|
||||
resolver = IntentResolver()
|
||||
|
||||
# Visual Discovery: Let the VLM SEE the screen
|
||||
result = resolver.resolve(
|
||||
"tap profile tab",
|
||||
candidates,
|
||||
device,
|
||||
)
|
||||
|
||||
assert result is not None, "Visual discovery returned None — VLM couldn't find 'profile tab' on screen"
|
||||
|
||||
# Check that it actually selected the correct tab
|
||||
selected_id = (result.resource_id or "").lower()
|
||||
|
||||
# On the home_feed_with_ad_dump, the profile tab should be selected
|
||||
assert (
|
||||
"profile_tab" in selected_id
|
||||
), f"Visual discovery picked wrong node! Got: id='{result.resource_id}', desc='{result.content_desc}'"
|
||||
|
||||
Reference in New Issue
Block a user