test: enforce real VLM execution in explore tab guard and disable fallback hallucination
This commit is contained in:
40
tests/e2e/test_production_bug_foreign_app_20260501.py
Normal file
40
tests/e2e/test_production_bug_foreign_app_20260501.py
Normal file
@@ -0,0 +1,40 @@
|
||||
import os
|
||||
|
||||
from GramAddict.core.perception.intent_resolver import IntentResolver
|
||||
from GramAddict.core.perception.spatial_parser import SpatialParser
|
||||
|
||||
|
||||
class TestExploreTabGuard:
|
||||
def test_visual_discovery_never_picks_header_for_tab_intent(self, make_real_device_with_image):
|
||||
"""
|
||||
Production bug 2026-05-01 22:01: VLM picked the profile header (box 36)
|
||||
instead of the Explore tab at the bottom, which unexpectedly opened the Play Store.
|
||||
We must guard against picking top-screen elements when looking for bottom tabs.
|
||||
This test uses a REAL LLM call via the image to prove we don't hallucinate.
|
||||
"""
|
||||
xml_path = os.path.join(os.path.dirname(__file__), "fixtures", "other_profile_real.xml")
|
||||
# We use a dummy valid image so the VLM can process the annotated boxes from the XML.
|
||||
# It won't find the explore tab, so it should return None rather than hallucinating the header.
|
||||
img_path = os.path.join(os.path.dirname(__file__), "..", "fixtures", "user_profile_dump.jpg")
|
||||
|
||||
with open(xml_path, "r", encoding="utf-8") as f:
|
||||
xml = f.read()
|
||||
|
||||
parser = SpatialParser()
|
||||
root = parser.parse(xml)
|
||||
candidates = parser.get_clickable_nodes(root)
|
||||
|
||||
# Real device that returns the real image
|
||||
device = make_real_device_with_image(img_path, xml)
|
||||
|
||||
resolver = IntentResolver()
|
||||
intent = "tap explore tab"
|
||||
|
||||
# This will trigger an actual VLM call since device.screenshot() will return the real image
|
||||
selected_node = resolver.resolve(intent, candidates, device=device, screen_height=2400)
|
||||
|
||||
# Since the explore tab is NOT in the other_profile_real.xml, it must return None.
|
||||
# It must absolutely NOT return the profile header.
|
||||
assert (
|
||||
selected_node is None
|
||||
), f"Hallucinated a node instead of returning None! Picked: {selected_node.content_desc or selected_node.text}"
|
||||
Reference in New Issue
Block a user