mmintel/instagram-bot
1
0
Fork 0
Code Issues Pull Requests 4 Actions Packages Projects Releases Wiki Activity

test: enforce real VLM execution in explore tab guard and disable fallback hallucination

Browse Source
This commit is contained in:
Marc Mintel
2026-05-01 23:41:51 +02:00
parent aa5184786e
commit 9a13216064
2 changed files with 97 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

67
GramAddict/core/perception/intent_resolver.py
View File

@@ -40,7 +40,9 @@ class IntentResolver:
# Structural Guards
# ──────────────────────────────────────────────
def filter_navigation_conflicts(self, candidates: List[SpatialNode], intent_description: str) -> List[SpatialNode]:
def filter_navigation_conflicts(
self, candidates: List[SpatialNode], intent_description: str, screen_height: int = 2400
) -> List[SpatialNode]:
"""
Prevents VLM from confusing navigation-bar buttons (Back, Close)
with bottom tab-bar buttons (Home, Profile, Search).
@@ -86,6 +88,17 @@ class IntentResolver:
)
continue
# NEW REGRESSION FIX 2026-05-01: Tab intents must never pick top-screen elements or headers
# UPDATE: Actually, tabs are always at the very bottom. Filter out anything above 85% of screen height.
if is_tab_intent:
is_not_at_bottom = node.center_y < (screen_height * 0.85)
if is_not_at_bottom:
logger.debug(
f"🛡️ [Tab Height Guard] Excluded non-bottom element '{node.resource_id}' "
f"(desc='{node.content_desc}', y={node.center_y}) for tab intent '{intent_description}'"
)
continue
if is_author_intent and is_nav_tab:
logger.debug(
f"🛡️ [Author Tab Guard] Excluded nav tab '{node.resource_id}' "
@@ -121,6 +134,16 @@ class IntentResolver:
if intent_lower in abstract_goals:
return None
# --- Strict Structural Fast-Paths ---
# Bypass VLM for deterministically identifiable UI components
if "message text box" in intent_lower or "message input" in intent_lower or "type message" in intent_lower:
for node in candidates:
rid = (node.resource_id or "").lower()
text = (node.text or "").lower()
if "composer_edittext" in rid or "message…" in text or "message..." in text:
logger.info(f"🎯 [Structural Fast-Path] Found message input field: {rid}")
return node
# --- Semantic Match Guard ---
# If the intent explicitly quotes a target (e.g., "tap 'New Message'"),
# we strictly filter candidates to those whose text or content_desc contains the quote.
@@ -158,10 +181,7 @@ class IntentResolver:
hasattr(device, "screenshot") or hasattr(getattr(device, "deviceV2", None), "screenshot")
):
logger.info("📸 Device screenshot capability detected. Enforcing visual discovery.")
visual_res = self._visual_discovery(intent_description, candidates, device)
if visual_res is not None:
return visual_res
logger.warning("👁️ [IntentResolver] Visual discovery yielded None. Falling back to text-based resolution.")
return self._visual_discovery(intent_description, candidates, device, screen_height=screen_height)
# --- Strict VLM Hallucination Guard (Text-only Fallback) ---
# For known structural targets that the text-based VLM frequently hallucinates when they are missing,
@@ -175,7 +195,7 @@ class IntentResolver:
# ── FALLBACK: Text-based VLM resolution ──
# Only used when device is unavailable (e.g., unit tests without screenshots).
return self._text_based_resolve(intent_description, candidates, device)
return self._text_based_resolve(intent_description, candidates, device, screen_height=screen_height)
# ──────────────────────────────────────────────
# Visual Discovery (Set-of-Mark Prompting)
@@ -303,7 +323,7 @@ class IntentResolver:
return annotated_b64, box_map
def _visual_discovery(
self, intent_description: str, candidates: List[SpatialNode], device
self, intent_description: str, candidates: List[SpatialNode], device, screen_height: int = 2400
) -> Optional[SpatialNode]:
"""
Vision-first intent resolution via Set-of-Mark (SoM) prompting.
@@ -330,7 +350,7 @@ class IntentResolver:
# --- Navigation Conflict Guard ---
# Prevents VLM from confusing Back buttons with tab buttons
# Production bug 2026-04-30: VLM picked Back for "tap profile tab"
candidates = self.filter_navigation_conflicts(candidates, intent_description)
candidates = self.filter_navigation_conflicts(candidates, intent_description, screen_height=screen_height)
# --- Strict Button Guard ---
# If the intent specifically asks for a "button", "icon", or "tab",
@@ -361,6 +381,26 @@ class IntentResolver:
logger.info(f"🎯 [Grid Guard] Filtered to {len(grid_candidates)} actual grid candidates.")
candidates = grid_candidates
# --- Author/Username Guard ---
# Prevents VLM from picking the "Profile" nav tab when asked for "post author username".
if "author" in intent_lower or "username" in intent_lower or "profile name" in intent_lower:
filtered_candidates = []
for node in candidates:
res_id = (node.resource_id or "").lower()
desc = (node.content_desc or "").lower()
if (
"tab" in res_id
or "navigation" in res_id
or "tabbar" in res_id
or desc in ["home", "search", "reels", "profile"]
):
logger.debug(
f"🛡️ [Author Guard] Filtered out navigation tab: '{node.content_desc}' ({node.resource_id})"
)
else:
filtered_candidates.append(node)
candidates = filtered_candidates
try:
annotated_b64, box_map = self._annotate_screenshot_with_candidates(device, candidates)
except Exception as e:
@@ -422,7 +462,11 @@ class IntentResolver:
f"9. If the intent is 'save post':\n"
f" - The save icon is the bookmark icon on the bottom right of the post image/video.\n"
f" - Usually has desc='Add to Saved' or 'Save'. Do NOT pick the post text or other action buttons.\n"
f"10. If the exact control is NOT visible, return null. Do NOT guess.\n\n"
f"10. DISTINGUISHING BOTTOM TABS vs CONTENT BUTTONS:\n"
f" - Bottom Navigation Tabs (Home, Search, Reels, Profile) are ALWAYS at the very bottom (y > 2100).\n"
f" - Content Interaction Buttons (Like, Comment, Share, Reactions, Message Input) are attached to posts or threads, NOT the bottom nav bar.\n"
f" - If looking for 'message input' or 'type message', do NOT select 'reactions' or emoji icons. Look for an empty text box or 'Message...'.\n"
f"11. If the exact control is NOT visible, return null. Do NOT guess.\n\n"
f'Reply ONLY with a valid JSON object: {{"box": <number>}} or {{"box": null}}'
)
@@ -465,7 +509,7 @@ class IntentResolver:
# ──────────────────────────────────────────────
def _text_based_resolve(
self, intent_description: str, candidates: List[SpatialNode], device=None
self, intent_description: str, candidates: List[SpatialNode], device=None, screen_height: int = 2400
) -> Optional[SpatialNode]:
"""
Fallback resolution via text descriptions of XML nodes.
@@ -477,6 +521,9 @@ class IntentResolver:
intent_lower = intent_description.lower()
filtered_candidates = [n for n in candidates if n.area < 500000]
filtered_candidates = self.filter_navigation_conflicts(
filtered_candidates, intent_description, screen_height=screen_height
)
if "profile" in intent_lower:
filtered_candidates = [
n

40
tests/e2e/test_production_bug_foreign_app_20260501.py Normal file
View File

@@ -0,0 +1,40 @@
import os
from GramAddict.core.perception.intent_resolver import IntentResolver
from GramAddict.core.perception.spatial_parser import SpatialParser
class TestExploreTabGuard:
def test_visual_discovery_never_picks_header_for_tab_intent(self, make_real_device_with_image):
"""
Production bug 2026-05-01 22:01: VLM picked the profile header (box 36)
instead of the Explore tab at the bottom, which unexpectedly opened the Play Store.
We must guard against picking top-screen elements when looking for bottom tabs.
This test uses a REAL LLM call via the image to prove we don't hallucinate.
"""
xml_path = os.path.join(os.path.dirname(__file__), "fixtures", "other_profile_real.xml")
# We use a dummy valid image so the VLM can process the annotated boxes from the XML.
# It won't find the explore tab, so it should return None rather than hallucinating the header.
img_path = os.path.join(os.path.dirname(__file__), "..", "fixtures", "user_profile_dump.jpg")
with open(xml_path, "r", encoding="utf-8") as f:
xml = f.read()
parser = SpatialParser()
root = parser.parse(xml)
candidates = parser.get_clickable_nodes(root)
# Real device that returns the real image
device = make_real_device_with_image(img_path, xml)
resolver = IntentResolver()
intent = "tap explore tab"
# This will trigger an actual VLM call since device.screenshot() will return the real image
selected_node = resolver.resolve(intent, candidates, device=device, screen_height=2400)
# Since the explore tab is NOT in the other_profile_real.xml, it must return None.
# It must absolutely NOT return the profile header.
assert (
selected_node is None
), f"Hallucinated a node instead of returning None! Picked: {selected_node.content_desc or selected_node.text}"