fix(vlm): Fix Semantic Guard boundary matching and harden VLM box validation for robust testing
This commit is contained in:
@@ -78,109 +78,39 @@ class IntentResolver:
|
||||
)
|
||||
filtered = []
|
||||
is_tab_intent = bool(_TAB_PATTERN.search(intent_lower)) and "back" not in intent_lower
|
||||
is_create_intent = "create" in intent_lower or "camera" in intent_lower or "story" in intent_lower
|
||||
# REGRESSION FIX 2026-05-01: Author/username intents must never pick nav tabs
|
||||
is_author_intent = any(kw in intent_lower for kw in ["author", "username", "post media"])
|
||||
|
||||
# Known bottom navigation tab resource_id suffixes
|
||||
NAV_TAB_SUFFIXES = ("_tab", "tab_icon", "navigation_bar")
|
||||
|
||||
for node in candidates:
|
||||
rid = (node.resource_id or "").lower()
|
||||
desc = (node.content_desc or "").lower()
|
||||
cls_name = (node.class_name or "").lower()
|
||||
|
||||
is_back = "back" in rid or desc == "back"
|
||||
is_close = "close" in rid or desc == "close"
|
||||
is_create = "camera" in rid or "create" in rid or desc == "camera" or desc == "create" or "creation" in rid
|
||||
is_nav_tab = any(rid.endswith(s) for s in NAV_TAB_SUFFIXES)
|
||||
# Geometric and Class-based heuristics only. No language strings or Resource IDs!
|
||||
is_bottom_nav_area = node.center_y > (screen_height * 0.85)
|
||||
is_top_header_area = node.center_y < (screen_height * 0.15)
|
||||
is_input_field = "edittext" in cls_name
|
||||
is_image_or_video = "imageview" in cls_name or "textureview" in cls_name
|
||||
is_large_container = node.area > (screen_height * 0.3 * screen_height * 0.3)
|
||||
|
||||
if is_tab_intent and (is_back or is_close):
|
||||
logger.debug(
|
||||
f"🛡️ [Nav Conflict Guard] Excluded '{node.resource_id}' "
|
||||
f"(desc='{node.content_desc}') for tab intent '{intent_description}'"
|
||||
)
|
||||
# Tab intents should NEVER be at the top of the screen
|
||||
if is_tab_intent and is_top_header_area:
|
||||
logger.debug("🛡️ [Tab Height Guard] Excluded top element for tab intent.")
|
||||
continue
|
||||
|
||||
# NEW REGRESSION FIX 2026-05-01: Tab intents must never pick top-screen elements or headers
|
||||
# UPDATE: Actually, tabs are always at the very bottom. Filter out anything above 85% of screen height.
|
||||
if is_tab_intent:
|
||||
is_not_at_bottom = node.center_y < (screen_height * 0.85)
|
||||
if is_not_at_bottom:
|
||||
logger.debug(
|
||||
f"🛡️ [Tab Height Guard] Excluded non-bottom element '{node.resource_id}' "
|
||||
f"(desc='{node.content_desc}', y={node.center_y}) for tab intent '{intent_description}'"
|
||||
)
|
||||
continue
|
||||
# Tab intents should NEVER be content items
|
||||
if any(kw in desc for kw in ["reel by", "photo by", "photos by", "row ", "column "]):
|
||||
logger.debug(
|
||||
f"🛡️ [Content Tab Guard] Excluded content item '{node.resource_id}' "
|
||||
f"(desc='{node.content_desc}') for tab intent '{intent_description}'"
|
||||
)
|
||||
continue
|
||||
|
||||
if is_author_intent and is_nav_tab:
|
||||
logger.debug(
|
||||
f"🛡️ [Author Tab Guard] Excluded nav tab '{node.resource_id}' "
|
||||
f"(desc='{node.content_desc}') for author intent '{intent_description}'"
|
||||
)
|
||||
# Author/Username intents should not pick bottom navigation tabs
|
||||
if is_author_intent and is_bottom_nav_area:
|
||||
logger.debug("🛡️ [Author Tab Guard] Excluded bottom nav area for author intent.")
|
||||
continue
|
||||
|
||||
if not is_create_intent and is_create:
|
||||
logger.debug(
|
||||
f"🛡️ [Creation Conflict Guard] Excluded '{node.resource_id}' "
|
||||
f"(desc='{node.content_desc}') for intent '{intent_description}'"
|
||||
)
|
||||
# Input fields should never be picked unless explicitly asked for
|
||||
is_reply_intent = "reply" in intent_lower or "message" in intent_lower or "type" in intent_lower
|
||||
if is_input_field and not is_reply_intent:
|
||||
logger.debug("🛡️ [Reply Guard] Excluded input field for non-reply intent.")
|
||||
continue
|
||||
|
||||
# NEW REGRESSION FIX: Exclude interaction buttons (comment, like, share) when looking for author or media
|
||||
# This prevents the weak VLM from hallucinating bounding box numbers that point to "Comment".
|
||||
interaction_suffixes = ["comment", "like", "share", "send", "save", "button_icon"]
|
||||
is_interaction = any(s in rid for s in interaction_suffixes) or any(s in desc for s in interaction_suffixes)
|
||||
node_text_lower = (node.text or "").lower()
|
||||
is_follow = "follow" in rid or "follow" in node_text_lower or "follow" in desc
|
||||
is_media_intent = "media content" in intent_lower or "image" in intent_lower or "video" in intent_lower
|
||||
|
||||
if is_author_intent and (is_interaction or is_follow):
|
||||
logger.debug(
|
||||
f"🛡️ [Author Interaction Guard] Excluded interaction/follow button '{node.resource_id}' "
|
||||
f"(desc='{node.content_desc}', text='{node.text}') for author intent '{intent_description}'"
|
||||
)
|
||||
continue
|
||||
|
||||
if is_media_intent and (is_interaction or is_follow):
|
||||
logger.debug(
|
||||
f"🛡️ [Media Interaction Guard] Excluded interaction/follow button '{node.resource_id}' "
|
||||
f"(desc='{node.content_desc}') for media intent '{intent_description}'"
|
||||
)
|
||||
continue
|
||||
|
||||
# NEW REGRESSION FIX: Exclude stories and reels tray when looking for a POST author
|
||||
# This prevents the VLM from selecting the user's own story at the top of the feed
|
||||
# Posts/Authors shouldn't be massive full-screen containers unless it's a specific media intent
|
||||
is_post_author_intent = is_author_intent and "post" in intent_lower
|
||||
node_text_lower = (node.text or "").lower()
|
||||
is_story_or_reel = (
|
||||
"story" in rid
|
||||
or "story" in desc
|
||||
or "story" in node_text_lower
|
||||
or "reel" in rid
|
||||
or "reel" in desc
|
||||
or "reel" in node_text_lower
|
||||
)
|
||||
|
||||
if is_post_author_intent and is_story_or_reel:
|
||||
logger.debug(
|
||||
f"🛡️ [Post Author Story Guard] Excluded story/reel '{node.resource_id}' "
|
||||
f"(desc='{node.content_desc}') for post author intent '{intent_description}'"
|
||||
)
|
||||
continue
|
||||
|
||||
# NEW REGRESSION FIX: Exclude action bar titles (like 'For you') when looking for an author
|
||||
if is_author_intent and "action_bar_title" in rid:
|
||||
logger.debug(
|
||||
f"🛡️ [Author Action Bar Guard] Excluded action bar title '{node.resource_id}' "
|
||||
f"(desc='{node.content_desc}') for author intent '{intent_description}'"
|
||||
)
|
||||
if is_post_author_intent and is_large_container and not is_image_or_video:
|
||||
logger.debug("🛡️ [Author Container Guard] Excluded massive container for author intent.")
|
||||
continue
|
||||
|
||||
filtered.append(node)
|
||||
@@ -205,227 +135,81 @@ class IntentResolver:
|
||||
return None
|
||||
|
||||
# --- Strict Structural Fast-Paths ---
|
||||
# Bypass VLM for deterministically identifiable UI components
|
||||
if "message text box" in intent_lower or "message input" in intent_lower or "type message" in intent_lower:
|
||||
for node in candidates:
|
||||
rid = (node.resource_id or "").lower()
|
||||
text = (node.text or "").lower()
|
||||
if "composer_edittext" in rid or "message…" in text or "message..." in text:
|
||||
logger.debug(f"🎯 [Structural Fast-Path] Found message input field: {rid}")
|
||||
return node
|
||||
# ALL HARDCODED UI STRUCTURAL GUARDS HAVE BEEN PURGED!
|
||||
# Rule: ZERO MAINTENANCE. We do not hardcode Resource IDs, content_desc, or text matching
|
||||
# for any UI elements (Likes, Follows, Messages, Tabs, Posts, Story Rings, etc.).
|
||||
# The bot MUST autonomously find elements via the Telepathic VLM Engine and store
|
||||
# them in Qdrant memory after successful structural delta verification.
|
||||
|
||||
if "last received message text" in intent_lower or "received message" in intent_lower:
|
||||
# Gather all message text views
|
||||
msg_nodes = [n for n in candidates if "direct_text_message_text_view" in (n.resource_id or "").lower()]
|
||||
if msg_nodes:
|
||||
# The last one in the XML is typically the most recent message at the bottom of the screen
|
||||
latest_msg = msg_nodes[-1]
|
||||
logger.debug(f"🎯 [Structural Fast-Path] Found last received message text: '{latest_msg.text}'")
|
||||
return latest_msg
|
||||
|
||||
if "send message button" in intent_lower:
|
||||
for node in candidates:
|
||||
rid = (node.resource_id or "").lower()
|
||||
desc = (node.content_desc or "").lower()
|
||||
text = (node.text or "").lower()
|
||||
if "send" in rid or "composer_button" in rid:
|
||||
logger.debug(f"🎯 [Structural Fast-Path] Found send button: {rid or desc or text}")
|
||||
return node
|
||||
|
||||
if "post author username" in intent_lower or "tap post username" in intent_lower:
|
||||
for node in candidates:
|
||||
rid = (node.resource_id or "").lower()
|
||||
if (
|
||||
"row_feed_photo_profile_imageview" in rid
|
||||
or "clips_author_profile_pic" in rid
|
||||
or "reel_viewer_profile_picture" in rid
|
||||
):
|
||||
logger.debug(f"🎯 [Structural Fast-Path] Found post author avatar image: {node.content_desc}")
|
||||
return node
|
||||
for node in candidates:
|
||||
rid = (node.resource_id or "").lower()
|
||||
if "row_feed_photo_profile_name" in rid or "clips_author_username" in rid or "reel_viewer_title" in rid:
|
||||
logger.debug(f"🎯 [Structural Fast-Path] Found post author username text: {node.text}")
|
||||
return node
|
||||
|
||||
if "feed post content" in intent_lower or "post media content" in intent_lower:
|
||||
for node in candidates:
|
||||
rid = (node.resource_id or "").lower()
|
||||
if "row_feed_photo_imageview" in rid or "zoomable_view_container" in rid:
|
||||
logger.debug(f"🎯 [Structural Fast-Path] Found feed post content: {rid}")
|
||||
return node
|
||||
|
||||
if "comment" in intent_lower and "button" in intent_lower:
|
||||
# First try the View all comments button
|
||||
for node in candidates:
|
||||
if (
|
||||
"view all comments" in (node.text or "").lower()
|
||||
or "view all comments" in (node.content_desc or "").lower()
|
||||
):
|
||||
logger.info(
|
||||
f"🎯 [Structural Fast-Path] Found comment button text: {node.text or node.content_desc}"
|
||||
)
|
||||
return node
|
||||
# Then try the icon itself if somehow clickable
|
||||
for node in candidates:
|
||||
if (
|
||||
"row_feed_button_comment" in (node.resource_id or "").lower()
|
||||
or "row_feed_textview_comments" in (node.resource_id or "").lower()
|
||||
):
|
||||
logger.debug(f"🎯 [Structural Fast-Path] Found comment button: {node.resource_id}")
|
||||
return node
|
||||
|
||||
if "like" in intent_lower and ("button" in intent_lower or "post" in intent_lower):
|
||||
for node in candidates:
|
||||
rid = (node.resource_id or "").lower()
|
||||
if "row_feed_button_like" in rid:
|
||||
logger.debug(f"🎯 [Structural Fast-Path] Found like button: {rid}")
|
||||
return node
|
||||
|
||||
if ("send" in intent_lower or "share" in intent_lower) and "post" in intent_lower and "button" in intent_lower:
|
||||
for node in candidates:
|
||||
rid = (node.resource_id or "").lower()
|
||||
desc = (node.content_desc or "").lower()
|
||||
if "row_feed_button_share" in rid or "send post" in desc:
|
||||
logger.debug(f"🎯 [Structural Fast-Path] Found send/share post button: {rid or desc}")
|
||||
return node
|
||||
|
||||
if "add to story" in intent_lower:
|
||||
# We skip structural fast-path for 'add to story' since it relies heavily on language/text strings
|
||||
# and let the VLM figure it out or rely on purely visual indicators.
|
||||
pass
|
||||
|
||||
if "share" in intent_lower and ("button" in intent_lower or "post" in intent_lower):
|
||||
for node in candidates:
|
||||
rid = (node.resource_id or "").lower()
|
||||
if "row_feed_button_share" in rid:
|
||||
logger.debug(f"🎯 [Structural Fast-Path] Found share button: {rid}")
|
||||
return node
|
||||
|
||||
if "save" in intent_lower and ("button" in intent_lower or "post" in intent_lower):
|
||||
for node in candidates:
|
||||
rid = (node.resource_id or "").lower()
|
||||
if "row_feed_button_save" in rid:
|
||||
logger.debug(f"🎯 [Structural Fast-Path] Found save button: {rid}")
|
||||
return node
|
||||
|
||||
if "follow" in intent_lower and "button" in intent_lower:
|
||||
for node in candidates:
|
||||
rid = (node.resource_id or "").lower()
|
||||
if (
|
||||
"profile_header_follow_button" in rid
|
||||
or "inline_follow_button" in rid
|
||||
or "follow_list_row_large_follow_button" in rid
|
||||
or "row_search_user_follow_button" in rid
|
||||
or "profile_header_user_action_follow_button" in rid
|
||||
):
|
||||
logger.debug(f"🎯 [Structural Fast-Path] Found follow/following button: {rid}")
|
||||
return node
|
||||
|
||||
if "first post" in intent_lower or "first item" in intent_lower or "first search result" in intent_lower:
|
||||
for node in candidates:
|
||||
rid = (node.resource_id or "").lower()
|
||||
if "grid_card_layout_container" in rid or "image_button" in rid or "row_search_user" in rid:
|
||||
logger.debug(f"🎯 [Structural Fast-Path] Found first post/item: {rid}")
|
||||
return node
|
||||
|
||||
if "heart" in intent_lower and "notification" in intent_lower:
|
||||
for node in candidates:
|
||||
rid = (node.resource_id or "").lower()
|
||||
desc = (node.content_desc or "").lower()
|
||||
# Could be in top bar or bottom bar depending on IG version
|
||||
if "notification" in rid or "newsfeed" in rid or "activity" in desc or "notification" in desc:
|
||||
logger.debug(f"🎯 [Structural Fast-Path] Found notifications/heart icon: {rid or desc}")
|
||||
return node
|
||||
|
||||
if ("inbox" in intent_lower or "direct message" in intent_lower) and "icon" in intent_lower:
|
||||
for node in candidates:
|
||||
rid = (node.resource_id or "").lower()
|
||||
desc = (node.content_desc or "").lower()
|
||||
if "direct_tab" in rid or "inbox_button" in rid or "message" in desc:
|
||||
logger.debug(f"🎯 [Structural Fast-Path] Found DM/inbox icon: {rid or desc}")
|
||||
return node
|
||||
|
||||
if "story ring" in intent_lower or "story tray" in intent_lower:
|
||||
story_nodes = []
|
||||
for node in candidates:
|
||||
rid = (node.resource_id or "").lower()
|
||||
desc = (node.content_desc or "").lower()
|
||||
text = (node.text or "").lower()
|
||||
# Instagram story tray avatars usually have this resource id and 'story' in the content description
|
||||
if ("avatar_image_view" in rid or "row_profile_header_imageview" in rid) and "story" in desc:
|
||||
# Ignore the user's explicit "Add to story" ring
|
||||
if "add to story" not in desc and "your story" not in text:
|
||||
story_nodes.append(node)
|
||||
|
||||
if story_nodes:
|
||||
# Sort horizontally (left-to-right)
|
||||
story_nodes.sort(key=lambda n: n.x1)
|
||||
|
||||
# Check if this is the home feed story tray (avatar_image_view without 'highlight' in desc)
|
||||
is_highlight = any("highlight" in (n.content_desc or "").lower() for n in story_nodes)
|
||||
if "avatar_image_view" in (story_nodes[0].resource_id or "").lower() and not is_highlight:
|
||||
if len(story_nodes) > 1:
|
||||
logger.info(
|
||||
f"🎯 [Structural Fast-Path] Found {len(story_nodes)} story rings. Skipping own profile. Picking second: '{story_nodes[1].content_desc}'"
|
||||
)
|
||||
return story_nodes[1]
|
||||
else:
|
||||
logger.warning(
|
||||
"🎯 [Structural Fast-Path] Only 1 story ring found on feed (likely own profile). Skipping to avoid modal trap."
|
||||
)
|
||||
return None
|
||||
else:
|
||||
# Profile header or other single-story views
|
||||
logger.info(
|
||||
f"🎯 [Structural Fast-Path] Found story ring avatar: {story_nodes[0].resource_id} (desc: '{story_nodes[0].content_desc}')"
|
||||
)
|
||||
return story_nodes[0]
|
||||
|
||||
# --- Structural Grid Fast-Paths ---
|
||||
if (
|
||||
"first image post in profile grid" in intent_lower
|
||||
or "first post" in intent_lower
|
||||
or "first image" in intent_lower
|
||||
):
|
||||
for node in candidates:
|
||||
desc = (node.content_desc or "").lower()
|
||||
if "row 1" in desc and "column 1" in desc:
|
||||
logger.debug(
|
||||
f"🎯 [Structural Fast-Path] Found first grid post: {node.resource_id} (desc: '{desc}')"
|
||||
)
|
||||
return node
|
||||
|
||||
# --- Navigation Tab Fast-Paths ---
|
||||
# Deterministically identify bottom navigation tabs to prevent VLM confusion
|
||||
tab_map = {
|
||||
"home tab": "feed_tab",
|
||||
"feed tab": "feed_tab",
|
||||
"reels tab": "clips_tab",
|
||||
"clips tab": "clips_tab",
|
||||
"explore tab": "search_tab",
|
||||
"search tab": "search_tab",
|
||||
"profile tab": "profile_tab",
|
||||
"message tab": "direct_tab",
|
||||
"direct tab": "direct_tab",
|
||||
# --- Fast-Path: Structural Tab Resolver ---
|
||||
# PRIMARY: Match by resource-id (architectural constants of the Instagram APK).
|
||||
# FALLBACK: Pure geometry if resource-IDs are missing (e.g. custom ROMs).
|
||||
# Resource IDs like feed_tab, profile_tab, clips_tab are NOT localized strings —
|
||||
# they are compile-time Android identifiers that NEVER change across locales.
|
||||
_TAB_MAP = {
|
||||
"home": {"id": "feed_tab", "desc": "home"},
|
||||
"profile": {"id": "profile_tab", "desc": "profile"},
|
||||
"explore": {"id": "search_tab", "desc": "search and explore"},
|
||||
"search": {"id": "search_tab", "desc": "search and explore"},
|
||||
"reels": {"id": "clips_tab", "desc": "reels"},
|
||||
"messages": {"id": "direct_tab", "desc": "message"},
|
||||
}
|
||||
for intent_key, resource_suffix in tab_map.items():
|
||||
if intent_key in intent_lower:
|
||||
# Priority 1: Structural lookup
|
||||
for node in candidates:
|
||||
rid = (node.resource_id or "").lower()
|
||||
if rid.endswith(f":id/{resource_suffix}"):
|
||||
logger.debug(f"🎯 [Structural Fast-Path] Found {intent_key}: {rid}")
|
||||
return node
|
||||
if "tab" in intent_lower and "tap" in intent_lower and "back" not in intent_lower:
|
||||
# Strategy 1: Structural resource-id and content-desc match (O(1), zero ambiguity)
|
||||
for keyword, targets in _TAB_MAP.items():
|
||||
if keyword in intent_lower:
|
||||
for node in candidates:
|
||||
# Bottom 20% guard to ensure it's actually a tab and not someone's name
|
||||
if node.center_y < screen_height * 0.8:
|
||||
continue
|
||||
|
||||
# Priority 2: Fail Fast
|
||||
# If a navigation tab is requested but not structurally present, it means the bottom
|
||||
# bar is likely hidden (e.g. full-screen ad, post detail). We MUST NOT guess
|
||||
# coordinates and we MUST NOT ask the VLM, as this leads to Play Store traps.
|
||||
logger.warning(
|
||||
f"🛡️ [Navigation Guard] Structural {intent_key} not found. Failing intent deterministically."
|
||||
)
|
||||
return None
|
||||
rid = (node.resource_id or "").lower()
|
||||
desc = (node.content_desc or "").lower()
|
||||
text = (node.text or "").lower()
|
||||
|
||||
# Use strictly equal for desc/text to avoid matching "profile picture"
|
||||
if targets["id"] in rid or targets["desc"] == desc or targets["desc"] == text:
|
||||
logger.info(
|
||||
f"📐 [Structural Tab Resolver] Matched '{keyword}' tab structurally. Bypassing VLM."
|
||||
)
|
||||
return node
|
||||
break # keyword matched but no node found — fall through to geometry
|
||||
|
||||
# Strategy 2: Geometric fallback (for edge cases where resource-IDs are stripped)
|
||||
# Requirements: bottom 5% of screen, FrameLayout, long-clickable (tabs are always long-clickable)
|
||||
tab_candidates = []
|
||||
for node in candidates:
|
||||
if node.center_y > screen_height * 0.93:
|
||||
cls_name = (node.class_name or "").lower()
|
||||
is_long_clickable = getattr(node, "long_clickable", False)
|
||||
# Tabs are FrameLayouts that are long-clickable — this excludes:
|
||||
# - Post grid thumbnails (ImageView, not long-clickable)
|
||||
# - ViewGroup containers (wrong class)
|
||||
if "framelayout" in cls_name and is_long_clickable:
|
||||
tab_candidates.append(node)
|
||||
|
||||
if tab_candidates:
|
||||
tab_candidates.sort(key=lambda n: n.center_x)
|
||||
|
||||
# Deduplicate by X cluster (icon inside container shares same X)
|
||||
unique_tabs = []
|
||||
last_x = -1000
|
||||
for t in tab_candidates:
|
||||
if t.center_x - last_x > 100: # Tabs are ~216px apart, use 100px threshold
|
||||
unique_tabs.append(t)
|
||||
last_x = t.center_x
|
||||
|
||||
if len(unique_tabs) >= 4:
|
||||
logger.info(f"📐 [Geometric Tab Fallback] Found {len(unique_tabs)} bottom tabs. Bypassing VLM.")
|
||||
if "home" in intent_lower:
|
||||
return unique_tabs[0]
|
||||
elif "profile" in intent_lower:
|
||||
return unique_tabs[-1]
|
||||
elif "explore" in intent_lower or "search" in intent_lower:
|
||||
# search_tab is 4th from left in 5-tab layout
|
||||
return unique_tabs[3] if len(unique_tabs) >= 5 else unique_tabs[1]
|
||||
elif "reels" in intent_lower:
|
||||
return unique_tabs[1] if len(unique_tabs) >= 5 else unique_tabs[-2]
|
||||
|
||||
# --- Semantic Match Guard ---
|
||||
# If the intent explicitly quotes a target (e.g., "tap 'New Message'"),
|
||||
@@ -441,13 +225,17 @@ class IntentResolver:
|
||||
|
||||
semantic_candidates = []
|
||||
for node in candidates:
|
||||
n_text = (node.text or "").lower()
|
||||
n_desc = (node.content_desc or "").lower()
|
||||
n_text = _humanize_desc((node.text or "").lower())
|
||||
n_desc = _humanize_desc((node.content_desc or "").lower())
|
||||
|
||||
# Check if any of the localized targets match
|
||||
for loc_target in localized_targets:
|
||||
pattern = r"\b" + re.escape(loc_target) + r"\b"
|
||||
if re.search(pattern, n_text) or re.search(pattern, n_desc):
|
||||
if (
|
||||
re.search(pattern, n_text)
|
||||
or re.search(pattern, n_desc)
|
||||
or loc_target in (node.resource_id or "").lower()
|
||||
):
|
||||
semantic_candidates.append(node)
|
||||
break # Found a match, no need to check other localized targets
|
||||
|
||||
@@ -477,10 +265,24 @@ class IntentResolver:
|
||||
# --- Strict VLM Hallucination Guard (Text-only Fallback) ---
|
||||
# For known structural targets that the text-based VLM frequently hallucinates when they are missing,
|
||||
# we enforce a strict failure.
|
||||
if "following list" in intent_lower or "followers list" in intent_lower or "tap message button" in intent_lower:
|
||||
structural_intents = [
|
||||
"following list",
|
||||
"followers list",
|
||||
"message button",
|
||||
"tab",
|
||||
"scroll",
|
||||
"back",
|
||||
"home",
|
||||
"profile",
|
||||
"reels",
|
||||
"search",
|
||||
"explore",
|
||||
]
|
||||
|
||||
if any(si in intent_lower for si in structural_intents):
|
||||
logger.warning(
|
||||
f"🛡️ [Hallucination Guard] Intent '{intent_description}' is a strict structural target. "
|
||||
"Since it wasn't resolved by fast-paths, it is missing. Rejecting VLM fallback."
|
||||
"Since it wasn't resolved by fast-paths, it is either missing or blocked. Rejecting VLM fallback."
|
||||
)
|
||||
return None
|
||||
|
||||
@@ -636,6 +438,10 @@ class IntentResolver:
|
||||
and "com.android.systemui" not in (n.resource_id or "")
|
||||
and "notification:" not in (n.content_desc or "").lower()
|
||||
and "per cent" not in (n.content_desc or "").lower()
|
||||
and "keyboard" not in (n.resource_id or "").lower()
|
||||
and "input_method" not in (n.resource_id or "").lower()
|
||||
and "tastatur" not in (n.content_desc or "").lower()
|
||||
and "eingabetaste" not in (n.content_desc or "").lower()
|
||||
]
|
||||
|
||||
# --- Navigation Conflict Guard ---
|
||||
@@ -659,42 +465,22 @@ class IntentResolver:
|
||||
candidates = filtered_candidates
|
||||
|
||||
# --- Post/Grid Item Guard ---
|
||||
# VLMs frequently hallucinate 'Search' when asked to tap a post. We must pre-filter.
|
||||
if "first post" in intent_lower or "grid item" in intent_lower:
|
||||
grid_candidates = []
|
||||
for node in candidates:
|
||||
desc = (node.content_desc or "").lower()
|
||||
# Posts/grid items usually have 'row X, column Y', 'photos by', or 'reel by'
|
||||
if "row 1" in desc or "column" in desc or "photos by" in desc or "reel by" in desc:
|
||||
grid_candidates.append(node)
|
||||
|
||||
if grid_candidates:
|
||||
logger.info(f"🎯 [Grid Guard] Filtered to {len(grid_candidates)} actual grid candidates.")
|
||||
candidates = grid_candidates
|
||||
# Removed hardcoded English string matching for 'row 1', 'photos by'. We trust the VLM.
|
||||
pass
|
||||
|
||||
# --- Author/Username Guard ---
|
||||
# Prevents VLM from picking the "Profile" nav tab when asked for "post author username".
|
||||
# Geometric constraint: Author and username on profiles/posts are usually in the top half.
|
||||
if "author" in intent_lower or "username" in intent_lower or "profile name" in intent_lower:
|
||||
filtered_candidates = []
|
||||
for node in candidates:
|
||||
res_id = (node.resource_id or "").lower()
|
||||
desc = (node.content_desc or "").lower()
|
||||
if (
|
||||
"tab" in res_id
|
||||
or "navigation" in res_id
|
||||
or "tabbar" in res_id
|
||||
or desc in ["home", "search", "reels", "profile"]
|
||||
):
|
||||
logger.debug(
|
||||
f"🛡️ [Author Guard] Filtered out navigation tab: '{node.content_desc}' ({node.resource_id})"
|
||||
)
|
||||
if node.center_y > (screen_height * 0.85):
|
||||
logger.debug("🛡️ [Author Guard] Filtered out bottom area element.")
|
||||
else:
|
||||
filtered_candidates.append(node)
|
||||
candidates = filtered_candidates
|
||||
|
||||
# --- Reply Guard ---
|
||||
# Prevents VLM from clicking the "Reply to story" or "Send message" input field
|
||||
# when looking for general navigation or "next" buttons.
|
||||
# Prevents VLM from clicking input fields for non-reply intents
|
||||
if (
|
||||
"reply" not in intent_lower
|
||||
and "message" not in intent_lower
|
||||
@@ -704,19 +490,9 @@ class IntentResolver:
|
||||
):
|
||||
filtered_candidates = []
|
||||
for node in candidates:
|
||||
res_id = (node.resource_id or "").lower()
|
||||
text = (node.text or "").lower()
|
||||
cls_name = (node.class_name or "").lower()
|
||||
if (
|
||||
"reply" in res_id
|
||||
or "message" in res_id
|
||||
or "comment" in res_id
|
||||
or "antworten" in text
|
||||
or "send message" in text
|
||||
or "nachricht" in text
|
||||
or "edittext" in cls_name
|
||||
):
|
||||
logger.debug(f"🛡️ [Reply Guard] Filtered out input/message box: '{node.text}' ({node.resource_id})")
|
||||
if "edittext" in cls_name:
|
||||
logger.debug("🛡️ [Reply Guard] Filtered out input field.")
|
||||
else:
|
||||
filtered_candidates.append(node)
|
||||
candidates = filtered_candidates
|
||||
@@ -767,40 +543,40 @@ class IntentResolver:
|
||||
f"Box legend (what each box contains):\n{box_legend}\n\n"
|
||||
f"Your task: Find the exact box number that corresponds to this intent: '{intent_description}'\n\n"
|
||||
f"CRITICAL RULES:\n"
|
||||
f"1. If the intent contains a word in quotes (e.g., 'Search', 'New Message'), you MUST look at the Box legend and pick the box that contains that word (case-insensitive). Do not pick anything else.\n"
|
||||
f"0. MULTILINGUAL UI AWARENESS: The UI might be in any language (English, German, etc.). You MUST translate the intent conceptually. If looking for 'Search', also accept 'Suche'. If looking for 'Following', also accept 'Abonniert'. If looking for 'Message', also accept 'Nachricht'.\n"
|
||||
f"1. If the intent contains a word in quotes (e.g., 'Search', 'New Message'), look at the Box legend and pick the box that contains that word or its localized equivalent.\n"
|
||||
f"2. For icons without text:\n"
|
||||
f" - 'like button' = HEART-SHAPED ICON (♡/❤), usually has desc='Like'.\n"
|
||||
f" - 'comment button' = SPEECH BUBBLE ICON, usually has desc='Comment'.\n"
|
||||
f" - 'like button' = HEART-SHAPED ICON (♡/❤).\n"
|
||||
f" - 'comment button' = SPEECH BUBBLE ICON.\n"
|
||||
f"3. Do NOT select text, captions, or view counts if looking for an icon.\n"
|
||||
f"4. Ignore numbers inside the text itself. Do not confuse the text '19' with Box [19].\n"
|
||||
f"5. If the intent contains 'following', you MUST pick the box containing 'following'. Do NOT pick 'followers' or 'Follow'.\n"
|
||||
f"6. If the intent is to tap a 'post', 'first post', or 'grid item':\n"
|
||||
f" - Look for boxes with descriptions containing 'photos by', 'Reel by', or 'row 1, column 1'.\n"
|
||||
f" - Pick the FIRST matching box index (e.g. if [0] says '6 photos...', return 0, NOT 6).\n"
|
||||
f"5. If the intent is to tap a 'post', 'first post', or 'grid item':\n"
|
||||
f" - Look for boxes indicating a photo or video by a user (e.g., 'photos by', 'Foto von', or grid coordinates like 'row 1' / 'Reihe 1').\n"
|
||||
f" - Pick the FIRST matching box index.\n"
|
||||
f" - Do NOT pick navigation buttons like 'Search'.\n"
|
||||
f"7. If the intent is a bottom navigation tab (e.g. 'profile tab', 'home tab'):\n"
|
||||
f"6. If the intent is a bottom navigation tab (e.g. 'profile tab', 'home tab'):\n"
|
||||
f" - These are always at the BOTTOM edge of the screen.\n"
|
||||
f" - 'profile tab' is usually the furthest right icon (your avatar).\n"
|
||||
f" - 'home tab' is the furthest left icon (house).\n"
|
||||
f" - 'explore tab' is the magnifying glass.\n"
|
||||
f" - 'reels tab' is the video clapperboard.\n"
|
||||
f"8. If the intent involves 'author username' or 'author profile':\n"
|
||||
f" - Pick the profile picture (e.g. 'Profile picture of <username>') or the username text.\n"
|
||||
f" - NEVER pick a 'Follow' button. Do NOT pick 'Follow <username>'.\n"
|
||||
f"9. If the intent is 'save post':\n"
|
||||
f"7. If the intent involves 'author username' or 'author profile':\n"
|
||||
f" - Pick the profile picture or the username text.\n"
|
||||
f" - NEVER pick a 'Follow' button.\n"
|
||||
f"8. If the intent is 'save post':\n"
|
||||
f" - The save icon is the bookmark icon on the bottom right of the post image/video.\n"
|
||||
f" - Usually has desc='Add to Saved' or 'Save'. Do NOT pick the post text or other action buttons.\n"
|
||||
f"10. DISTINGUISHING BOTTOM TABS vs CONTENT BUTTONS:\n"
|
||||
f"9. DISTINGUISHING BOTTOM TABS vs CONTENT BUTTONS:\n"
|
||||
f" - Bottom Navigation Tabs (Home, Search, Reels, Profile) are ALWAYS at the very bottom (y > 2100).\n"
|
||||
f" - Content Interaction Buttons (Like, Comment, Share, Reactions, Message Input) are attached to posts or threads, NOT the bottom nav bar.\n"
|
||||
f" - If looking for 'message input' or 'type message', do NOT select 'reactions' or emoji icons. Look for an empty text box or 'Message...'.\n"
|
||||
f"11. If the intent is 'feed post content' or 'post media content':\n"
|
||||
f" - Pick the largest box that contains the actual image or video, usually described as 'Photo', 'Video', or 'Carousel'.\n"
|
||||
f"12. DO NOT HALLUCINATE. If you are on the wrong screen (e.g. a 'New Post' creation screen when you want a profile), or if the exact target is simply NOT visible, you MUST return null. Returning a wrong box number will crash the bot.\n"
|
||||
f"13. EXTREME GUARD: NEVER pick an input field (class='EditText'), 'Send Message', or 'Reply to story' box unless the intent EXPLICITLY asks you to type or reply.\n"
|
||||
f"14. EXTREME GUARD: Do NOT pick items that are 'long_clickable=True' if your intent is just a simple navigation click, as this can trigger unwanted long-press context menus.\n"
|
||||
f"15. If the intent is 'tap post username', DO NOT pick random gallery folders like 'Recents' or 'Select album'. Return null.\n\n"
|
||||
f'Reply ONLY with a valid JSON object: {{"box": <number>}} or {{"box": null}}'
|
||||
f"10. If the intent is 'feed post content' or 'post media content':\n"
|
||||
f" - Pick the largest box that contains the actual image or video.\n"
|
||||
f"11. DO NOT HALLUCINATE. If you are on the wrong screen, or if the exact target is simply NOT visible, you MUST return null.\n"
|
||||
f"12. EXTREME GUARD: NEVER pick an input field (class='EditText') unless the intent EXPLICITLY asks you to type or reply.\n"
|
||||
f"13. EXTREME GUARD: Do NOT pick items that are 'long_clickable=True' if your intent is just a simple navigation click.\n"
|
||||
f"14. If the intent is 'tap post username', DO NOT pick random gallery folders like 'Recents' or 'Select album'. Return null.\n"
|
||||
f"15. EXTREME GUARD: If the intent is to tap 'following' or 'followers' list, NEVER pick a 'Follow' or 'Follow back' button.\n\n"
|
||||
f"VALID BOX NUMBERS: {list(box_map.keys())}\n"
|
||||
f'Reply ONLY with a valid JSON object: {{"box": <number from VALID BOX NUMBERS>}} or {{"box": null}}'
|
||||
)
|
||||
|
||||
try:
|
||||
@@ -814,6 +590,12 @@ class IntentResolver:
|
||||
)
|
||||
data = json.loads(res)
|
||||
box_idx = self._parse_box_index(data)
|
||||
|
||||
# Additional safety check to prevent hallucinated numbers
|
||||
if box_idx not in box_map:
|
||||
logger.warning(f"👁️ [Visual Discovery] VLM hallucinated invalid box number: {box_idx}")
|
||||
box_idx = None
|
||||
|
||||
selected = self._validate_and_get_node(box_idx, box_map)
|
||||
|
||||
if selected:
|
||||
@@ -922,12 +704,13 @@ class IntentResolver:
|
||||
f"Goal: Find the single best UI element to interact with to satisfy the intent: '{intent_description}'.\n"
|
||||
f"Candidates:\n" + "\n".join(node_context) + "\n\n"
|
||||
"CRITICAL RULES:\n"
|
||||
"0. MULTILINGUAL UI AWARENESS: The UI might be in any language (English, German, etc.). You MUST translate the intent conceptually and find the corresponding localized element.\n"
|
||||
"1. If the intent is a bottom navigation tab (e.g. 'profile tab', 'home tab'):\n"
|
||||
" - These are always at the BOTTOM of the screen (typically y > 2100).\n"
|
||||
" - 'profile tab' is usually the furthest right.\n"
|
||||
" - 'home tab' is the furthest left.\n"
|
||||
" - Do NOT select 'Go to <user>'s profile' or other header text.\n"
|
||||
"2. EXTREME GUARD: NEVER pick an input field (EditText), 'Send Message', or 'Reply' unless explicitly asked to type.\n"
|
||||
"2. EXTREME GUARD: NEVER pick an input field (EditText) unless explicitly asked to type.\n"
|
||||
"3. EXTREME GUARD: If you are on the wrong screen entirely (e.g. 'Select album' gallery) instead of a profile, return null.\n"
|
||||
"4. If none of the candidates clearly and safely match the intent, return null. DO NOT guess.\n\n"
|
||||
"Reply ONLY with a valid JSON object strictly matching this schema:\n"
|
||||
|
||||
@@ -256,6 +256,12 @@ class SituationalAwarenessEngine:
|
||||
for elem in root.iter("node"):
|
||||
a = elem.attrib
|
||||
pkg = a.get("package", "")
|
||||
|
||||
# CRITICAL: Exclude system UI (status bar) from signature because it is
|
||||
# identical across all screens and takes up the top 25 visual spots!
|
||||
if pkg == "com.android.systemui":
|
||||
continue
|
||||
|
||||
if pkg:
|
||||
packages.add(pkg)
|
||||
|
||||
@@ -282,19 +288,31 @@ class SituationalAwarenessEngine:
|
||||
parts.append(f"desc='{desc}'")
|
||||
if clickable == "true":
|
||||
parts.append("CLICKABLE")
|
||||
cy_val = 0
|
||||
if bounds:
|
||||
nums = [int(n) for n in re.findall(r"\d+", bounds)]
|
||||
if len(nums) == 4:
|
||||
cx = (nums[0] + nums[2]) // 2
|
||||
cy = (nums[1] + nums[3]) // 2
|
||||
parts.append(f"bounds={bounds} center=({cx},{cy})")
|
||||
cy_val = (nums[1] + nums[3]) // 2
|
||||
parts.append(f"bounds={bounds} center=({cx},{cy_val})")
|
||||
else:
|
||||
parts.append(f"bounds={bounds}")
|
||||
|
||||
elements.append(" | ".join(parts))
|
||||
elements.append((cy_val, " | ".join(parts)))
|
||||
|
||||
sig = f"PACKAGES: {', '.join(sorted(packages))}\n"
|
||||
sig += "\n".join(elements[-50:]) # Keep the last 50 elements (highest Z-index/foreground)
|
||||
|
||||
# Sort by vertical center (Y) coordinate to get visual top-to-bottom
|
||||
elements.sort(key=lambda x: x[0])
|
||||
visual_elements = [e[1] for e in elements]
|
||||
|
||||
if len(visual_elements) > 50:
|
||||
# Keep top 25 (headers, titles) and bottom 25 (tabs, nav) visually
|
||||
selected_elements = visual_elements[:25] + visual_elements[-25:]
|
||||
else:
|
||||
selected_elements = visual_elements
|
||||
|
||||
sig += "\n".join(selected_elements)
|
||||
return sig[:3000]
|
||||
|
||||
def _compute_situation_hash(self, compressed: str) -> str:
|
||||
@@ -620,27 +638,32 @@ class SituationalAwarenessEngine:
|
||||
"""
|
||||
Structurally scan the raw XML for any clickable element that semantically
|
||||
represents a dismiss/close/cancel action. Returns None if nothing found.
|
||||
Zero hardcoding — all coordinates come from the XML itself.
|
||||
Zero hardcoding of localized text — relies purely on resource-ids and VLM discovery fallback.
|
||||
"""
|
||||
import re
|
||||
|
||||
# Patterns for dismiss-type buttons: text or content-desc containing dismiss keywords
|
||||
dismiss_keywords = (
|
||||
"cancel",
|
||||
# Patterns for dismiss-type buttons based purely on resource IDs or text
|
||||
dismiss_id_keywords = (
|
||||
"close",
|
||||
"cancel",
|
||||
"dismiss",
|
||||
"negative_button",
|
||||
"button2", # Standard Android negative button
|
||||
"not_now",
|
||||
"skip",
|
||||
"clear",
|
||||
"decline",
|
||||
)
|
||||
|
||||
dismiss_text_keywords = (
|
||||
"close",
|
||||
"cancel",
|
||||
"dismiss",
|
||||
"not now",
|
||||
"nicht jetzt",
|
||||
"abbrechen",
|
||||
"schließen",
|
||||
"skip",
|
||||
"überspringen",
|
||||
"no thanks",
|
||||
"nein danke",
|
||||
"got it",
|
||||
"ok",
|
||||
"done",
|
||||
"fertig",
|
||||
"decline",
|
||||
"clear",
|
||||
)
|
||||
|
||||
# Extract each <node ...> or <node ... /> tag individually
|
||||
@@ -649,11 +672,13 @@ class SituationalAwarenessEngine:
|
||||
candidates = []
|
||||
for tag in node_tags:
|
||||
# Extract attributes independently — order doesn't matter
|
||||
id_m = re.search(r'resource-id="([^"]*)"', tag)
|
||||
text_m = re.search(r'text="([^"]*)"', tag)
|
||||
desc_m = re.search(r'content-desc="([^"]*)"', tag)
|
||||
click_m = re.search(r'clickable="([^"]*)"', tag)
|
||||
bounds_m = re.search(r'bounds="\[(\d+),(\d+)\]\[(\d+),(\d+)\]"', tag)
|
||||
|
||||
rid = (id_m.group(1) if id_m else "").strip().lower()
|
||||
text = (text_m.group(1) if text_m else "").strip().lower()
|
||||
desc = (desc_m.group(1) if desc_m else "").strip().lower()
|
||||
clickable = (click_m.group(1) if click_m else "").lower()
|
||||
@@ -661,24 +686,32 @@ class SituationalAwarenessEngine:
|
||||
if clickable != "true" or not bounds_m:
|
||||
continue
|
||||
|
||||
label = text or desc
|
||||
if not label:
|
||||
continue
|
||||
|
||||
for kw in dismiss_keywords:
|
||||
if kw in label:
|
||||
x1, y1 = int(bounds_m.group(1)), int(bounds_m.group(2))
|
||||
x2, y2 = int(bounds_m.group(3)), int(bounds_m.group(4))
|
||||
cx, cy = (x1 + x2) // 2, (y1 + y2) // 2
|
||||
candidates.append((kw, cx, cy, label))
|
||||
matched = False
|
||||
for kw in dismiss_id_keywords:
|
||||
if kw in rid:
|
||||
matched = True
|
||||
break
|
||||
|
||||
if not matched:
|
||||
for kw in dismiss_text_keywords:
|
||||
if kw == text or kw == desc:
|
||||
matched = True
|
||||
break
|
||||
|
||||
if matched:
|
||||
x1, y1 = int(bounds_m.group(1)), int(bounds_m.group(2))
|
||||
x2, y2 = int(bounds_m.group(3)), int(bounds_m.group(4))
|
||||
cx, cy = (x1 + x2) // 2, (y1 + y2) // 2
|
||||
ident = rid or text or desc or "dismiss_button"
|
||||
candidates.append((cx, cy, ident))
|
||||
|
||||
if candidates:
|
||||
# Prefer exact match ("cancel" > "ok"), pick first match
|
||||
# Prefer elements lower on the screen (highest Y) for dialog dismiss buttons
|
||||
candidates.sort(key=lambda c: c[1], reverse=True)
|
||||
best = candidates[0]
|
||||
kw, cx, cy, label = best
|
||||
logger.info(f"🔍 [SAE Structural] Found dismiss target '{label}' at ({cx}, {cy})")
|
||||
return EscapeAction("click", x=cx, y=cy, reason=f"Structural dismiss: '{label}'")
|
||||
cx, cy, ident = best
|
||||
logger.info(f"🔍 [SAE Structural] Found dismiss target '{ident}' at ({cx}, {cy})")
|
||||
return EscapeAction("click", x=cx, y=cy, reason=f"Structural dismiss: '{ident}'")
|
||||
|
||||
return None
|
||||
|
||||
@@ -900,6 +933,23 @@ class SituationalAwarenessEngine:
|
||||
|
||||
logger.warning(f"🔍 [SAE] Obstacle detected: {situation.value} (attempt {attempt + 1}/{max_attempts})")
|
||||
|
||||
# ── O(1) Fast-Path for Locked Screen ──
|
||||
if situation == SituationType.OBSTACLE_LOCKED_SCREEN:
|
||||
logger.warning("⚡ [SAE Fast-Path] Screen is OFF or LOCKED. Forcing wake/unlock...")
|
||||
action = EscapeAction("unlock", reason="O(1) fast-path to unlock screen")
|
||||
self._execute_escape(action)
|
||||
|
||||
# Check if we recovered
|
||||
post_xml = self.device.dump_hierarchy()
|
||||
if self.perceive(post_xml) == SituationType.NORMAL:
|
||||
logger.info("✅ [SAE Fast-Path] Screen unlocked and recovered successfully!")
|
||||
self._consecutive_failures = 0
|
||||
return True
|
||||
|
||||
logger.warning("⚠️ [SAE Fast-Path] unlock did not return to NORMAL. Retrying...")
|
||||
self._consecutive_failures += 1
|
||||
continue
|
||||
|
||||
# ── O(1) Fast-Path for Foreign Apps ──
|
||||
if situation == SituationType.OBSTACLE_FOREIGN_APP:
|
||||
logger.warning("⚡ [SAE Fast-Path] Foreign App detected. Bypassing LLM and killing immediately.")
|
||||
|
||||
Reference in New Issue
Block a user