diff --git a/GramAddict/core/perception/intent_resolver.py b/GramAddict/core/perception/intent_resolver.py index eacf892..df61fb0 100644 --- a/GramAddict/core/perception/intent_resolver.py +++ b/GramAddict/core/perception/intent_resolver.py @@ -40,7 +40,9 @@ class IntentResolver: # Structural Guards # ────────────────────────────────────────────── - def filter_navigation_conflicts(self, candidates: List[SpatialNode], intent_description: str) -> List[SpatialNode]: + def filter_navigation_conflicts( + self, candidates: List[SpatialNode], intent_description: str, screen_height: int = 2400 + ) -> List[SpatialNode]: """ Prevents VLM from confusing navigation-bar buttons (Back, Close) with bottom tab-bar buttons (Home, Profile, Search). @@ -86,6 +88,17 @@ class IntentResolver: ) continue + # NEW REGRESSION FIX 2026-05-01: Tab intents must never pick top-screen elements or headers + # UPDATE: Actually, tabs are always at the very bottom. Filter out anything above 85% of screen height. + if is_tab_intent: + is_not_at_bottom = node.center_y < (screen_height * 0.85) + if is_not_at_bottom: + logger.debug( + f"🛡️ [Tab Height Guard] Excluded non-bottom element '{node.resource_id}' " + f"(desc='{node.content_desc}', y={node.center_y}) for tab intent '{intent_description}'" + ) + continue + if is_author_intent and is_nav_tab: logger.debug( f"🛡️ [Author Tab Guard] Excluded nav tab '{node.resource_id}' " @@ -121,6 +134,16 @@ class IntentResolver: if intent_lower in abstract_goals: return None + # --- Strict Structural Fast-Paths --- + # Bypass VLM for deterministically identifiable UI components + if "message text box" in intent_lower or "message input" in intent_lower or "type message" in intent_lower: + for node in candidates: + rid = (node.resource_id or "").lower() + text = (node.text or "").lower() + if "composer_edittext" in rid or "message…" in text or "message..." in text: + logger.info(f"🎯 [Structural Fast-Path] Found message input field: {rid}") + return node + # --- Semantic Match Guard --- # If the intent explicitly quotes a target (e.g., "tap 'New Message'"), # we strictly filter candidates to those whose text or content_desc contains the quote. @@ -158,10 +181,7 @@ class IntentResolver: hasattr(device, "screenshot") or hasattr(getattr(device, "deviceV2", None), "screenshot") ): logger.info("📸 Device screenshot capability detected. Enforcing visual discovery.") - visual_res = self._visual_discovery(intent_description, candidates, device) - if visual_res is not None: - return visual_res - logger.warning("👁️ [IntentResolver] Visual discovery yielded None. Falling back to text-based resolution.") + return self._visual_discovery(intent_description, candidates, device, screen_height=screen_height) # --- Strict VLM Hallucination Guard (Text-only Fallback) --- # For known structural targets that the text-based VLM frequently hallucinates when they are missing, @@ -175,7 +195,7 @@ class IntentResolver: # ── FALLBACK: Text-based VLM resolution ── # Only used when device is unavailable (e.g., unit tests without screenshots). - return self._text_based_resolve(intent_description, candidates, device) + return self._text_based_resolve(intent_description, candidates, device, screen_height=screen_height) # ────────────────────────────────────────────── # Visual Discovery (Set-of-Mark Prompting) @@ -303,7 +323,7 @@ class IntentResolver: return annotated_b64, box_map def _visual_discovery( - self, intent_description: str, candidates: List[SpatialNode], device + self, intent_description: str, candidates: List[SpatialNode], device, screen_height: int = 2400 ) -> Optional[SpatialNode]: """ Vision-first intent resolution via Set-of-Mark (SoM) prompting. @@ -330,7 +350,7 @@ class IntentResolver: # --- Navigation Conflict Guard --- # Prevents VLM from confusing Back buttons with tab buttons # Production bug 2026-04-30: VLM picked Back for "tap profile tab" - candidates = self.filter_navigation_conflicts(candidates, intent_description) + candidates = self.filter_navigation_conflicts(candidates, intent_description, screen_height=screen_height) # --- Strict Button Guard --- # If the intent specifically asks for a "button", "icon", or "tab", @@ -361,6 +381,26 @@ class IntentResolver: logger.info(f"🎯 [Grid Guard] Filtered to {len(grid_candidates)} actual grid candidates.") candidates = grid_candidates + # --- Author/Username Guard --- + # Prevents VLM from picking the "Profile" nav tab when asked for "post author username". + if "author" in intent_lower or "username" in intent_lower or "profile name" in intent_lower: + filtered_candidates = [] + for node in candidates: + res_id = (node.resource_id or "").lower() + desc = (node.content_desc or "").lower() + if ( + "tab" in res_id + or "navigation" in res_id + or "tabbar" in res_id + or desc in ["home", "search", "reels", "profile"] + ): + logger.debug( + f"🛡️ [Author Guard] Filtered out navigation tab: '{node.content_desc}' ({node.resource_id})" + ) + else: + filtered_candidates.append(node) + candidates = filtered_candidates + try: annotated_b64, box_map = self._annotate_screenshot_with_candidates(device, candidates) except Exception as e: @@ -422,7 +462,11 @@ class IntentResolver: f"9. If the intent is 'save post':\n" f" - The save icon is the bookmark icon on the bottom right of the post image/video.\n" f" - Usually has desc='Add to Saved' or 'Save'. Do NOT pick the post text or other action buttons.\n" - f"10. If the exact control is NOT visible, return null. Do NOT guess.\n\n" + f"10. DISTINGUISHING BOTTOM TABS vs CONTENT BUTTONS:\n" + f" - Bottom Navigation Tabs (Home, Search, Reels, Profile) are ALWAYS at the very bottom (y > 2100).\n" + f" - Content Interaction Buttons (Like, Comment, Share, Reactions, Message Input) are attached to posts or threads, NOT the bottom nav bar.\n" + f" - If looking for 'message input' or 'type message', do NOT select 'reactions' or emoji icons. Look for an empty text box or 'Message...'.\n" + f"11. If the exact control is NOT visible, return null. Do NOT guess.\n\n" f'Reply ONLY with a valid JSON object: {{"box": }} or {{"box": null}}' ) @@ -465,7 +509,7 @@ class IntentResolver: # ────────────────────────────────────────────── def _text_based_resolve( - self, intent_description: str, candidates: List[SpatialNode], device=None + self, intent_description: str, candidates: List[SpatialNode], device=None, screen_height: int = 2400 ) -> Optional[SpatialNode]: """ Fallback resolution via text descriptions of XML nodes. @@ -477,6 +521,9 @@ class IntentResolver: intent_lower = intent_description.lower() filtered_candidates = [n for n in candidates if n.area < 500000] + filtered_candidates = self.filter_navigation_conflicts( + filtered_candidates, intent_description, screen_height=screen_height + ) if "profile" in intent_lower: filtered_candidates = [ n diff --git a/tests/e2e/test_production_bug_foreign_app_20260501.py b/tests/e2e/test_production_bug_foreign_app_20260501.py new file mode 100644 index 0000000..f305d11 --- /dev/null +++ b/tests/e2e/test_production_bug_foreign_app_20260501.py @@ -0,0 +1,40 @@ +import os + +from GramAddict.core.perception.intent_resolver import IntentResolver +from GramAddict.core.perception.spatial_parser import SpatialParser + + +class TestExploreTabGuard: + def test_visual_discovery_never_picks_header_for_tab_intent(self, make_real_device_with_image): + """ + Production bug 2026-05-01 22:01: VLM picked the profile header (box 36) + instead of the Explore tab at the bottom, which unexpectedly opened the Play Store. + We must guard against picking top-screen elements when looking for bottom tabs. + This test uses a REAL LLM call via the image to prove we don't hallucinate. + """ + xml_path = os.path.join(os.path.dirname(__file__), "fixtures", "other_profile_real.xml") + # We use a dummy valid image so the VLM can process the annotated boxes from the XML. + # It won't find the explore tab, so it should return None rather than hallucinating the header. + img_path = os.path.join(os.path.dirname(__file__), "..", "fixtures", "user_profile_dump.jpg") + + with open(xml_path, "r", encoding="utf-8") as f: + xml = f.read() + + parser = SpatialParser() + root = parser.parse(xml) + candidates = parser.get_clickable_nodes(root) + + # Real device that returns the real image + device = make_real_device_with_image(img_path, xml) + + resolver = IntentResolver() + intent = "tap explore tab" + + # This will trigger an actual VLM call since device.screenshot() will return the real image + selected_node = resolver.resolve(intent, candidates, device=device, screen_height=2400) + + # Since the explore tab is NOT in the other_profile_real.xml, it must return None. + # It must absolutely NOT return the profile header. + assert ( + selected_node is None + ), f"Hallucinated a node instead of returning None! Picked: {selected_node.content_desc or selected_node.text}"