diff --git a/GramAddict/core/perception/intent_resolver.py b/GramAddict/core/perception/intent_resolver.py index b7712e5..7a18e58 100644 --- a/GramAddict/core/perception/intent_resolver.py +++ b/GramAddict/core/perception/intent_resolver.py @@ -36,6 +36,50 @@ class IntentResolver: 3. Fallback → text-based VLM (when no device/screenshot available) """ + # ────────────────────────────────────────────── + # Structural Guards + # ────────────────────────────────────────────── + + def filter_navigation_conflicts( + self, candidates: List[SpatialNode], intent_description: str + ) -> List[SpatialNode]: + """ + Prevents VLM from confusing navigation-bar buttons (Back, Close) + with bottom tab-bar buttons (Home, Profile, Search). + + Production bug 2026-04-30: VLM picked action_bar_button_back + for "tap profile tab" → account switch failed. + + Rules: + - For tab intents: exclude nodes with "back" in resource_id or + content_desc == "Back" + - For back/close intents: no filtering (Back is the correct target) + """ + intent_lower = intent_description.lower() + + # Only apply for tab-related intents + is_tab_intent = "tab" in intent_lower and "back" not in intent_lower + if not is_tab_intent: + return candidates + + filtered = [] + for node in candidates: + rid = (node.resource_id or "").lower() + desc = (node.content_desc or "").lower() + + is_back = "back" in rid or desc == "back" + is_close = "close" in rid or desc == "close" + + if is_back or is_close: + logger.debug( + f"🛡️ [Nav Conflict Guard] Excluded '{node.resource_id}' " + f"(desc='{node.content_desc}') for tab intent '{intent_description}'" + ) + continue + filtered.append(node) + + return filtered + # ────────────────────────────────────────────── # Public API # ────────────────────────────────────────────── @@ -259,6 +303,11 @@ class IntentResolver: and "per cent" not in (n.content_desc or "").lower() ] + # --- Navigation Conflict Guard --- + # Prevents VLM from confusing Back buttons with tab buttons + # Production bug 2026-04-30: VLM picked Back for "tap profile tab" + candidates = self.filter_navigation_conflicts(candidates, intent_description) + # --- Strict Button Guard --- # If the intent specifically asks for a "button", "icon", or "tab", # filter out candidates that contain long text (e.g. captions, comments) diff --git a/tests/e2e/test_workflow_account_switch.py b/tests/e2e/test_workflow_account_switch.py new file mode 100644 index 0000000..531dcb8 --- /dev/null +++ b/tests/e2e/test_workflow_account_switch.py @@ -0,0 +1,152 @@ +""" +E2E: Account Switcher Regression +=================================== +PRODUCTION BUG 2026-04-30 11:50: +Bot was on OTHER_PROFILE (ehsan.nura). Account switcher called +telepath.find_best_node(xml, "tap profile tab") which returned the +BACK BUTTON (id: action_bar_button_back, desc: Back) instead of the +actual profile tab in the bottom tab bar. + +This caused: +1. Bot navigated away from OwnProfile instead of opening account selector +2. Account switcher couldn't find target account in bottom sheet +3. Session halted: "Cannot verify or switch to target account" + +Root cause: VLM confused "profile tab" with "Back" button because +Back is more visually prominent on OTHER_PROFILE screens. + +Mock: ONLY the device (XML dumps). +Real: Cognitive stack, PluginRegistry, SessionState, Config. +""" + +import os +import xml.etree.ElementTree as ET + +E2E_FIXTURES_DIR = os.path.join(os.path.dirname(__file__), "fixtures") +FIXTURES_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), "fixtures") + + +def _load_fixture(name, e2e=False): + base = E2E_FIXTURES_DIR if e2e else FIXTURES_DIR + with open(os.path.join(base, name), "r", encoding="utf-8") as f: + return f.read() + + +def _extract_tab_bar_nodes(xml_str): + """Extract only nodes that are in the bottom tab bar.""" + import re + + clean_xml = re.sub(r"<\?xml.*?\?>", "", xml_str).strip() + root = ET.fromstring(clean_xml) + + tab_nodes = [] + for elem in root.iter("node"): + res_id = elem.attrib.get("resource-id", "") + content_desc = elem.attrib.get("content-desc", "") + bounds = elem.attrib.get("bounds", "") + + # Tab bar nodes have specific resource-ids and are at the bottom + if "tab_bar" in res_id or "tab_icon" in res_id: + tab_nodes.append({ + "resource-id": res_id, + "content-desc": content_desc, + "text": elem.attrib.get("text", ""), + "bounds": bounds, + "clickable": elem.attrib.get("clickable", "false"), + }) + + # Profile tab specifically + if "profile" in content_desc.lower() and "tab" in content_desc.lower(): + tab_nodes.append({ + "resource-id": res_id, + "content-desc": content_desc, + "text": elem.attrib.get("text", ""), + "bounds": bounds, + "clickable": elem.attrib.get("clickable", "false"), + }) + + return tab_nodes + + +def test_other_profile_has_back_button_and_profile_tab(): + """ + STRUCTURAL GUARD: On OTHER_PROFILE, both a Back button AND a profile + tab must exist. If our XML fixture doesn't have both, we can't test + the confusion bug. + """ + xml = _load_fixture("other_profile_real.xml", e2e=True) + + has_back = "action_bar_button_back" in xml or 'content-desc="Back"' in xml + has_profile_tab = "profile_tab" in xml or 'content-desc="Profile"' in xml + + assert has_back, ( + "other_profile_real.xml is missing the Back button — " + "cannot reproduce the VLM confusion bug" + ) + # Note: if profile tab is missing from fixture, the bug is even worse + # because there's nothing correct for the VLM to find + + +def test_account_switcher_navigates_to_own_profile_first(): + """ + The account_switcher.verify_and_switch_account() MUST first navigate + to OwnProfile before checking identity. If it starts on OTHER_PROFILE, + the nav must happen first. + """ + import inspect + from GramAddict.core.account_switcher import verify_and_switch_account + + source = inspect.getsource(verify_and_switch_account) + + # The function must call navigate_to("OwnProfile") BEFORE + # calling find_best_node for profile tab + nav_pos = source.find('navigate_to("OwnProfile"') + assert nav_pos >= 0, ( + "account_switcher does not navigate to OwnProfile — " + "it will try to switch from arbitrary screens!" + ) + + +def test_back_button_is_not_profile_tab(): + """ + STRUCTURAL ASSERTION: The Back button (action_bar_button_back) + must NEVER be considered a valid match for "tap profile tab". + + This is the exact bug: VLM returned Back button for "tap profile tab" + which navigated away instead of opening the account selector. + """ + xml = _load_fixture("other_profile_real.xml", e2e=True) + + import re + + clean_xml = re.sub(r"<\?xml.*?\?>", "", xml).strip() + root = ET.fromstring(clean_xml) + + back_buttons = [] + profile_tabs = [] + + for elem in root.iter("node"): + res_id = elem.attrib.get("resource-id", "") + desc = elem.attrib.get("content-desc", "") + bounds = elem.attrib.get("bounds", "") + + if "action_bar_button_back" in res_id or desc == "Back": + back_buttons.append({"id": res_id, "desc": desc, "bounds": bounds}) + + if "profile" in desc.lower() and elem.attrib.get("selected", "false") == "true": + profile_tabs.append({"id": res_id, "desc": desc, "bounds": bounds}) + + assert len(back_buttons) > 0, ( + "No Back button found in OTHER_PROFILE XML — fixture is incomplete" + ) + + # The back button bounds must be in the TOP of the screen (action bar) + # while profile tab must be at the BOTTOM (tab bar) + for back in back_buttons: + coords = re.findall(r"\d+", back["bounds"]) + if len(coords) >= 4: + back_top = int(coords[1]) + assert back_top < 500, ( + f"Back button at y={back_top} — expected top of screen! " + "If this is in the tab bar, the VLM confusion is structural." + ) diff --git a/tests/e2e/test_workflow_vlm_tab_confusion.py b/tests/e2e/test_workflow_vlm_tab_confusion.py new file mode 100644 index 0000000..06d871f --- /dev/null +++ b/tests/e2e/test_workflow_vlm_tab_confusion.py @@ -0,0 +1,121 @@ +""" +E2E: VLM Tab vs. Back Button Confusion Regression +==================================================== +PRODUCTION BUG 2026-04-30 11:50: +When the bot is on OTHER_PROFILE and asks for "tap profile tab", the +VLM returned the BACK BUTTON instead of the bottom tab bar profile tab. + +This test exercises the intent resolver with real OTHER_PROFILE XML +to verify it never confuses navigation-bar buttons with bottom-tab buttons. + +This is a STRUCTURAL test — it verifies the XML filtering logic, +not the VLM. The VLM is NOT needed here because the intent resolver +has structural guards that should prevent this. + +Mock: ONLY the device (XML dumps). +Real: IntentResolver, SpatialParser — all real. +""" + +import os +import re +import xml.etree.ElementTree as ET + +E2E_FIXTURES_DIR = os.path.join(os.path.dirname(__file__), "fixtures") + + +def _load_e2e_fixture(name): + with open(os.path.join(E2E_FIXTURES_DIR, name), "r", encoding="utf-8") as f: + return f.read() + + +def _find_all_clickable_nodes(xml_str): + """Parse XML and return all clickable nodes with their metadata.""" + clean_xml = re.sub(r"<\?xml.*?\?>", "", xml_str).strip() + root = ET.fromstring(clean_xml) + + nodes = [] + for elem in root.iter("node"): + if elem.attrib.get("clickable", "false") == "true": + res_id = elem.attrib.get("resource-id", "") + desc = elem.attrib.get("content-desc", "") + text = elem.attrib.get("text", "") + bounds = elem.attrib.get("bounds", "") + + coords = re.findall(r"\d+", bounds) + y_center = 0 + if len(coords) >= 4: + y_center = (int(coords[1]) + int(coords[3])) // 2 + + nodes.append({ + "resource-id": res_id, + "content-desc": desc, + "text": text, + "bounds": bounds, + "y_center": y_center, + }) + return nodes + + +def test_back_button_never_valid_for_profile_tab_intent(): + """ + Structural assertion: When searching for 'tap profile tab' candidates, + nodes with resource-id containing 'back' or content-desc='Back' + must NEVER be selected. + + This is the EXACT bug from production. The VLM picked box [4] + which was action_bar_button_back (desc='Back'). + """ + xml = _load_e2e_fixture("other_profile_real.xml") + nodes = _find_all_clickable_nodes(xml) + + back_nodes = [ + n for n in nodes + if "back" in n["resource-id"].lower() or n["content-desc"].lower() == "back" + ] + + tab_bar_nodes = [ + n for n in nodes + if n["y_center"] > 2000 # Tab bar is at the very bottom + ] + + assert len(back_nodes) > 0, ( + "No Back button found — cannot validate the confusion guard." + ) + + # Back buttons must be in the top portion of the screen (action bar) + for node in back_nodes: + assert node["y_center"] < 500, ( + f"Back button at y={node['y_center']} — expected top of screen. " + f"Node: id='{node['resource-id']}', desc='{node['content-desc']}'" + ) + + # Back button IDs must not contain 'profile' or 'tab' + for node in back_nodes: + combined = f"{node['resource-id']} {node['content-desc']}".lower() + assert "profile" not in combined or "back" in combined, ( + f"Back button has 'profile' in its identifiers — " + f"this would confuse semantic matching! Node: {node}" + ) + + +def test_profile_tab_exists_in_tab_bar(): + """ + The bottom tab bar must contain a profile-related tab. + If it's missing, the VLM has nothing correct to select. + """ + xml = _load_e2e_fixture("other_profile_real.xml") + nodes = _find_all_clickable_nodes(xml) + + # Tab bar nodes are typically at y > 2200 on a 2400px screen + bottom_nodes = [n for n in nodes if n["y_center"] > 2200] + + # At least one bottom node should exist (the tab bar) + # Note: This might fail if the fixture was captured with tab bar hidden + if len(bottom_nodes) == 0: + # Check if tab_bar resource-id exists anywhere + has_tab_bar = "tab_bar" in xml + assert has_tab_bar, ( + "No tab bar found in other_profile_real.xml — " + "the bot has NO correct target for 'tap profile tab'! " + "This means the VLM will ALWAYS pick the wrong element." + ) diff --git a/tests/unit/test_intent_resolver_back_guard.py b/tests/unit/test_intent_resolver_back_guard.py new file mode 100644 index 0000000..d1e35cb --- /dev/null +++ b/tests/unit/test_intent_resolver_back_guard.py @@ -0,0 +1,152 @@ +""" +Unit Test: Intent Resolver Back-Button Guard +============================================== +RED phase: Tests that the intent resolver's candidate filtering +MUST exclude Back/navigation buttons when searching for tab intents. + +PRODUCTION BUG 2026-04-30: +VLM returned Back button for "tap profile tab" → account switch failed. +""" + +from GramAddict.core.perception.spatial_parser import SpatialParser, SpatialNode + + +def _flatten_tree(node: SpatialNode): + """Recursively flatten a SpatialNode tree into a list.""" + result = [node] + for child in node.children: + result.extend(_flatten_tree(child)) + return result + + +def _parse_all_nodes(xml_str): + """Parse XML into flat list of SpatialNodes.""" + parser = SpatialParser() + root = parser.parse(xml_str) + if root is None: + return [] + return _flatten_tree(root) + + +OTHER_PROFILE_XML_MINIMAL = """ + + + + + + + + + + + + + + +""" + + +def test_back_button_excluded_for_tab_intents(): + """ + RED: When resolving "tap profile tab", the _visual_discovery pre-filter + must exclude any node with resource-id containing "back" or + content-desc == "Back". + + This is the structural guard that would have prevented the bug. + """ + nodes = _parse_all_nodes(OTHER_PROFILE_XML_MINIMAL) + + intent = "tap profile tab" + intent_lower = intent.lower() + + # Replicate intent_resolver.py:252-260 pre-filter + candidates = [ + n for n in nodes + if 200 < n.area < 400000 + and "com.android.systemui" not in (n.resource_id or "") + ] + + # Verify the Back button IS in candidates (precondition) + back_present = any( + "back" in (n.resource_id or "").lower() + or (n.content_desc or "").lower() == "back" + for n in candidates + ) + assert back_present, ( + "Precondition failed: Back button not in candidates. " + "Cannot test the filtering guard." + ) + + # NOW apply the guard that MUST exist in production code. + # Import and test the actual production filter. + from GramAddict.core.perception.intent_resolver import IntentResolver + + resolver = IntentResolver() + filtered = resolver.filter_navigation_conflicts(candidates, intent) + + # Assert Back button is EXCLUDED after guard + back_in_result = [ + n for n in filtered + if "back" in (n.resource_id or "").lower() + or (n.content_desc or "").lower() == "back" + ] + assert len(back_in_result) == 0, ( + f"Back button survived filtering for tab intent! " + f"Nodes: {[(n.resource_id, n.content_desc) for n in back_in_result]}" + ) + + # Assert profile tab IS still in the filtered results + profile_tabs = [ + n for n in filtered + if "profile_tab" in (n.resource_id or "") + or "profile" in (n.content_desc or "").lower() + ] + assert len(profile_tabs) > 0, ( + "Profile tab was also filtered out! Over-filtering." + ) + + +def test_back_button_kept_for_non_tab_intents(): + """ + The Back button guard must ONLY apply for tab intents. + For "press back" or "tap back button", Back must be kept. + """ + nodes = _parse_all_nodes(OTHER_PROFILE_XML_MINIMAL) + candidates = [ + n for n in nodes + if 200 < n.area < 400000 + and "com.android.systemui" not in (n.resource_id or "") + ] + + from GramAddict.core.perception.intent_resolver import IntentResolver + + resolver = IntentResolver() + + # For "tap back button", the Back button MUST survive + filtered = resolver.filter_navigation_conflicts(candidates, "tap back button") + back_nodes = [ + n for n in filtered + if "back" in (n.resource_id or "").lower() + or (n.content_desc or "").lower() == "back" + ] + assert len(back_nodes) > 0, ( + "Back button was filtered out for 'tap back button' intent! " + "The guard is too aggressive." + )