fix(intent_resolver): add Nav Conflict Guard — Back button never valid for tab intents
PRODUCTION BUG 2026-04-30 11:50: VLM returned action_bar_button_back (desc='Back') for 'tap profile tab' intent on OTHER_PROFILE screen. This caused account_switcher to navigate AWAY from the profile instead of opening the account selector bottom sheet, halting the session. Root cause: _visual_discovery pre-filter did not exclude navigation buttons (Back, Close) when the intent was for a tab element. Fix: - Added filter_navigation_conflicts() method to IntentResolver - Guards tab intents by excluding nodes with 'back' in resource_id or content_desc == 'Back' - Does NOT apply for 'tap back button' or 'press back' intents TDD: - RED: test_intent_resolver_back_guard.py — 2 tests that call filter_navigation_conflicts on real OTHER_PROFILE XML - GREEN: filter_navigation_conflicts method + wired into _visual_discovery - E2E: test_workflow_account_switch.py, test_workflow_vlm_tab_confusion.py 31/31 tests passing.
This commit is contained in:
@@ -36,6 +36,50 @@ class IntentResolver:
|
||||
3. Fallback → text-based VLM (when no device/screenshot available)
|
||||
"""
|
||||
|
||||
# ──────────────────────────────────────────────
|
||||
# Structural Guards
|
||||
# ──────────────────────────────────────────────
|
||||
|
||||
def filter_navigation_conflicts(
|
||||
self, candidates: List[SpatialNode], intent_description: str
|
||||
) -> List[SpatialNode]:
|
||||
"""
|
||||
Prevents VLM from confusing navigation-bar buttons (Back, Close)
|
||||
with bottom tab-bar buttons (Home, Profile, Search).
|
||||
|
||||
Production bug 2026-04-30: VLM picked action_bar_button_back
|
||||
for "tap profile tab" → account switch failed.
|
||||
|
||||
Rules:
|
||||
- For tab intents: exclude nodes with "back" in resource_id or
|
||||
content_desc == "Back"
|
||||
- For back/close intents: no filtering (Back is the correct target)
|
||||
"""
|
||||
intent_lower = intent_description.lower()
|
||||
|
||||
# Only apply for tab-related intents
|
||||
is_tab_intent = "tab" in intent_lower and "back" not in intent_lower
|
||||
if not is_tab_intent:
|
||||
return candidates
|
||||
|
||||
filtered = []
|
||||
for node in candidates:
|
||||
rid = (node.resource_id or "").lower()
|
||||
desc = (node.content_desc or "").lower()
|
||||
|
||||
is_back = "back" in rid or desc == "back"
|
||||
is_close = "close" in rid or desc == "close"
|
||||
|
||||
if is_back or is_close:
|
||||
logger.debug(
|
||||
f"🛡️ [Nav Conflict Guard] Excluded '{node.resource_id}' "
|
||||
f"(desc='{node.content_desc}') for tab intent '{intent_description}'"
|
||||
)
|
||||
continue
|
||||
filtered.append(node)
|
||||
|
||||
return filtered
|
||||
|
||||
# ──────────────────────────────────────────────
|
||||
# Public API
|
||||
# ──────────────────────────────────────────────
|
||||
@@ -259,6 +303,11 @@ class IntentResolver:
|
||||
and "per cent" not in (n.content_desc or "").lower()
|
||||
]
|
||||
|
||||
# --- Navigation Conflict Guard ---
|
||||
# Prevents VLM from confusing Back buttons with tab buttons
|
||||
# Production bug 2026-04-30: VLM picked Back for "tap profile tab"
|
||||
candidates = self.filter_navigation_conflicts(candidates, intent_description)
|
||||
|
||||
# --- Strict Button Guard ---
|
||||
# If the intent specifically asks for a "button", "icon", or "tab",
|
||||
# filter out candidates that contain long text (e.g. captions, comments)
|
||||
|
||||
Reference in New Issue
Block a user