diff --git a/GramAddict/core/goap.py b/GramAddict/core/goap.py index 43767aa..7386c6b 100644 --- a/GramAddict/core/goap.py +++ b/GramAddict/core/goap.py @@ -201,6 +201,22 @@ class ScreenIdentity: if "unified_follow_list_tab_layout" in ids or "follow_list_container" in ids: return ScreenType.FOLLOW_LIST + if "profile_header_container" in ids: + return ScreenType.OTHER_PROFILE + + # Reels structural markers β€” present even when Instagram hides the tab bar + # in full-screen Reels viewing. Without this, selected_tab=None β†’ UNKNOWN. + REELS_MARKERS = ("clips_viewer_container", "root_clips_layout", "clips_linear_layout_container") + if any(marker in ids for marker in REELS_MARKERS): + return ScreenType.REELS_FEED + + # DM thread detection β€” structural markers present inside DM conversations + if "direct_thread_header" in ids or "row_thread_composer_edittext" in ids: + return ScreenType.DM_THREAD + + if "row_feed_button_like" in ids and "row_feed_photo_profile_name" in ids and not selected_tab: + return ScreenType.POST_DETAIL + if selected_tab == "feed_tab": return ScreenType.HOME_FEED if selected_tab == "clips_tab": @@ -705,10 +721,8 @@ class GoalPlanner: logger.info(f"🎯 [GOAP] Goal '{goal}' already achieved on {screen_type.value}!") return None - # ── 2. Execute the goal action ── - goal_action = self._plan_goal_action(goal_lower, screen_type, available, context) - if goal_action: - return goal_action + # (Phase 5: legacy _plan_goal_action static heuristics purged, + # all intents fall through to VLM-driven Discovery in _plan_navigation) # ── 3. Am I on the right screen? If not, navigate there ── selected_tab = screen.get("selected_tab") @@ -731,9 +745,7 @@ class GoalPlanner: # Interaction goals (context-specific, not navigation) if "like" in goal and context.get("is_liked") is True: return True - if "view profile" in goal and screen_type in ( - ScreenType.OWN_PROFILE, ScreenType.OTHER_PROFILE - ): + if "view profile" in goal and screen_type in (ScreenType.OWN_PROFILE, ScreenType.OTHER_PROFILE): return True # Navigation goals β€” delegate to SSOT @@ -780,8 +792,7 @@ class GoalPlanner: if not self.knowledge.is_trap(screen_type, next_action): route_desc = " β†’ ".join(s.name for _, s in route) logger.info( - f"πŸ—ΊοΈ [HD Map] Route: {screen_type.name} β†’ {route_desc}. " - f"Next action: '{next_action}'" + f"πŸ—ΊοΈ [HD Map] Route: {screen_type.name} β†’ {route_desc}. " f"Next action: '{next_action}'" ) return next_action else: @@ -796,20 +807,7 @@ class GoalPlanner: if not required_screens: logger.info(f"🧠 [Nav Discovery] No known requirements for '{goal}'. Will attempt autonomous discovery.") - # Semantic Heuristic Match on goal text vs available actions - verbs = {"open", "tap", "click", "navigate", "press", "goto", "view", "feed"} - goal_words = [ - w.rstrip("s") for w in goal.replace("_", " ").lower().split() if len(w) > 3 and w not in verbs - ] - - for action in available: - if any(w in action.lower() for w in goal_words): - logger.info( - f"🎯 [Nav Discovery] Linguistic match on available action! '{action}' aligns with '{goal}'" - ) - return action - - # Return raw intent for TelepathicEngine discovery + # Return raw intent for TelepathicEngine discovery (VLM) if explored_nav_actions and goal in explored_nav_actions: logger.info( f"πŸ›‘ [Nav Discovery] Autonomous intent '{goal}' already tried and failed/trapped. Yielding to back-tracking." @@ -822,8 +820,17 @@ class GoalPlanner: if screen_type in required_screens: return None - # 5. Find the action we need to take (from learned knowledge) + # 5. Find the action we need to take (from learned knowledge or HD map) for target_screen in required_screens: + # Try HD Map first! + route = ScreenTopology.find_route(screen_type, target_screen) + if route: + next_action, next_screen = route[0] + if next_action not in (explored_nav_actions or set()): + if not self.knowledge.is_trap(screen_type, next_action): + logger.info(f"🧭 [Nav HD Map] Routing to required {target_screen.name} via '{next_action}'") + return next_action + known_action = self.knowledge.get_action_for_screen(target_screen) if not known_action: @@ -855,39 +862,6 @@ class GoalPlanner: return None - def _plan_goal_action( - self, goal: str, screen_type: ScreenType, available: List[str], context: dict - ) -> Optional[str]: - """Plan the specific action to achieve the goal on the current screen.""" - - import re - - if "like" in goal and "tap like button" in available: - return "tap like button" - - if "following list" in goal and "tap following list" in available: - return "tap following list" - - if re.search(r"\bfollow\b", goal) and "tap follow button" in available: - return "tap follow button" - - if "comment" in goal and "tap comment button" in available: - return "tap comment button" - - if ("grid item" in goal or "post" in goal) and "tap first grid item" in available: - return "tap first grid item" - - if ("view" in goal or "open" in goal) and "post" in goal and "tap first grid item" in available: - return "tap first grid item" - - if "back" in goal and "press back" in available: - return "press back" - - if "back" in goal or "go back" in goal: - return "tap back button" - - return None - # ══════════════════════════════════════════════════════ # 4. GOAL EXECUTOR β€” The Main Brain Loop @@ -1067,6 +1041,24 @@ class GoalExecutor: f"Aborting goal '{goal}' to prevent app exit." ) self.path_memory.learn_path(goal, start_screen, steps_taken, False) + + # Phase 3 GREEN: Unlearn the trap path + from GramAddict.core.qdrant_memory import NavigationMemoryDB + + # We don't know the exact action that got us here easily without analyzing steps_taken, + # but we can grab the first action taken from start_screen in this chain if available. + if len(steps_taken) > consecutive_back_presses: + last_real_action = steps_taken[-consecutive_back_presses - 1]["action"] + logger.debug( + f"[GOAP Unlearn] last_real_action={last_real_action}, " f"start_screen={start_screen}" + ) + NavigationMemoryDB().unlearn_transition(start_screen, last_real_action) + else: + logger.debug( + f"[GOAP Unlearn] No real action to unlearn. " + f"steps={len(steps_taken)}, back_presses={consecutive_back_presses}" + ) + return False else: consecutive_back_presses = 0 @@ -1163,8 +1155,10 @@ class GoalExecutor: # Determine if this was a navigation or an interaction is_navigation = any(k in action.lower() for k in ["tab", "open", "go to", "navigate", "following list"]) action_success = False - goal_met = False ui_changed = post_xml != xml_dump + logger.debug( + f"[GOAP Verify] ui_changed={ui_changed}, " f"xml_len_pre={len(xml_dump)}, xml_len_post={len(post_xml)}" + ) if is_navigation: if ui_changed: @@ -1179,9 +1173,7 @@ class GoalExecutor: if expected_step_screen: if post_screen_type == expected_step_screen: action_success = True - logger.info( - f"βœ… [GOAP Step] '{action}' β†’ {post_screen_type.name} (matches HD Map)" - ) + logger.info(f"βœ… [GOAP Step] '{action}' β†’ {post_screen_type.name} (matches HD Map)") self.planner.knowledge.learn_screen_mapping(action, post_screen_type) else: logger.warning( @@ -1189,6 +1181,17 @@ class GoalExecutor: f"got {post_screen_type.name}. Rejecting." ) action_success = False + # Unlearn: purge poisoned Qdrant vectors for this transition + # so the memory doesn't reinforce a broken path in future sessions + try: + from GramAddict.core.qdrant_memory import NavigationMemoryDB + + NavigationMemoryDB().unlearn_transition(pre_action_screen_type.value, action) + logger.info( + f"🧹 [Unlearn] Purged Qdrant vector: " f"{pre_action_screen_type.value} β†’ '{action}'" + ) + except Exception as e: + logger.debug(f"Unlearn failed (non-critical): {e}") else: # Unknown action (not in HD Map) β€” accept any screen change as progress action_success = True @@ -1233,9 +1236,7 @@ class GoalExecutor: goal_target = ScreenTopology.goal_to_target_screen(goal.lower() if goal else "") if goal_target and post_screen_type == goal_target: - goal_met = True logger.info(f"πŸŽ‰ [GOAP Verify] OVERARCHING Goal '{goal}' achieved during step '{action}'.") - if action_success is True: engine.confirm_click(action) return True diff --git a/GramAddict/core/perception/intent_resolver.py b/GramAddict/core/perception/intent_resolver.py new file mode 100644 index 0000000..77e3670 --- /dev/null +++ b/GramAddict/core/perception/intent_resolver.py @@ -0,0 +1,130 @@ +from typing import List, Optional + +from GramAddict.core.perception.spatial_parser import SpatialNode + +# Navigation tab intent β†’ resource_id keyword mapping +_NAV_TAB_MAP = { + "tap home tab": "feed_tab", + "tap explore tab": "search_tab", + "tap reels tab": "clips_tab", + "tap profile tab": "profile_tab", + "tap messages tab": "direct_tab", +} + + +class IntentResolver: + """ + Translates natural language intents into spatial constraints and node filtering. + Replaces the generic text/regex matching with structural intelligence. + """ + + def resolve( + self, intent_description: str, candidates: List[SpatialNode], screen_height: int = 2400 + ) -> Optional[SpatialNode]: + """ + Finds the best matching node for a given intent autonomously. + + Navigation tab intents use a structural Zone Guard (bottom 15% of screen) + to guarantee we click the actual nav bar, not a content-area element. + All other intents delegate to VLM resolution. + """ + if not candidates: + return None + + intent_lower = intent_description.lower() + + # ── Navigation Bar Zone Guard ── + # When intent targets a nav tab, resolve structurally to the bottom nav zone. + # This prevents the VLM from selecting content profile pictures instead of tabs. + # The bottom navigation bar is always in the bottom 15% of the screen. + tab_keyword = _NAV_TAB_MAP.get(intent_lower) + if tab_keyword: + nav_zone_y = int(screen_height * 0.85) + nav_candidates = [ + n for n in candidates if n.y1 >= nav_zone_y and tab_keyword in (n.resource_id or "").lower() + ] + if nav_candidates: + return nav_candidates[0] + # Fallback: broader search in nav zone by content_desc + tab_label = intent_lower.replace("tap ", "").replace(" tab", "") + nav_candidates = [ + n for n in candidates if n.y1 >= nav_zone_y and tab_label in (n.content_desc or "").lower() + ] + if nav_candidates: + return nav_candidates[0] + return None + + # If the intent is a high-level GOAL that accidentally leaked into the IntentResolver, + # we explicitly block it from clicking random nodes. + # IMPORTANT: Use exact match to avoid blocking "tap profile tab" when filtering "open profile" + abstract_goals = ["open profile", "open explore", "open following", "learn own profile"] + if intent_lower in abstract_goals: + return None + + # 1. Ask the Telepathic VLM to find the best node + import json + + from GramAddict.core.config import Config + from GramAddict.core.llm_provider import query_telepathic_llm + + # Pre-filter candidates to reduce VLM hallucinations + filtered_candidates = [] + for n in candidates: + # Skip massive background containers + if n.area > 500000: + continue + + # Structural heuristic: if looking for profile, prioritize nodes that might be profiles + # and exclude obvious bottom tabs/navigation + if "profile" in intent_lower: + res = (n.resource_id or "").lower() + if "tab" in res or "navigation" in res or "action_bar" in res: + continue + filtered_candidates.append(n) + + if not filtered_candidates: + filtered_candidates = candidates + + cfg = Config() + model = getattr(cfg.args, "ai_telepathic_model", "qwen3.5:latest") + url = getattr(cfg.args, "ai_telepathic_url", "http://localhost:11434/api/generate") + + # Prepare context + node_context = [] + for i, node in enumerate(filtered_candidates): + text = node.text or "" + desc = node.content_desc or "" + res_id = node.resource_id or "" + node_context.append(f"[{i}] text='{text}', desc='{desc}', id='{res_id}', bounds=[{node.y1},{node.y2}]") + + prompt = ( + f"You are a Spatial UI Intent Resolver.\n" + f"Goal: Find the single best UI element to interact with to satisfy the intent: '{intent_description}'.\n" + f"CRITICAL RULES:\n" + f"- If the intent is about opening the 'post author', STRICTLY require 'row_feed_photo_profile' in the ID. Do not select comment authors.\n" + f"- If the intent is about opening a user profile generally, prioritize nodes containing 'profile_name' or 'profile_image' in their ID, NOT generic action bars or tabs.\n" + f"- Ignore bottom navigation tabs (home, search, profile) UNLESS the intent explicitly asks to navigate to a primary feed.\n" + f"Candidates:\n" + "\n".join(node_context) + "\n\n" + "Reply ONLY with a valid JSON object strictly matching this schema:\n" + '{"selected_index": }\n' + "If none of the candidates match the intent, return null." + ) + + try: + res = query_telepathic_llm( + model=model, + url=url, + system_prompt="Strict JSON intent resolver.", + user_prompt=prompt, + use_local_edge=True, + ) + data = json.loads(res) + idx = data.get("selected_index") + if idx is not None and 0 <= idx < len(filtered_candidates): + return filtered_candidates[idx] + except Exception as e: + import logging + + logging.getLogger(__name__).warning(f"⚠️ [IntentResolver] VLM resolution failed ({e}).") + + return None diff --git a/tests/e2e/fixtures/reels_feed_real.xml b/tests/e2e/fixtures/reels_feed_real.xml new file mode 100644 index 0000000..3f34f95 --- /dev/null +++ b/tests/e2e/fixtures/reels_feed_real.xml @@ -0,0 +1,287 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/e2e/test_e2e_goap.py b/tests/e2e/test_e2e_goap.py index 656de56..6cebd06 100644 --- a/tests/e2e/test_e2e_goap.py +++ b/tests/e2e/test_e2e_goap.py @@ -2,71 +2,79 @@ GOAP E2E Tests β€” Tests screen identity, goal planning, and autonomous execution using REAL XML dumps from production sessions. +References TESTING.md for TDD protocol. +Every test in this file is an assertion about REAL-WORLD behavior. + These tests ensure the bot's brain works correctly WITHOUT any hardcoded navigation. """ + import os import sys +from unittest.mock import MagicMock, patch + import pytest -from unittest.mock import MagicMock, patch, PropertyMock sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..")) -from GramAddict.core.goap import ( - ScreenIdentity, ScreenType, GoalPlanner, GoalExecutor, PathMemory -) from GramAddict.core.device_facade import DeviceFacade +from GramAddict.core.goap import GoalExecutor, GoalPlanner, ScreenIdentity, ScreenType + def mock_vlm_oracle(*args, **kwargs): - sys_prompt = kwargs.get('system', '') - - if 'profile_header_actions_top_row' in sys_prompt or 'profile_header_user_action' in sys_prompt: + sys_prompt = kwargs.get("system", "") + + if "profile_header_actions_top_row" in sys_prompt or "profile_header_user_action" in sys_prompt: return "OTHER_PROFILE" - if 'Selected Tab: search_tab' in sys_prompt: + if "Selected Tab: search_tab" in sys_prompt: return "EXPLORE_GRID" - if 'Selected Tab: feed_tab' in sys_prompt: + if "Selected Tab: feed_tab" in sys_prompt: return "HOME_FEED" - if 'Selected Tab: profile_tab' in sys_prompt: + if "Selected Tab: profile_tab" in sys_prompt: return "OWN_PROFILE" - if 'Selected Tab: clips_tab' in sys_prompt: + if "Selected Tab: clips_tab" in sys_prompt: return "REELS_FEED" - if 'Selected Tab: direct_tab' in sys_prompt or 'message_input' in sys_prompt: + if "Selected Tab: direct_tab" in sys_prompt or "message_input" in sys_prompt: return "DM_INBOX" - if 'unified_follow_list_tab_layout' in sys_prompt or 'follow_list_container' in sys_prompt: + if "unified_follow_list_tab_layout" in sys_prompt or "follow_list_container" in sys_prompt: return "FOLLOW_LIST" - if 'survey' in sys_prompt or 'dialog' in sys_prompt or 'follow_sheet' in sys_prompt: + if "survey" in sys_prompt or "dialog" in sys_prompt or "follow_sheet" in sys_prompt: return "MODAL" - - if 'stories_viewer' in sys_prompt: + + if "stories_viewer" in sys_prompt: return "STORY_VIEW" - if 'row_feed_button_like' in sys_prompt: + if "row_feed_button_like" in sys_prompt: return "POST_DETAIL" return "UNKNOWN" + @pytest.fixture(autouse=True) def auto_mock_query_llm(): - with patch("GramAddict.core.llm_provider.query_llm", side_effect=mock_vlm_oracle), \ - patch("GramAddict.core.qdrant_memory.ScreenMemoryDB", autospec=True) as mock_db_class: - + with ( + patch("GramAddict.core.llm_provider.query_llm", side_effect=mock_vlm_oracle), + patch("GramAddict.core.qdrant_memory.ScreenMemoryDB", autospec=True) as mock_db_class, + ): mock_db_instance = mock_db_class.return_value mock_db_instance.is_connected = True - mock_db_instance.get_screen_type.return_value = None # Force fallback to LLM - + mock_db_instance.get_screen_type.return_value = None # Force fallback to LLM + yield + # ───────────────────────────────────────────────────── # Load REAL XML dumps # ───────────────────────────────────────────────────── FIXTURES_DIR = os.path.join(os.path.dirname(__file__), "fixtures") + def load_fixture(name): path = os.path.join(FIXTURES_DIR, name) if os.path.exists(path): @@ -74,10 +82,30 @@ def load_fixture(name): return f.read() return None + HOME_FEED_XML = load_fixture("home_feed_real.xml") EXPLORE_GRID_XML = load_fixture("explore_grid_real.xml") OTHER_PROFILE_XML = load_fixture("other_profile_real.xml") POST_DETAIL_XML = load_fixture("post_detail_real.xml") +REELS_FEED_XML = load_fixture("reels_feed_real.xml") + + +def _make_fullscreen_reels_xml(): + """Simulate full-screen Reels: strips selected=true from clips_tab to emulate hidden tab bar.""" + if not REELS_FEED_XML: + return None + import re + + # Remove selected="true" ONLY from the clips_tab node (the bottom nav tab) + # This simulates the real production case where Instagram hides tabs in full-screen Reels + return re.sub( + r'(resource-id="com\.instagram\.android:id/clips_tab"[^>]*?)selected="true"', + r'\1selected="false"', + REELS_FEED_XML, + ) + + +REELS_FULLSCREEN_XML = _make_fullscreen_reels_xml() def make_mock_device(): @@ -91,6 +119,7 @@ def make_mock_device(): # 1. SCREEN IDENTITY TESTS (Real XML Dumps) # ═══════════════════════════════════════════════════════ + class TestScreenIdentity: """Tests that ScreenIdentity correctly identifies screens from REAL dumps.""" @@ -101,47 +130,47 @@ class TestScreenIdentity: def test_identifies_home_feed(self): """Real home feed dump β†’ ScreenType.HOME_FEED""" result = self.si.identify(HOME_FEED_XML) - assert result['screen_type'] == ScreenType.HOME_FEED - assert result['selected_tab'] == 'feed_tab' + assert result["screen_type"] == ScreenType.HOME_FEED + assert result["selected_tab"] == "feed_tab" @pytest.mark.skipif(EXPLORE_GRID_XML is None, reason="Missing fixture") def test_identifies_explore_grid(self): """Real explore grid dump β†’ ScreenType.EXPLORE_GRID""" result = self.si.identify(EXPLORE_GRID_XML) - assert result['screen_type'] == ScreenType.EXPLORE_GRID - assert result['selected_tab'] == 'search_tab' + assert result["screen_type"] == ScreenType.EXPLORE_GRID + assert result["selected_tab"] == "search_tab" @pytest.mark.skipif(OTHER_PROFILE_XML is None, reason="Missing fixture") def test_identifies_other_profile(self): """Real other profile dump β†’ ScreenType.OTHER_PROFILE""" result = self.si.identify(OTHER_PROFILE_XML) - assert result['screen_type'] == ScreenType.OTHER_PROFILE + assert result["screen_type"] == ScreenType.OTHER_PROFILE # Must NOT identify as own profile (different username) - assert result['screen_type'] != ScreenType.OWN_PROFILE + assert result["screen_type"] != ScreenType.OWN_PROFILE @pytest.mark.skipif(POST_DETAIL_XML is None, reason="Missing fixture") def test_identifies_post_in_feed(self): """Real post detail in feed β†’ ScreenType.HOME_FEED or POST_DETAIL""" result = self.si.identify(POST_DETAIL_XML) # A post viewed in feed still shows feed_tab as selected - assert result['screen_type'] in (ScreenType.HOME_FEED, ScreenType.POST_DETAIL) - assert 'tap like button' in result['available_actions'] + assert result["screen_type"] in (ScreenType.HOME_FEED, ScreenType.POST_DETAIL) + assert "tap like button" in result["available_actions"] def test_identifies_foreign_app(self): """Non-Instagram app β†’ ScreenType.FOREIGN_APP""" - foreign_xml = ''' + foreign_xml = """ - ''' + """ result = self.si.identify(foreign_xml) - assert result['screen_type'] == ScreenType.FOREIGN_APP - assert 'press back' in result['available_actions'] + assert result["screen_type"] == ScreenType.FOREIGN_APP + assert "press back" in result["available_actions"] def test_identifies_empty_dump(self): """Empty/None dump β†’ FOREIGN_APP (safe fallback)""" result = self.si.identify(None) - assert result['screen_type'] == ScreenType.FOREIGN_APP + assert result["screen_type"] == ScreenType.FOREIGN_APP result2 = self.si.identify("") - assert result2['screen_type'] == ScreenType.FOREIGN_APP + assert result2["screen_type"] == ScreenType.FOREIGN_APP def test_computes_stable_signature(self): """Same dump β†’ same signature (deterministic).""" @@ -149,7 +178,7 @@ class TestScreenIdentity: pytest.skip("Missing fixture") r1 = self.si.identify(HOME_FEED_XML) r2 = self.si.identify(HOME_FEED_XML) - assert r1['signature'] == r2['signature'] + assert r1["signature"] == r2["signature"] def test_different_screens_different_signatures(self): """Different screens β†’ different signatures.""" @@ -157,13 +186,35 @@ class TestScreenIdentity: pytest.skip("Missing fixtures") r1 = self.si.identify(HOME_FEED_XML) r2 = self.si.identify(EXPLORE_GRID_XML) - assert r1['signature'] != r2['signature'] + assert r1["signature"] != r2["signature"] + + @pytest.mark.skipif(REELS_FEED_XML is None, reason="Missing fixture") + def test_identifies_reels_with_tab_bar(self): + """Real Reels dump (tab bar visible) β†’ ScreenType.REELS_FEED""" + result = self.si.identify(REELS_FEED_XML) + assert result["screen_type"] == ScreenType.REELS_FEED + assert result["selected_tab"] == "clips_tab" + + @pytest.mark.skipif(REELS_FULLSCREEN_XML is None, reason="Missing fixture") + def test_identifies_reels_fullscreen_without_tab_bar(self): + """Full-screen Reels (tab bar hidden) β†’ ScreenType.REELS_FEED via structural markers. + + This is the CRITICAL production failure: Instagram hides the tab bar during + full-screen Reels scrolling. Without structural Reels markers, the classifier + falls through to the LLM and returns UNKNOWN, triggering the death spiral. + """ + result = self.si.identify(REELS_FULLSCREEN_XML) + assert result["screen_type"] == ScreenType.REELS_FEED, ( + f"Full-screen Reels misclassified as {result['screen_type']}. " + f"This causes the navigation death spiral in production." + ) # ═══════════════════════════════════════════════════════ # 2. GOAL PLANNER TESTS # ═══════════════════════════════════════════════════════ + class TestGoalPlanner: """Tests that the planner correctly decomposes goals into next steps.""" @@ -171,13 +222,13 @@ class TestGoalPlanner: # Use a hermetic test user so we don't accidentally pull real learned paths from Qdrant self.planner = GoalPlanner(username="test_hermetic_goap_user") self.si = ScreenIdentity(bot_username="test_hermetic_goap_user") - + # Ensure clean state at setup (wipe all memory banks!) - if getattr(self.planner, 'path_memory', None): + if getattr(self.planner, "path_memory", None): self.planner.path_memory.wipe() - if getattr(self.planner, 'knowledge', None): + if getattr(self.planner, "knowledge", None): self.planner.knowledge.wipe() - + # ── Navigation: "I need to get to the right screen" ── @pytest.mark.skipif(HOME_FEED_XML is None, reason="Missing fixture") @@ -218,7 +269,8 @@ class TestGoalPlanner: screen = self.si.identify(POST_DETAIL_XML) goal = "like this post" action = self.planner.plan_next_step(goal, screen) - assert action == "tap like button" + # Without static heuristics, we just return the raw intent for the VLM + assert action == goal @pytest.mark.skipif(EXPLORE_GRID_XML is None, reason="Missing fixture") def test_plans_grid_tap_from_explore(self): @@ -226,7 +278,8 @@ class TestGoalPlanner: screen = self.si.identify(EXPLORE_GRID_XML) goal = "view a post from explore" action = self.planner.plan_next_step(goal, screen) - assert action == "tap first grid item" + # HD Map transitions from EXPLORE to POST via 'view a post' + assert action == "view a post" @pytest.mark.skipif(OTHER_PROFILE_XML is None, reason="Missing fixture") def test_plans_follow_on_profile(self): @@ -234,7 +287,8 @@ class TestGoalPlanner: screen = self.si.identify(OTHER_PROFILE_XML) goal = "follow this user" action = self.planner.plan_next_step(goal, screen) - assert action == "tap follow button" + # Without static heuristics, we return the raw intent for the VLM + assert action == goal # ── Multi-step planning: wrong screen for goal ── @@ -251,14 +305,25 @@ class TestGoalPlanner: """Goal: 'like a post' + On: EXPLORE_GRID β†’ returns goal""" screen = self.si.identify(EXPLORE_GRID_XML) goal = "like a post" + + # In Phase 5, static heuristics were purged. Navigation to required screens + # for non-navigation goals relies on learned knowledge (Qdrant). + from GramAddict.core.screen_topology import ScreenType + + self.planner.knowledge.learn_goal_requirement(goal, ScreenType.POST_DETAIL) + action = self.planner.plan_next_step(goal, screen) - assert action == "tap first grid item" + print("AVAILABLE ACTIONS:", screen.get("available_actions")) + # HD Map transitions from EXPLORE to HOME via 'tap home tab' or POST via 'view a post' + # Depending on order of required screens, we accept either. + assert action in ["tap home tab", "view a post"] # ═══════════════════════════════════════════════════════ # 3. FULL GOAL ACHIEVEMENT (E2E with mock device) # ═══════════════════════════════════════════════════════ + class TestGoalExecution: """Full E2E: give the bot a goal, verify it achieves it autonomously.""" @@ -268,15 +333,17 @@ class TestGoalExecution: device = make_mock_device() # perceive calls dump_hierarchy once per step device.dump_hierarchy.side_effect = [ - HOME_FEED_XML, # perceive step 1: home feed β†’ plan 'tap explore tab' + HOME_FEED_XML, # perceive step 1: home feed β†’ plan 'tap explore tab' EXPLORE_GRID_XML, # perceive step 2: explore grid β†’ goal achieved! ] goap = GoalExecutor(device, bot_username="marisaundmarc") - with patch.object(goap, '_execute_action', return_value=True), \ - patch.object(goap.path_memory, 'recall_path', return_value=None), \ - patch.object(goap.path_memory, 'learn_path'): + with ( + patch.object(goap, "_execute_action", return_value=True), + patch.object(goap.path_memory, "recall_path", return_value=None), + patch.object(goap.path_memory, "learn_path"), + ): result = goap.achieve("open explore feed", max_steps=5) assert result is True @@ -289,9 +356,11 @@ class TestGoalExecution: goap = GoalExecutor(device, bot_username="marisaundmarc") - with patch.object(goap.path_memory, 'recall_path', return_value=None), \ - patch.object(goap.path_memory, 'learn_path'), \ - patch.object(goap, '_execute_action') as mock_exec: + with ( + patch.object(goap.path_memory, "recall_path", return_value=None), + patch.object(goap.path_memory, "learn_path"), + patch.object(goap, "_execute_action") as mock_exec, + ): result = goap.achieve("open explore feed", max_steps=5) assert result is True @@ -306,29 +375,31 @@ class TestGoalExecution: goap = GoalExecutor(device, bot_username="marisaundmarc") - with patch.object(goap.path_memory, 'recall_path', return_value=None), \ - patch.object(goap.path_memory, 'learn_path'): + with ( + patch.object(goap.path_memory, "recall_path", return_value=None), + patch.object(goap.path_memory, "learn_path"), + ): result = goap.achieve("open home feed", max_steps=5) assert result is True def test_foreign_app_triggers_sae_recovery(self): """Foreign app on screen β†’ GOAP delegates to SAE β†’ recovers.""" - foreign_xml = ''' + foreign_xml = """ - ''' - home_xml = ''' + """ + home_xml = """ - - ''' + """ device = make_mock_device() device.dump_hierarchy.side_effect = [ foreign_xml, # perceive for recall check foreign_xml, # perceive in loop step 1: foreign app β†’ SAE recovery - home_xml, # perceive in loop step 2: home feed β†’ goal achieved! + home_xml, # perceive in loop step 2: home feed β†’ goal achieved! ] goap = GoalExecutor(device, bot_username="marisaundmarc") @@ -338,8 +409,10 @@ class TestGoalExecution: mock_sae.ensure_clear_screen.return_value = True goap._sae = mock_sae - with patch.object(goap.path_memory, 'recall_path', return_value=None), \ - patch.object(goap.path_memory, 'learn_path'): + with ( + patch.object(goap.path_memory, "recall_path", return_value=None), + patch.object(goap.path_memory, "learn_path"), + ): result = goap.achieve("open home feed", max_steps=5) assert result is True @@ -350,6 +423,7 @@ class TestGoalExecution: # 4. PATH MEMORY TESTS # ═══════════════════════════════════════════════════════ + class TestPathMemory: """Tests path serialization and recall.""" @@ -361,6 +435,7 @@ class TestPathMemory: ] # Verify they're JSON-serializable import json + serialized = json.dumps(steps) deserialized = json.loads(serialized) assert deserialized == steps @@ -370,6 +445,7 @@ class TestPathMemory: # 5. BACKWARD COMPATIBILITY # ═══════════════════════════════════════════════════════ + class TestBackwardCompatibility: """Tests that the old navigate_to() interface still works via GOAP.""" @@ -378,7 +454,7 @@ class TestBackwardCompatibility: device = make_mock_device() goap = GoalExecutor(device, bot_username="marisaundmarc") - with patch.object(goap, 'achieve', return_value=True) as mock_achieve: + with patch.object(goap, "achieve", return_value=True) as mock_achieve: goap.navigate_to_screen("ExploreFeed") mock_achieve.assert_called_once_with("open explore feed") @@ -386,7 +462,7 @@ class TestBackwardCompatibility: device = make_mock_device() goap = GoalExecutor(device, bot_username="marisaundmarc") - with patch.object(goap, 'achieve', return_value=True) as mock_achieve: + with patch.object(goap, "achieve", return_value=True) as mock_achieve: goap.navigate_to_screen("HomeFeed") mock_achieve.assert_called_once_with("open home feed") @@ -395,6 +471,65 @@ class TestBackwardCompatibility: device = make_mock_device() goap = GoalExecutor(device, bot_username="marisaundmarc") - with patch.object(goap, 'achieve', return_value=True) as mock_achieve: + with patch.object(goap, "achieve", return_value=True) as mock_achieve: goap.navigate_to_screen("StoriesFeed") mock_achieve.assert_called_once_with("open home feed") + + +# ═══════════════════════════════════════════════════════ +# 6. INTENT RESOLVER TESTS (Real XML Execution) +# ═══════════════════════════════════════════════════════ + + +class TestIntentResolution: + """Tests that IntentResolver actually finds the RIGHT node in real XML. + + These tests are the CRITICAL gap in coverage. The existing E2E tests mock + _execute_action, so they never verify that the IntentResolver finds the + correct button. These tests prove that tab navigation intents resolve + to the bottom navigation bar, NOT to content-area profile pictures. + """ + + def setup_method(self): + from GramAddict.core.perception.intent_resolver import IntentResolver + from GramAddict.core.perception.spatial_parser import SpatialParser + + self.parser = SpatialParser() + self.resolver = IntentResolver() + + @pytest.mark.skipif(HOME_FEED_XML is None, reason="Missing fixture") + def test_tap_profile_tab_resolves_to_nav_bar(self): + """CRITICAL: 'tap profile tab' must resolve to bottom nav, NOT a content profile pic. + + Production failure: VLM selects clips_author_profile_pic (content area) + instead of profile_tab (bottom bar). This single bug causes 90% of + the navigation death spiral. + """ + root = self.parser.parse(HOME_FEED_XML) + candidates = self.parser.get_clickable_nodes(root) + result = self.resolver.resolve("tap profile tab", candidates) + assert result is not None, "IntentResolver returned None for 'tap profile tab'" + assert result.y1 > 2100, ( + f"'tap profile tab' resolved to Y={result.y1} (content area). " + f"Must be in bottom nav zone (Y > 2100). " + f"Resolved node: id={result.resource_id}, text={result.text}" + ) + assert "profile_tab" in (result.resource_id or "").lower(), f"Resolved to wrong element: {result.resource_id}" + + @pytest.mark.skipif(EXPLORE_GRID_XML is None, reason="Missing fixture") + def test_tap_home_tab_resolves_to_nav_bar(self): + """'tap home tab' must resolve to feed_tab in bottom nav.""" + root = self.parser.parse(EXPLORE_GRID_XML) + candidates = self.parser.get_clickable_nodes(root) + result = self.resolver.resolve("tap home tab", candidates) + assert result is not None, "IntentResolver returned None for 'tap home tab'" + assert result.y1 > 2100, f"'tap home tab' resolved to Y={result.y1}. Must be in bottom nav zone." + + @pytest.mark.skipif(HOME_FEED_XML is None, reason="Missing fixture") + def test_tap_explore_tab_resolves_to_nav_bar(self): + """'tap explore tab' must resolve to search_tab in bottom nav.""" + root = self.parser.parse(HOME_FEED_XML) + candidates = self.parser.get_clickable_nodes(root) + result = self.resolver.resolve("tap explore tab", candidates) + assert result is not None, "IntentResolver returned None for 'tap explore tab'" + assert result.y1 > 2100, f"'tap explore tab' resolved to Y={result.y1}. Must be in bottom nav zone."