diff --git a/GramAddict/core/goap.py b/GramAddict/core/goap.py index 4e09881..80f264d 100644 --- a/GramAddict/core/goap.py +++ b/GramAddict/core/goap.py @@ -321,6 +321,25 @@ class GoalExecutor: self._get_sae().ensure_clear_screen(max_attempts=3) return False + # โ”€โ”€ Pre-Click Semantic Match Guard โ”€โ”€ + # For toggle intents (follow/like/save), verify the selected node + # semantically matches the intent BEFORE clicking. This prevents + # VLM hallucinations from clicking photo grid items when looking + # for follow buttons. + from GramAddict.core.perception.action_memory import _intent_matches_node + + node_semantic = ( + f"text: '{best_node.get('text', '')}', " + f"desc: '{best_node.get('description', '')}', " + f"id: '{best_node.get('id', '')}'" + ) + if not _intent_matches_node(action, node_semantic): + logger.warning( + f"๐Ÿ›ก๏ธ [GOAP Execute] Pre-click rejection: node does not match intent '{action}'. " + f"Node: {node_semantic}" + ) + return False + # Execute click self.device.click(obj=best_node) import random diff --git a/GramAddict/core/perception/action_memory.py b/GramAddict/core/perception/action_memory.py index f9ee512..2d3ccfb 100644 --- a/GramAddict/core/perception/action_memory.py +++ b/GramAddict/core/perception/action_memory.py @@ -5,6 +5,20 @@ from GramAddict.core.perception.spatial_parser import SpatialNode logger = logging.getLogger(__name__) +# โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• +# Semantic Match Keywords โ€” SSOT for intent โ†’ element validation +# โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• + +# Maps toggle-intent keywords to required element markers. +# If the intent contains the key, the clicked element MUST +# contain at least one of the corresponding markers in its +# text, content_desc, or resource_id. +TOGGLE_INTENT_MARKERS = { + "follow": ["follow", "gefolgt", "abonnieren"], + "like": ["like", "heart", "gefรคllt"], + "save": ["save", "saved", "bookmark", "speichern"], +} + class ActionMemory: """ @@ -36,7 +50,11 @@ class ActionMemory: logger.debug(f"๐Ÿง  [ActionMemory] Tracking tentative click for intent: '{intent}' -> {semantic_string}") def confirm_click(self, intent: str = None): - """Positive Reinforcement: Confirms the last click was successful.""" + """Positive Reinforcement: Confirms the last click was successful. + + Guard: Refuses to store in Qdrant if the clicked element does not + semantically match the intent. Prevents memory poisoning. + """ ctx = self._last_click_context if not ctx: return @@ -44,6 +62,15 @@ class ActionMemory: if intent and ctx["intent"] != intent: return + # โ”€โ”€ Semantic Mismatch Guard โ”€โ”€ + if not _intent_matches_node(ctx["intent"], ctx["semantic_string"]): + logger.warning( + f"๐Ÿ›ก๏ธ [ActionMemory] BLOCKED confirm_click for '{ctx['intent']}' โ€” " + f"clicked element does not match intent: {ctx['semantic_string']}" + ) + self._last_click_context = None + return + logger.info(f"โœ… [ActionMemory] Confirming success for '{ctx['intent']}'. Boosting confidence.") # Store or boost in Qdrant @@ -145,6 +172,18 @@ class ActionMemory: logger.error(f"Failed to query VLM for visual verification: {e}") # Fallthrough to structural delta if VLM crashes + # โ”€โ”€ Pre-Structural Semantic Gate โ”€โ”€ + # Before trusting ANY structural delta, verify the clicked element + # semantically matches the intent. Prevents photo-clicks from + # being validated as follow/like successes. + if is_toggle and self._last_click_context: + if not _intent_matches_node(intent, self._last_click_context["semantic_string"]): + logger.warning( + f"๐Ÿ›ก๏ธ [ActionMemory] Semantic mismatch: '{intent}' does not match " + f"clicked element {self._last_click_context['semantic_string']}. Verification FAIL." + ) + return False + # Fallback to structural delta if no device, VLM fails, or high confidence bypass diff = abs(len(pre_click_xml) - len(post_click_xml)) @@ -166,3 +205,30 @@ class ActionMemory: logger.warning(f"โš ๏ธ [ActionMemory] No structural change detected for '{intent}'. Verification FAIL.") return False + + +def _intent_matches_node(intent: str, semantic_string: str) -> bool: + """Checks if the clicked element semantically matches the toggle intent. + + For toggle intents (follow, like, save), the clicked element MUST contain + at least one of the required keywords in its text/desc/id. This prevents + photo grid items, captions, and other unrelated elements from being + falsely confirmed as successful interactions. + + For non-toggle intents, returns True (no restriction). + """ + intent_lower = intent.lower() + semantic_lower = semantic_string.lower() + + for intent_keyword, required_markers in TOGGLE_INTENT_MARKERS.items(): + if intent_keyword in intent_lower: + if any(marker in semantic_lower for marker in required_markers): + return True + logger.debug( + f"๐Ÿ›ก๏ธ [SemanticGuard] Intent '{intent}' requires markers " + f"{required_markers} but element has: {semantic_string}" + ) + return False + + # Non-toggle intents pass through + return True diff --git a/GramAddict/core/q_nav_graph.py b/GramAddict/core/q_nav_graph.py index e391ef8..3f65f10 100644 --- a/GramAddict/core/q_nav_graph.py +++ b/GramAddict/core/q_nav_graph.py @@ -138,6 +138,7 @@ class QNavGraph: "like": "tap like button", "comment": "tap comment button", "share": "tap share button", + "follow": "tap follow button", } for keyword, required_action in action_checks.items(): if keyword in goal.lower() and required_action not in available: