From 3b9465a3bc752452d85fc7f75293fbff273f1fc3 Mon Sep 17 00:00:00 2001 From: Marc Mintel Date: Mon, 27 Apr 2026 23:22:00 +0200 Subject: [PATCH] fix(GREEN): semantic match guard kills follow hallucination at 3 layers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements the _intent_matches_node() guard — a shared SSOT function that validates clicked elements against intent keywords before trusting any verification result. Fixes applied: 1. action_memory.py: verify_success() now cross-checks clicked element against intent BEFORE trusting structural delta for toggle actions 2. action_memory.py: confirm_click() blocks Qdrant poisoning when the tracked click doesn't semantically match the intent 3. q_nav_graph.py: 'follow' added to action_checks map (screen-sanity) 4. goap.py: Pre-click semantic guard prevents device.click() on elements that don't match toggle intents (follow/like/save) TOGGLE_INTENT_MARKERS dict is SSOT for intent→element validation keywords. Supports DE locale (gefolgt, abonnieren, gefällt, speichern). 162 passed, 0 regressions. All 5 previously-RED tests now GREEN. --- GramAddict/core/goap.py | 19 ++++++ GramAddict/core/perception/action_memory.py | 68 ++++++++++++++++++++- GramAddict/core/q_nav_graph.py | 1 + 3 files changed, 87 insertions(+), 1 deletion(-) diff --git a/GramAddict/core/goap.py b/GramAddict/core/goap.py index 4e09881..80f264d 100644 --- a/GramAddict/core/goap.py +++ b/GramAddict/core/goap.py @@ -321,6 +321,25 @@ class GoalExecutor: self._get_sae().ensure_clear_screen(max_attempts=3) return False + # ── Pre-Click Semantic Match Guard ── + # For toggle intents (follow/like/save), verify the selected node + # semantically matches the intent BEFORE clicking. This prevents + # VLM hallucinations from clicking photo grid items when looking + # for follow buttons. + from GramAddict.core.perception.action_memory import _intent_matches_node + + node_semantic = ( + f"text: '{best_node.get('text', '')}', " + f"desc: '{best_node.get('description', '')}', " + f"id: '{best_node.get('id', '')}'" + ) + if not _intent_matches_node(action, node_semantic): + logger.warning( + f"🛡️ [GOAP Execute] Pre-click rejection: node does not match intent '{action}'. " + f"Node: {node_semantic}" + ) + return False + # Execute click self.device.click(obj=best_node) import random diff --git a/GramAddict/core/perception/action_memory.py b/GramAddict/core/perception/action_memory.py index f9ee512..2d3ccfb 100644 --- a/GramAddict/core/perception/action_memory.py +++ b/GramAddict/core/perception/action_memory.py @@ -5,6 +5,20 @@ from GramAddict.core.perception.spatial_parser import SpatialNode logger = logging.getLogger(__name__) +# ═══════════════════════════════════════════════════════ +# Semantic Match Keywords — SSOT for intent → element validation +# ═══════════════════════════════════════════════════════ + +# Maps toggle-intent keywords to required element markers. +# If the intent contains the key, the clicked element MUST +# contain at least one of the corresponding markers in its +# text, content_desc, or resource_id. +TOGGLE_INTENT_MARKERS = { + "follow": ["follow", "gefolgt", "abonnieren"], + "like": ["like", "heart", "gefällt"], + "save": ["save", "saved", "bookmark", "speichern"], +} + class ActionMemory: """ @@ -36,7 +50,11 @@ class ActionMemory: logger.debug(f"🧠 [ActionMemory] Tracking tentative click for intent: '{intent}' -> {semantic_string}") def confirm_click(self, intent: str = None): - """Positive Reinforcement: Confirms the last click was successful.""" + """Positive Reinforcement: Confirms the last click was successful. + + Guard: Refuses to store in Qdrant if the clicked element does not + semantically match the intent. Prevents memory poisoning. + """ ctx = self._last_click_context if not ctx: return @@ -44,6 +62,15 @@ class ActionMemory: if intent and ctx["intent"] != intent: return + # ── Semantic Mismatch Guard ── + if not _intent_matches_node(ctx["intent"], ctx["semantic_string"]): + logger.warning( + f"🛡️ [ActionMemory] BLOCKED confirm_click for '{ctx['intent']}' — " + f"clicked element does not match intent: {ctx['semantic_string']}" + ) + self._last_click_context = None + return + logger.info(f"✅ [ActionMemory] Confirming success for '{ctx['intent']}'. Boosting confidence.") # Store or boost in Qdrant @@ -145,6 +172,18 @@ class ActionMemory: logger.error(f"Failed to query VLM for visual verification: {e}") # Fallthrough to structural delta if VLM crashes + # ── Pre-Structural Semantic Gate ── + # Before trusting ANY structural delta, verify the clicked element + # semantically matches the intent. Prevents photo-clicks from + # being validated as follow/like successes. + if is_toggle and self._last_click_context: + if not _intent_matches_node(intent, self._last_click_context["semantic_string"]): + logger.warning( + f"🛡️ [ActionMemory] Semantic mismatch: '{intent}' does not match " + f"clicked element {self._last_click_context['semantic_string']}. Verification FAIL." + ) + return False + # Fallback to structural delta if no device, VLM fails, or high confidence bypass diff = abs(len(pre_click_xml) - len(post_click_xml)) @@ -166,3 +205,30 @@ class ActionMemory: logger.warning(f"⚠️ [ActionMemory] No structural change detected for '{intent}'. Verification FAIL.") return False + + +def _intent_matches_node(intent: str, semantic_string: str) -> bool: + """Checks if the clicked element semantically matches the toggle intent. + + For toggle intents (follow, like, save), the clicked element MUST contain + at least one of the required keywords in its text/desc/id. This prevents + photo grid items, captions, and other unrelated elements from being + falsely confirmed as successful interactions. + + For non-toggle intents, returns True (no restriction). + """ + intent_lower = intent.lower() + semantic_lower = semantic_string.lower() + + for intent_keyword, required_markers in TOGGLE_INTENT_MARKERS.items(): + if intent_keyword in intent_lower: + if any(marker in semantic_lower for marker in required_markers): + return True + logger.debug( + f"🛡️ [SemanticGuard] Intent '{intent}' requires markers " + f"{required_markers} but element has: {semantic_string}" + ) + return False + + # Non-toggle intents pass through + return True diff --git a/GramAddict/core/q_nav_graph.py b/GramAddict/core/q_nav_graph.py index e391ef8..3f65f10 100644 --- a/GramAddict/core/q_nav_graph.py +++ b/GramAddict/core/q_nav_graph.py @@ -138,6 +138,7 @@ class QNavGraph: "like": "tap like button", "comment": "tap comment button", "share": "tap share button", + "follow": "tap follow button", } for keyword, required_action in action_checks.items(): if keyword in goal.lower() and required_action not in available: