fix(GREEN): semantic match guard kills follow hallucination at 3 layers

Implements the _intent_matches_node() guard — a shared SSOT function that
validates clicked elements against intent keywords before trusting any
verification result.

Fixes applied:
1. action_memory.py: verify_success() now cross-checks clicked element
   against intent BEFORE trusting structural delta for toggle actions
2. action_memory.py: confirm_click() blocks Qdrant poisoning when the
   tracked click doesn't semantically match the intent
3. q_nav_graph.py: 'follow' added to action_checks map (screen-sanity)
4. goap.py: Pre-click semantic guard prevents device.click() on elements
   that don't match toggle intents (follow/like/save)

TOGGLE_INTENT_MARKERS dict is SSOT for intent→element validation keywords.
Supports DE locale (gefolgt, abonnieren, gefällt, speichern).

162 passed, 0 regressions. All 5 previously-RED tests now GREEN.
This commit is contained in:
2026-04-27 23:22:00 +02:00
parent 5d50228945
commit 3b9465a3bc
3 changed files with 87 additions and 1 deletions

View File

@@ -321,6 +321,25 @@ class GoalExecutor:
self._get_sae().ensure_clear_screen(max_attempts=3)
return False
# ── Pre-Click Semantic Match Guard ──
# For toggle intents (follow/like/save), verify the selected node
# semantically matches the intent BEFORE clicking. This prevents
# VLM hallucinations from clicking photo grid items when looking
# for follow buttons.
from GramAddict.core.perception.action_memory import _intent_matches_node
node_semantic = (
f"text: '{best_node.get('text', '')}', "
f"desc: '{best_node.get('description', '')}', "
f"id: '{best_node.get('id', '')}'"
)
if not _intent_matches_node(action, node_semantic):
logger.warning(
f"🛡️ [GOAP Execute] Pre-click rejection: node does not match intent '{action}'. "
f"Node: {node_semantic}"
)
return False
# Execute click
self.device.click(obj=best_node)
import random

View File

@@ -5,6 +5,20 @@ from GramAddict.core.perception.spatial_parser import SpatialNode
logger = logging.getLogger(__name__)
# ═══════════════════════════════════════════════════════
# Semantic Match Keywords — SSOT for intent → element validation
# ═══════════════════════════════════════════════════════
# Maps toggle-intent keywords to required element markers.
# If the intent contains the key, the clicked element MUST
# contain at least one of the corresponding markers in its
# text, content_desc, or resource_id.
TOGGLE_INTENT_MARKERS = {
"follow": ["follow", "gefolgt", "abonnieren"],
"like": ["like", "heart", "gefällt"],
"save": ["save", "saved", "bookmark", "speichern"],
}
class ActionMemory:
"""
@@ -36,7 +50,11 @@ class ActionMemory:
logger.debug(f"🧠 [ActionMemory] Tracking tentative click for intent: '{intent}' -> {semantic_string}")
def confirm_click(self, intent: str = None):
"""Positive Reinforcement: Confirms the last click was successful."""
"""Positive Reinforcement: Confirms the last click was successful.
Guard: Refuses to store in Qdrant if the clicked element does not
semantically match the intent. Prevents memory poisoning.
"""
ctx = self._last_click_context
if not ctx:
return
@@ -44,6 +62,15 @@ class ActionMemory:
if intent and ctx["intent"] != intent:
return
# ── Semantic Mismatch Guard ──
if not _intent_matches_node(ctx["intent"], ctx["semantic_string"]):
logger.warning(
f"🛡️ [ActionMemory] BLOCKED confirm_click for '{ctx['intent']}'"
f"clicked element does not match intent: {ctx['semantic_string']}"
)
self._last_click_context = None
return
logger.info(f"✅ [ActionMemory] Confirming success for '{ctx['intent']}'. Boosting confidence.")
# Store or boost in Qdrant
@@ -145,6 +172,18 @@ class ActionMemory:
logger.error(f"Failed to query VLM for visual verification: {e}")
# Fallthrough to structural delta if VLM crashes
# ── Pre-Structural Semantic Gate ──
# Before trusting ANY structural delta, verify the clicked element
# semantically matches the intent. Prevents photo-clicks from
# being validated as follow/like successes.
if is_toggle and self._last_click_context:
if not _intent_matches_node(intent, self._last_click_context["semantic_string"]):
logger.warning(
f"🛡️ [ActionMemory] Semantic mismatch: '{intent}' does not match "
f"clicked element {self._last_click_context['semantic_string']}. Verification FAIL."
)
return False
# Fallback to structural delta if no device, VLM fails, or high confidence bypass
diff = abs(len(pre_click_xml) - len(post_click_xml))
@@ -166,3 +205,30 @@ class ActionMemory:
logger.warning(f"⚠️ [ActionMemory] No structural change detected for '{intent}'. Verification FAIL.")
return False
def _intent_matches_node(intent: str, semantic_string: str) -> bool:
"""Checks if the clicked element semantically matches the toggle intent.
For toggle intents (follow, like, save), the clicked element MUST contain
at least one of the required keywords in its text/desc/id. This prevents
photo grid items, captions, and other unrelated elements from being
falsely confirmed as successful interactions.
For non-toggle intents, returns True (no restriction).
"""
intent_lower = intent.lower()
semantic_lower = semantic_string.lower()
for intent_keyword, required_markers in TOGGLE_INTENT_MARKERS.items():
if intent_keyword in intent_lower:
if any(marker in semantic_lower for marker in required_markers):
return True
logger.debug(
f"🛡️ [SemanticGuard] Intent '{intent}' requires markers "
f"{required_markers} but element has: {semantic_string}"
)
return False
# Non-toggle intents pass through
return True

View File

@@ -138,6 +138,7 @@ class QNavGraph:
"like": "tap like button",
"comment": "tap comment button",
"share": "tap share button",
"follow": "tap follow button",
}
for keyword, required_action in action_checks.items():
if keyword in goal.lower() and required_action not in available: