fix(GREEN): semantic match guard kills follow hallucination at 3 layers
Implements the _intent_matches_node() guard — a shared SSOT function that validates clicked elements against intent keywords before trusting any verification result. Fixes applied: 1. action_memory.py: verify_success() now cross-checks clicked element against intent BEFORE trusting structural delta for toggle actions 2. action_memory.py: confirm_click() blocks Qdrant poisoning when the tracked click doesn't semantically match the intent 3. q_nav_graph.py: 'follow' added to action_checks map (screen-sanity) 4. goap.py: Pre-click semantic guard prevents device.click() on elements that don't match toggle intents (follow/like/save) TOGGLE_INTENT_MARKERS dict is SSOT for intent→element validation keywords. Supports DE locale (gefolgt, abonnieren, gefällt, speichern). 162 passed, 0 regressions. All 5 previously-RED tests now GREEN.
This commit is contained in:
@@ -321,6 +321,25 @@ class GoalExecutor:
|
||||
self._get_sae().ensure_clear_screen(max_attempts=3)
|
||||
return False
|
||||
|
||||
# ── Pre-Click Semantic Match Guard ──
|
||||
# For toggle intents (follow/like/save), verify the selected node
|
||||
# semantically matches the intent BEFORE clicking. This prevents
|
||||
# VLM hallucinations from clicking photo grid items when looking
|
||||
# for follow buttons.
|
||||
from GramAddict.core.perception.action_memory import _intent_matches_node
|
||||
|
||||
node_semantic = (
|
||||
f"text: '{best_node.get('text', '')}', "
|
||||
f"desc: '{best_node.get('description', '')}', "
|
||||
f"id: '{best_node.get('id', '')}'"
|
||||
)
|
||||
if not _intent_matches_node(action, node_semantic):
|
||||
logger.warning(
|
||||
f"🛡️ [GOAP Execute] Pre-click rejection: node does not match intent '{action}'. "
|
||||
f"Node: {node_semantic}"
|
||||
)
|
||||
return False
|
||||
|
||||
# Execute click
|
||||
self.device.click(obj=best_node)
|
||||
import random
|
||||
|
||||
@@ -5,6 +5,20 @@ from GramAddict.core.perception.spatial_parser import SpatialNode
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ═══════════════════════════════════════════════════════
|
||||
# Semantic Match Keywords — SSOT for intent → element validation
|
||||
# ═══════════════════════════════════════════════════════
|
||||
|
||||
# Maps toggle-intent keywords to required element markers.
|
||||
# If the intent contains the key, the clicked element MUST
|
||||
# contain at least one of the corresponding markers in its
|
||||
# text, content_desc, or resource_id.
|
||||
TOGGLE_INTENT_MARKERS = {
|
||||
"follow": ["follow", "gefolgt", "abonnieren"],
|
||||
"like": ["like", "heart", "gefällt"],
|
||||
"save": ["save", "saved", "bookmark", "speichern"],
|
||||
}
|
||||
|
||||
|
||||
class ActionMemory:
|
||||
"""
|
||||
@@ -36,7 +50,11 @@ class ActionMemory:
|
||||
logger.debug(f"🧠 [ActionMemory] Tracking tentative click for intent: '{intent}' -> {semantic_string}")
|
||||
|
||||
def confirm_click(self, intent: str = None):
|
||||
"""Positive Reinforcement: Confirms the last click was successful."""
|
||||
"""Positive Reinforcement: Confirms the last click was successful.
|
||||
|
||||
Guard: Refuses to store in Qdrant if the clicked element does not
|
||||
semantically match the intent. Prevents memory poisoning.
|
||||
"""
|
||||
ctx = self._last_click_context
|
||||
if not ctx:
|
||||
return
|
||||
@@ -44,6 +62,15 @@ class ActionMemory:
|
||||
if intent and ctx["intent"] != intent:
|
||||
return
|
||||
|
||||
# ── Semantic Mismatch Guard ──
|
||||
if not _intent_matches_node(ctx["intent"], ctx["semantic_string"]):
|
||||
logger.warning(
|
||||
f"🛡️ [ActionMemory] BLOCKED confirm_click for '{ctx['intent']}' — "
|
||||
f"clicked element does not match intent: {ctx['semantic_string']}"
|
||||
)
|
||||
self._last_click_context = None
|
||||
return
|
||||
|
||||
logger.info(f"✅ [ActionMemory] Confirming success for '{ctx['intent']}'. Boosting confidence.")
|
||||
|
||||
# Store or boost in Qdrant
|
||||
@@ -145,6 +172,18 @@ class ActionMemory:
|
||||
logger.error(f"Failed to query VLM for visual verification: {e}")
|
||||
# Fallthrough to structural delta if VLM crashes
|
||||
|
||||
# ── Pre-Structural Semantic Gate ──
|
||||
# Before trusting ANY structural delta, verify the clicked element
|
||||
# semantically matches the intent. Prevents photo-clicks from
|
||||
# being validated as follow/like successes.
|
||||
if is_toggle and self._last_click_context:
|
||||
if not _intent_matches_node(intent, self._last_click_context["semantic_string"]):
|
||||
logger.warning(
|
||||
f"🛡️ [ActionMemory] Semantic mismatch: '{intent}' does not match "
|
||||
f"clicked element {self._last_click_context['semantic_string']}. Verification FAIL."
|
||||
)
|
||||
return False
|
||||
|
||||
# Fallback to structural delta if no device, VLM fails, or high confidence bypass
|
||||
diff = abs(len(pre_click_xml) - len(post_click_xml))
|
||||
|
||||
@@ -166,3 +205,30 @@ class ActionMemory:
|
||||
|
||||
logger.warning(f"⚠️ [ActionMemory] No structural change detected for '{intent}'. Verification FAIL.")
|
||||
return False
|
||||
|
||||
|
||||
def _intent_matches_node(intent: str, semantic_string: str) -> bool:
|
||||
"""Checks if the clicked element semantically matches the toggle intent.
|
||||
|
||||
For toggle intents (follow, like, save), the clicked element MUST contain
|
||||
at least one of the required keywords in its text/desc/id. This prevents
|
||||
photo grid items, captions, and other unrelated elements from being
|
||||
falsely confirmed as successful interactions.
|
||||
|
||||
For non-toggle intents, returns True (no restriction).
|
||||
"""
|
||||
intent_lower = intent.lower()
|
||||
semantic_lower = semantic_string.lower()
|
||||
|
||||
for intent_keyword, required_markers in TOGGLE_INTENT_MARKERS.items():
|
||||
if intent_keyword in intent_lower:
|
||||
if any(marker in semantic_lower for marker in required_markers):
|
||||
return True
|
||||
logger.debug(
|
||||
f"🛡️ [SemanticGuard] Intent '{intent}' requires markers "
|
||||
f"{required_markers} but element has: {semantic_string}"
|
||||
)
|
||||
return False
|
||||
|
||||
# Non-toggle intents pass through
|
||||
return True
|
||||
|
||||
@@ -138,6 +138,7 @@ class QNavGraph:
|
||||
"like": "tap like button",
|
||||
"comment": "tap comment button",
|
||||
"share": "tap share button",
|
||||
"follow": "tap follow button",
|
||||
}
|
||||
for keyword, required_action in action_checks.items():
|
||||
if keyword in goal.lower() and required_action not in available:
|
||||
|
||||
Reference in New Issue
Block a user