From a2a4a756031d1897a57ed60db6ec7d4b953a513d Mon Sep 17 00:00:00 2001
From: Marc Mintel <marc@mintel.me>
Date: Mon, 27 Apr 2026 11:41:51 +0200
Subject: [PATCH] refactor(perception): replace XML length heuristic with VLM
 screenshot verification

---
 GramAddict/core/goap.py                     |  2 +-
 GramAddict/core/perception/action_memory.py | 39 ++++++++++++++++++---
 GramAddict/core/telepathic_engine.py        |  4 +--
 3 files changed, 38 insertions(+), 7 deletions(-)

diff --git a/GramAddict/core/goap.py b/GramAddict/core/goap.py
index 6905cb3..ecf3ec9 100644
--- a/GramAddict/core/goap.py
+++ b/GramAddict/core/goap.py
@@ -381,7 +381,7 @@ class GoalExecutor:
         else:
             # For interactions (like, follow) or unknown goals, use XML delta + semantic verify
             if ui_changed:
-                verification = engine.verify_success(action, post_xml)
+                verification = engine.verify_success(action, post_xml, device=self.device)
                 if verification is True:
                     action_success = True
                     logger.info(f"✅ [GOAP Step] Interaction '{action}' successful.")
diff --git a/GramAddict/core/perception/action_memory.py b/GramAddict/core/perception/action_memory.py
index 4ac04ab..856055d 100644
--- a/GramAddict/core/perception/action_memory.py
+++ b/GramAddict/core/perception/action_memory.py
@@ -77,9 +77,9 @@ class ActionMemory:
 
         self._last_click_context = None
 
-    def verify_success(self, intent: str, pre_click_xml: str, post_click_xml: str) -> Optional[bool]:
+    def verify_success(self, intent: str, pre_click_xml: str, post_click_xml: str, device=None) -> Optional[bool]:
         """
-        Structural verification: Did the UI actually change after the click?
+        Structural and Visual verification: Did the UI actually change after the click?
         """
         # Specific check for explore grid
         if "first image in explore grid" in intent or "grid item" in intent:
@@ -88,17 +88,48 @@ class ActionMemory:
             if "explore_action_bar" in post_click_xml and "row_feed_button_like" not in post_click_xml:
                 return None  # Still on grid, inconclusive
 
-        # State toggles (like, save, follow) rarely change XML length by > 50 chars
+        # State toggles (like, save, follow) rarely change XML length predictably
         state_toggles = ["like", "save", "follow", "heart"]
         if any(t in intent.lower() for t in state_toggles):
+            if device:
+                logger.info(f"👁️ [ActionMemory] Handing over verification for '{intent}' to VLM visual analysis...")
+                from GramAddict.core.perception.semantic_evaluator import SemanticEvaluator
+                evaluator = SemanticEvaluator()
+                
+                # Ask VLM to be the absolute source of truth
+                prompt = (
+                    f"The user just attempted to perform the action: '{intent}'. "
+                    f"Look at the current screen carefully. Was the action successful? "
+                    f"If the intent was 'follow', does the button now indicate 'Following' or 'Requested'? "
+                    f"If it was 'like', is the heart icon clearly active/red? "
+                    f"If the screen shifted completely to a profile when you just wanted to like/follow from a feed, it FAILED. "
+                    f"Answer ONLY with the word YES or NO."
+                )
+                
+                try:
+                    screenshot = device.screenshot_b64()
+                    response = evaluator._query_vlm(prompt, screenshot)
+                    
+                    if response and "yes" in response.lower() and "no" not in response.lower():
+                        logger.debug(f"🧠 [ActionMemory] VLM visually confirmed success for '{intent}'.")
+                        return True
+                    else:
+                        logger.warning(f"⚠️ [ActionMemory] VLM visual verification FAILED for '{intent}'. VLM replied: '{response}'")
+                        return False
+                except Exception as e:
+                    logger.error(f"Failed to query VLM for visual verification: {e}")
+                    # Fallthrough to structural delta if VLM crashes
+            
+            # Fallback to structural delta if no device or VLM fails
             diff = abs(len(pre_click_xml) - len(post_click_xml))
             if diff > 1000:
                 logger.warning(f"⚠️ [ActionMemory] Massive structural shift ({diff} chars) for state-toggle '{intent}'. Navigated away by mistake? Verification FAIL.")
                 return False
             if diff > 0:
-                logger.debug(f"🧠 [ActionMemory] State toggle detected for '{intent}'. Verification PASS.")
+                logger.debug(f"🧠 [ActionMemory] Structural delta detected for '{intent}'. Verification PASS.")
                 return True
 
+
         if abs(len(pre_click_xml) - len(post_click_xml)) > 50:
             logger.debug(f"🧠 [ActionMemory] Structural change detected for '{intent}'. Verification PASS.")
             return True
diff --git a/GramAddict/core/telepathic_engine.py b/GramAddict/core/telepathic_engine.py
index 8594087..459f0df 100644
--- a/GramAddict/core/telepathic_engine.py
+++ b/GramAddict/core/telepathic_engine.py
@@ -180,11 +180,11 @@ class TelepathicEngine:
     def decay_click(self, intent: str = None):
         self._memory.reject_click(intent)  # Alias to reject
 
-    def verify_success(self, intent: str, post_click_xml: str) -> bool:
+    def verify_success(self, intent: str, post_click_xml: str, device=None) -> bool:
         pre_click_xml = ""
         if self._memory._last_click_context:
             pre_click_xml = self._memory._last_click_context.get("xml_context", "")
-        return self._memory.verify_success(intent, pre_click_xml, post_click_xml)
+        return self._memory.verify_success(intent, pre_click_xml, post_click_xml, device=device)
 
     # ──────────────────────────────────────────────
     # Semantic Evaluator Delegation