diff --git a/GramAddict/core/perception/intent_resolver.py b/GramAddict/core/perception/intent_resolver.py index 77e3670..eafc4e8 100644 --- a/GramAddict/core/perception/intent_resolver.py +++ b/GramAddict/core/perception/intent_resolver.py @@ -19,7 +19,7 @@ class IntentResolver: """ def resolve( - self, intent_description: str, candidates: List[SpatialNode], screen_height: int = 2400 + self, intent_description: str, candidates: List[SpatialNode], screen_height: int = 2400, device=None ) -> Optional[SpatialNode]: """ Finds the best matching node for a given intent autonomously. @@ -121,10 +121,86 @@ class IntentResolver: data = json.loads(res) idx = data.get("selected_index") if idx is not None and 0 <= idx < len(filtered_candidates): - return filtered_candidates[idx] + proposed_node = filtered_candidates[idx] + if device and not self._visual_critic(intent_description, proposed_node, device): + return None + return proposed_node except Exception as e: import logging logging.getLogger(__name__).warning(f"⚠️ [IntentResolver] VLM resolution failed ({e}).") return None + + def _visual_critic(self, intent_description: str, node: SpatialNode, device) -> bool: + """ + Validates the proposed node visually to prevent blind hallucinations. + Crops the UI snippet of the bounding box and asks the VLM to confirm it matches the intent. + """ + import logging + from GramAddict.core.llm_provider import query_telepathic_llm + from GramAddict.core.config import Config + import base64 + from io import BytesIO + + logger = logging.getLogger(__name__) + + try: + # 1. Crop the node's visual representation from the screen with padding + pad = 20 + img = device.deviceV2.screenshot() + width, height = img.size + + x1 = max(0, node.x1 - pad) + y1 = max(0, node.y1 - pad) + x2 = min(width, node.x2 + pad) + y2 = min(height, node.y2 + pad) + + cropped = img.crop((x1, y1, x2, y2)) + + # Skip validation if the area is bizarrely small + if (x2 - x1) < 10 or (y2 - y1) < 10: + return True + + buffered = BytesIO() + cropped.save(buffered, format="JPEG") + img_b64 = base64.b64encode(buffered.getvalue()).decode('utf-8') + + cfg = Config() + model = getattr(cfg.args, "ai_telepathic_model", "llava:latest") + url = getattr(cfg.args, "ai_telepathic_url", "http://localhost:11434/api/generate") + + prompt = ( + f"You are a strict UI Visual Validator.\n" + f"Look at this cropped image of a UI element.\n" + f"Does this element visually look like the correct target for the user intent: '{intent_description}'?\n" + f"Look at icons, text, and general appearance. Be highly skeptical.\n\n" + f"Reply ONLY with a valid JSON object strictly matching this schema:\n" + f'{{"valid": true}} or {{"valid": false}}' + ) + + logger.info(f"⚖️ [Visual Critic] Validating proposal (id='{node.resource_id}', text='{node.text}') for intent '{intent_description}'...") + res = query_telepathic_llm( + model=model, + url=url, + system_prompt="Strict visual JSON critic.", + user_prompt=prompt, + use_local_edge=True, + images_b64=[img_b64] + ) + + import json + data = json.loads(res) + is_valid = data.get("valid", True) + + if is_valid: + logger.info("✅ [Visual Critic] VLM confirmed the visual target.") + else: + logger.warning(f"❌ [Visual Critic] VLM REJECTED the target (Hallucination blocked).") + + return is_valid + + except Exception as e: + logger.warning(f"⚠️ [Visual Critic] Validation failed, falling back to accept: {e}") + return True +