feat(perception): implement Vision-Critic validation gate to block LLM hallucinations via cropped screenshot validation

This commit is contained in:
2026-04-27 14:57:20 +02:00
parent 0b68d4bc77
commit ac95dec9d8

View File

@@ -19,7 +19,7 @@ class IntentResolver:
"""
def resolve(
self, intent_description: str, candidates: List[SpatialNode], screen_height: int = 2400
self, intent_description: str, candidates: List[SpatialNode], screen_height: int = 2400, device=None
) -> Optional[SpatialNode]:
"""
Finds the best matching node for a given intent autonomously.
@@ -121,10 +121,86 @@ class IntentResolver:
data = json.loads(res)
idx = data.get("selected_index")
if idx is not None and 0 <= idx < len(filtered_candidates):
return filtered_candidates[idx]
proposed_node = filtered_candidates[idx]
if device and not self._visual_critic(intent_description, proposed_node, device):
return None
return proposed_node
except Exception as e:
import logging
logging.getLogger(__name__).warning(f"⚠️ [IntentResolver] VLM resolution failed ({e}).")
return None
def _visual_critic(self, intent_description: str, node: SpatialNode, device) -> bool:
"""
Validates the proposed node visually to prevent blind hallucinations.
Crops the UI snippet of the bounding box and asks the VLM to confirm it matches the intent.
"""
import logging
from GramAddict.core.llm_provider import query_telepathic_llm
from GramAddict.core.config import Config
import base64
from io import BytesIO
logger = logging.getLogger(__name__)
try:
# 1. Crop the node's visual representation from the screen with padding
pad = 20
img = device.deviceV2.screenshot()
width, height = img.size
x1 = max(0, node.x1 - pad)
y1 = max(0, node.y1 - pad)
x2 = min(width, node.x2 + pad)
y2 = min(height, node.y2 + pad)
cropped = img.crop((x1, y1, x2, y2))
# Skip validation if the area is bizarrely small
if (x2 - x1) < 10 or (y2 - y1) < 10:
return True
buffered = BytesIO()
cropped.save(buffered, format="JPEG")
img_b64 = base64.b64encode(buffered.getvalue()).decode('utf-8')
cfg = Config()
model = getattr(cfg.args, "ai_telepathic_model", "llava:latest")
url = getattr(cfg.args, "ai_telepathic_url", "http://localhost:11434/api/generate")
prompt = (
f"You are a strict UI Visual Validator.\n"
f"Look at this cropped image of a UI element.\n"
f"Does this element visually look like the correct target for the user intent: '{intent_description}'?\n"
f"Look at icons, text, and general appearance. Be highly skeptical.\n\n"
f"Reply ONLY with a valid JSON object strictly matching this schema:\n"
f'{{"valid": true}} or {{"valid": false}}'
)
logger.info(f"⚖️ [Visual Critic] Validating proposal (id='{node.resource_id}', text='{node.text}') for intent '{intent_description}'...")
res = query_telepathic_llm(
model=model,
url=url,
system_prompt="Strict visual JSON critic.",
user_prompt=prompt,
use_local_edge=True,
images_b64=[img_b64]
)
import json
data = json.loads(res)
is_valid = data.get("valid", True)
if is_valid:
logger.info("✅ [Visual Critic] VLM confirmed the visual target.")
else:
logger.warning(f"❌ [Visual Critic] VLM REJECTED the target (Hallucination blocked).")
return is_valid
except Exception as e:
logger.warning(f"⚠️ [Visual Critic] Validation failed, falling back to accept: {e}")
return True