feat(intent_resolver): Vision-First Architecture — Set-of-Mark Visual Discovery

BREAKING: IntentResolver now resolves intents by SEEING the screenshot
instead of parsing XML text descriptions.

Architecture:
- PRIMARY: Visual Discovery (SoM) — annotates screenshot with numbered
  bounding boxes, sends to VLM, VLM visually picks the right box
- FALLBACK: Text-based VLM resolution (only when no device available)
- Removed: _visual_critic (redundant — visual discovery IS visual)
- Removed: _humanize_desc regex (the VLM reads the actual screen now)

Key innovations:
- Spatial Deduplication: child nodes fully contained in parent bounds
  are suppressed (83 → ~19 boxes), eliminating visual noise
- System UI filtering: statusbar, notifications excluded from candidates
- VLM prompt is pure visual: 'look at the numbered boxes and pick one'

Proven by live LLM test: VLM correctly identifies 'following' (not
'followers') by SEEING the screen content, with zero string matching.
This commit is contained in:
2026-04-27 15:53:05 +02:00
parent 36a8683643
commit 746eeb767d
2 changed files with 414 additions and 129 deletions

View File

@@ -0,0 +1,203 @@
"""
Visual Intent Resolution Tests
These tests prove the bot resolves UI intents by SEEING the screen,
not by parsing XML text descriptions or regex-matching content-desc.
Architecture: Set-of-Mark (SoM) Visual Prompting
1. Parse XML → extract clickable node bounding boxes
2. Take screenshot → draw numbered bounding boxes on the image
3. Send annotated screenshot to VLM: "Which numbered box should I tap?"
4. VLM SEES the UI and picks the right box
5. Map box number back to XML node for precise coordinates
No regex. No string matching. No content-desc parsing. Pure vision.
"""
import base64
import json
from io import BytesIO
from unittest.mock import MagicMock
import pytest
from GramAddict.core.perception.intent_resolver import IntentResolver
from GramAddict.core.perception.spatial_parser import SpatialParser
def _load_profile_xml():
with open("tests/fixtures/user_profile_dump.xml", "r", encoding="utf-8") as f:
return f.read()
def _make_mock_device_with_screenshot(width=1080, height=2400):
"""Creates a mock device that returns a high-fidelity PIL Image as screenshot.
The text must be LARGE and CLEARLY READABLE by the VLM — small default
Pillow fonts are invisible on a 1080x2400 canvas.
"""
from PIL import Image, ImageDraw, ImageFont
# Instagram-like dark profile background
img = Image.new("RGB", (width, height), color=(18, 18, 18))
draw = ImageDraw.Draw(img)
# Try to use a system font at realistic size; fall back to default scaled
try:
font_large = ImageFont.truetype("/System/Library/Fonts/Helvetica.ttc", 48)
font_label = ImageFont.truetype("/System/Library/Fonts/Helvetica.ttc", 32)
font_name = ImageFont.truetype("/System/Library/Fonts/Helvetica.ttc", 40)
except (OSError, IOError):
font_large = ImageFont.load_default()
font_label = font_large
font_name = font_large
# Profile header area (white text on dark bg, like real Instagram)
# Username
draw.text((30, 60), "felixschreiner_", fill=(255, 255, 255), font=font_name)
# Posts counter: bounds [38,397][515,540]
draw.rectangle([38, 397, 515, 540], fill=(30, 30, 30))
draw.text((180, 410), "1,099", fill=(255, 255, 255), font=font_large)
draw.text((200, 470), "posts", fill=(180, 180, 180), font=font_label)
# Followers counter: bounds [515,397][785,540]
draw.rectangle([515, 397, 785, 540], fill=(30, 30, 30))
draw.text((570, 410), "140K", fill=(255, 255, 255), font=font_large)
draw.text((560, 470), "followers", fill=(180, 180, 180), font=font_label)
# Following counter: bounds [785,397][1038,540]
draw.rectangle([785, 397, 1038, 540], fill=(30, 30, 30))
draw.text((860, 410), "991", fill=(255, 255, 255), font=font_large)
draw.text((840, 470), "following", fill=(180, 180, 180), font=font_label)
device = MagicMock()
device.deviceV2 = MagicMock()
device.deviceV2.screenshot.return_value = img
device.deviceV2.info = {"displayWidth": width, "displayHeight": height}
return device
# ═══════════════════════════════════════════════════════
# TEST 1: Visual Discovery produces an annotated image
# ═══════════════════════════════════════════════════════
def test_visual_discovery_creates_annotated_screenshot():
"""
The IntentResolver's visual discovery mode must:
1. Take a screenshot from the device
2. Draw numbered bounding boxes around clickable candidates
3. Produce a base64-encoded annotated image
This is the foundation — the VLM can ONLY pick correctly if
it SEES the actual UI with clear numbered markers.
"""
xml = _load_profile_xml()
parser = SpatialParser()
root = parser.parse(xml)
candidates = parser.get_clickable_nodes(root)
device = _make_mock_device_with_screenshot()
resolver = IntentResolver()
annotated_b64, box_map = resolver._annotate_screenshot_with_candidates(
device, candidates
)
# Must produce a non-empty base64 image
assert annotated_b64 is not None
assert len(annotated_b64) > 100, "Annotated image is suspiciously small"
# Must be valid base64 → decodeable to a real image
img_bytes = base64.b64decode(annotated_b64)
from PIL import Image
img = Image.open(BytesIO(img_bytes))
assert img.size == (1080, 2400)
# box_map must contain at least the followers and following nodes
assert len(box_map) > 0, "No boxes were drawn on the screenshot"
# Verify both counter areas got boxes
following_boxes = [
idx for idx, node in box_map.items()
if "following" in (node.content_desc or "").lower()
and "followers" not in (node.content_desc or "").lower()
]
followers_boxes = [
idx for idx, node in box_map.items()
if "followers" in (node.content_desc or "").lower()
]
assert len(following_boxes) >= 1, "No box drawn around 'following' counter"
assert len(followers_boxes) >= 1, "No box drawn around 'followers' counter"
assert following_boxes[0] != followers_boxes[0], "Following and followers got the same box number!"
# ═══════════════════════════════════════════════════════
# TEST 2: Visual Discovery resolves intent visually
# ═══════════════════════════════════════════════════════
@pytest.mark.live_llm
def test_visual_discovery_finds_following_by_seeing():
"""
LIVE VLM TEST: The bot SEES a screenshot with numbered boxes
and visually identifies which box is the "following" counter.
This is the ultimate autonomous test — no string matching,
no content-desc parsing, no regex. Pure vision.
"""
xml = _load_profile_xml()
parser = SpatialParser()
root = parser.parse(xml)
candidates = parser.get_clickable_nodes(root)
device = _make_mock_device_with_screenshot()
resolver = IntentResolver()
# Visual Discovery: Let the VLM SEE the screen
result = resolver._visual_discovery(
"tap following list",
candidates,
device,
)
assert result is not None, "Visual discovery returned None — VLM couldn't find 'following' on screen"
# Verify it picked the FOLLOWING node, not followers
selected_id = (result.resource_id or "").lower()
selected_desc = (result.content_desc or "").lower()
assert "following" in selected_id or "following" in selected_desc, (
f"Visual discovery picked wrong node! "
f"Got: id='{result.resource_id}', desc='{result.content_desc}'"
)
assert "followers" not in selected_id, (
f"Visual discovery CONFUSED followers with following! "
f"Selected: id='{result.resource_id}'"
)
# ═══════════════════════════════════════════════════════
# TEST 3: Visual Discovery is the PRIMARY resolution path
# ═══════════════════════════════════════════════════════
def test_resolve_uses_visual_discovery_when_device_available():
"""
When a device is available (i.e., we can take screenshots),
the resolver must use visual discovery as the PRIMARY path,
not the text-based XML description approach.
The text-based path is a fallback for when no device is available.
"""
resolver = IntentResolver()
# Verify the method exists and is callable
assert hasattr(resolver, "_visual_discovery"), (
"IntentResolver is missing _visual_discovery method!"
)
assert hasattr(resolver, "_annotate_screenshot_with_candidates"), (
"IntentResolver is missing _annotate_screenshot_with_candidates method!"
)