feat(perception): implement extreme VLM guards and surface spatial metadata

1. Add long_clickable and class_name to SpatialNode and box_legend.
2. Inject strict guardrails into the SoM prompt to prevent EditText, Recents gallery, and long_clickable hallucinations.
3. Update text-based resolver with equivalent extreme fallback rules.
4. Ensures VLM explicitly rejects non-deterministic edge cases (e.g., Select Album).
This commit is contained in:
2026-05-05 11:01:28 +02:00
parent 5ada31c77e
commit 548fc374ae
2 changed files with 14 additions and 2 deletions

View File

@@ -750,6 +750,11 @@ class IntentResolver:
if node.text and node.text != node.content_desc:
text = _humanize_desc(node.text)
label_parts.append(f"text='{text[:50]}'")
if node.class_name:
cls_short = node.class_name.split(".")[-1]
label_parts.append(f"class='{cls_short}'")
if node.long_clickable:
label_parts.append("long_clickable=True")
if not label_parts:
label_parts.append("(no visible text)")
box_legend_lines.append(f" [{idx}] {', '.join(label_parts)}")
@@ -791,7 +796,10 @@ class IntentResolver:
f" - If looking for 'message input' or 'type message', do NOT select 'reactions' or emoji icons. Look for an empty text box or 'Message...'.\n"
f"11. If the intent is 'feed post content' or 'post media content':\n"
f" - Pick the largest box that contains the actual image or video, usually described as 'Photo', 'Video', or 'Carousel'.\n"
f"12. If the exact control is NOT visible, return null. Do NOT guess.\n\n"
f"12. DO NOT HALLUCINATE. If you are on the wrong screen (e.g. a 'New Post' creation screen when you want a profile), or if the exact target is simply NOT visible, you MUST return null. Returning a wrong box number will crash the bot.\n"
f"13. EXTREME GUARD: NEVER pick an input field (class='EditText'), 'Send Message', or 'Reply to story' box unless the intent EXPLICITLY asks you to type or reply.\n"
f"14. EXTREME GUARD: Do NOT pick items that are 'long_clickable=True' if your intent is just a simple navigation click, as this can trigger unwanted long-press context menus.\n"
f"15. If the intent is 'tap post username', DO NOT pick random gallery folders like 'Recents' or 'Select album'. Return null.\n\n"
f'Reply ONLY with a valid JSON object: {{"box": <number>}} or {{"box": null}}'
)
@@ -919,7 +927,9 @@ class IntentResolver:
" - 'profile tab' is usually the furthest right.\n"
" - 'home tab' is the furthest left.\n"
" - Do NOT select 'Go to <user>'s profile' or other header text.\n"
"2. If none of the candidates clearly and safely match the intent, return null.\n\n"
"2. EXTREME GUARD: NEVER pick an input field (EditText), 'Send Message', or 'Reply' unless explicitly asked to type.\n"
"3. EXTREME GUARD: If you are on the wrong screen entirely (e.g. 'Select album' gallery) instead of a profile, return null.\n"
"4. If none of the candidates clearly and safely match the intent, return null. DO NOT guess.\n\n"
"Reply ONLY with a valid JSON object strictly matching this schema:\n"
'{"selected_index": <integer or null>}\n'
)

View File

@@ -15,6 +15,7 @@ class SpatialNode:
content_desc: str = ""
resource_id: str = ""
clickable: bool = False
long_clickable: bool = False
scrollable: bool = False
# Spatial Properties
@@ -162,6 +163,7 @@ class SpatialParser:
resource_id=attrib.get("resource-id", "").strip(),
bounds=(left, top, right, bottom),
clickable=attrib.get("clickable", "false") == "true",
long_clickable=attrib.get("long-clickable", "false") == "true",
scrollable=attrib.get("scrollable", "false") == "true",
)
nodes_list.append(node)