feat(perception): implement extreme VLM guards and surface spatial metadata
1. Add long_clickable and class_name to SpatialNode and box_legend. 2. Inject strict guardrails into the SoM prompt to prevent EditText, Recents gallery, and long_clickable hallucinations. 3. Update text-based resolver with equivalent extreme fallback rules. 4. Ensures VLM explicitly rejects non-deterministic edge cases (e.g., Select Album).
This commit is contained in:
@@ -750,6 +750,11 @@ class IntentResolver:
|
||||
if node.text and node.text != node.content_desc:
|
||||
text = _humanize_desc(node.text)
|
||||
label_parts.append(f"text='{text[:50]}'")
|
||||
if node.class_name:
|
||||
cls_short = node.class_name.split(".")[-1]
|
||||
label_parts.append(f"class='{cls_short}'")
|
||||
if node.long_clickable:
|
||||
label_parts.append("long_clickable=True")
|
||||
if not label_parts:
|
||||
label_parts.append("(no visible text)")
|
||||
box_legend_lines.append(f" [{idx}] {', '.join(label_parts)}")
|
||||
@@ -791,7 +796,10 @@ class IntentResolver:
|
||||
f" - If looking for 'message input' or 'type message', do NOT select 'reactions' or emoji icons. Look for an empty text box or 'Message...'.\n"
|
||||
f"11. If the intent is 'feed post content' or 'post media content':\n"
|
||||
f" - Pick the largest box that contains the actual image or video, usually described as 'Photo', 'Video', or 'Carousel'.\n"
|
||||
f"12. If the exact control is NOT visible, return null. Do NOT guess.\n\n"
|
||||
f"12. DO NOT HALLUCINATE. If you are on the wrong screen (e.g. a 'New Post' creation screen when you want a profile), or if the exact target is simply NOT visible, you MUST return null. Returning a wrong box number will crash the bot.\n"
|
||||
f"13. EXTREME GUARD: NEVER pick an input field (class='EditText'), 'Send Message', or 'Reply to story' box unless the intent EXPLICITLY asks you to type or reply.\n"
|
||||
f"14. EXTREME GUARD: Do NOT pick items that are 'long_clickable=True' if your intent is just a simple navigation click, as this can trigger unwanted long-press context menus.\n"
|
||||
f"15. If the intent is 'tap post username', DO NOT pick random gallery folders like 'Recents' or 'Select album'. Return null.\n\n"
|
||||
f'Reply ONLY with a valid JSON object: {{"box": <number>}} or {{"box": null}}'
|
||||
)
|
||||
|
||||
@@ -919,7 +927,9 @@ class IntentResolver:
|
||||
" - 'profile tab' is usually the furthest right.\n"
|
||||
" - 'home tab' is the furthest left.\n"
|
||||
" - Do NOT select 'Go to <user>'s profile' or other header text.\n"
|
||||
"2. If none of the candidates clearly and safely match the intent, return null.\n\n"
|
||||
"2. EXTREME GUARD: NEVER pick an input field (EditText), 'Send Message', or 'Reply' unless explicitly asked to type.\n"
|
||||
"3. EXTREME GUARD: If you are on the wrong screen entirely (e.g. 'Select album' gallery) instead of a profile, return null.\n"
|
||||
"4. If none of the candidates clearly and safely match the intent, return null. DO NOT guess.\n\n"
|
||||
"Reply ONLY with a valid JSON object strictly matching this schema:\n"
|
||||
'{"selected_index": <integer or null>}\n'
|
||||
)
|
||||
|
||||
@@ -15,6 +15,7 @@ class SpatialNode:
|
||||
content_desc: str = ""
|
||||
resource_id: str = ""
|
||||
clickable: bool = False
|
||||
long_clickable: bool = False
|
||||
scrollable: bool = False
|
||||
|
||||
# Spatial Properties
|
||||
@@ -162,6 +163,7 @@ class SpatialParser:
|
||||
resource_id=attrib.get("resource-id", "").strip(),
|
||||
bounds=(left, top, right, bottom),
|
||||
clickable=attrib.get("clickable", "false") == "true",
|
||||
long_clickable=attrib.get("long-clickable", "false") == "true",
|
||||
scrollable=attrib.get("scrollable", "false") == "true",
|
||||
)
|
||||
nodes_list.append(node)
|
||||
|
||||
Reference in New Issue
Block a user