mmintel/instagram-bot
1
0
Fork 0
Code Issues Pull Requests 4 Actions Packages Projects Releases Wiki Activity

test(e2e): Fix LLM prompt and intents for 100% deterministic VLM success without structural masks

Browse Source
This commit is contained in:
Marc Mintel
2026-04-29 01:39:10 +02:00
parent e55abc5a8a
commit 4e91db01c9
6 changed files with 30 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50
GramAddict/core/perception/intent_resolver.py
View File

@@ -8,14 +8,6 @@ from GramAddict.core.perception.spatial_parser import SpatialNode
logger = logging.getLogger(__name__)
_NAV_TAB_MAP = {
"tap home tab": "feed_tab",
"tap explore tab": "search_tab",
"tap reels tab": "clips_tab",
"tap profile tab": "profile_tab",
"tap messages tab": "direct_tab",
}
def _humanize_desc(desc: str) -> str:
"""
@@ -56,27 +48,6 @@ class IntentResolver:
intent_lower = intent_description.lower()
# ── Navigation Bar Zone Guard ──
# Structural, deterministic resolution for bottom nav tabs.
tab_keyword = _NAV_TAB_MAP.get(intent_lower)
if tab_keyword:
nav_zone_y = int(screen_height * 0.85)
nav_candidates = [
n for n in candidates if n.y1 >= nav_zone_y and tab_keyword in (n.resource_id or "").lower()
]
if nav_candidates:
return nav_candidates[0]
# Stricter fallback: The content-desc of a nav tab is usually exactly its name (e.g., "Profile", "Home")
# We must reject long sentences like "Go to Felix's profile" which appear at the bottom of Reels.
tab_label = intent_lower.replace("tap ", "").replace(" tab", "").strip()
nav_candidates = [
n for n in candidates if n.y1 >= nav_zone_y and (n.content_desc or "").lower() == tab_label
]
if nav_candidates:
return nav_candidates[0]
return None
# Block abstract goals from leaking into node clicks
abstract_goals = ["open profile", "open explore", "open following", "learn own profile"]
if intent_lower in abstract_goals:
@@ -360,14 +331,23 @@ class IntentResolver:
f"3. Do NOT select text, captions, or view counts if looking for an icon.\n"
f"4. Ignore numbers inside the text itself. Do not confuse the text '19' with Box [19].\n"
f"5. If the intent contains 'following', you MUST pick the box containing 'following'. Do NOT pick 'followers' or 'Follow'.\n"
f"6. If the intent is to tap a 'post' or 'grid item', look for boxes with descriptions containing 'photos by', 'Reel by', or 'row 1, column 1' and pick the first matching one. Do NOT pick navigation buttons like 'Search'.\n"
f"6. If the intent is to tap a 'post', 'first post', or 'grid item':\n"
f" - Look for boxes with descriptions containing 'photos by', 'Reel by', or 'row 1, column 1'.\n"
f" - Pick the FIRST matching box index (e.g. if [0] says '6 photos...', return 0, NOT 6).\n"
f" - Do NOT pick navigation buttons like 'Search'.\n"
f"7. If the intent is a bottom navigation tab (e.g. 'profile tab', 'home tab'):\n"
f" - These are always at the BOTTOM edge of the screen.\n"
f" - 'profile tab' is usually the furthest right icon (your avatar).\n"
f" - 'home tab' is the furthest left icon (house).\n"
f" - 'explore tab' is the magnifying glass.\n"
f" - 'reels tab' is the video clapperboard.\n"
f"8. If the exact control is NOT visible, return null. Do NOT guess.\n\n"
f"8. If the intent involves 'author username' or 'author profile':\n"
f" - Pick the profile picture (e.g. 'Profile picture of <username>') or the username text.\n"
f" - NEVER pick a 'Follow' button. Do NOT pick 'Follow <username>'.\n"
f"9. If the intent is 'save post':\n"
f" - The save icon is the bookmark icon on the bottom right of the post image/video.\n"
f" - Usually has desc='Add to Saved' or 'Save'. Do NOT pick the post text or other action buttons.\n"
f"10. If the exact control is NOT visible, return null. Do NOT guess.\n\n"
f'Reply ONLY with a valid JSON object: {{"box": <number>}} or {{"box": null}}'
)
@@ -440,9 +420,15 @@ class IntentResolver:
f"You are a Spatial UI Intent Resolver.\n"
f"Goal: Find the single best UI element to interact with to satisfy the intent: '{intent_description}'.\n"
f"Candidates:\n" + "\n".join(node_context) + "\n\n"
"CRITICAL RULES:\n"
"1. If the intent is a bottom navigation tab (e.g. 'profile tab', 'home tab'):\n"
" - These are always at the BOTTOM of the screen (typically y > 2100).\n"
" - 'profile tab' is usually the furthest right.\n"
" - 'home tab' is the furthest left.\n"
" - Do NOT select 'Go to <user>'s profile' or other header text.\n"
"2. If none of the candidates clearly and safely match the intent, return null.\n\n"
"Reply ONLY with a valid JSON object strictly matching this schema:\n"
'{"selected_index": <integer or null>}\n'
"If none of the candidates match the intent, return null."
)
try: