refactor: purge localized hardcoded strings and enforce language agnosticism
This commit is contained in:
@@ -14,20 +14,23 @@ MAX_REPLIES_PER_INBOX_VISIT = 3
|
||||
_EMPTY_CONTEXT_SENTINELS = frozenset({"no previous context", "", "none", "n/a"})
|
||||
|
||||
|
||||
# Structural resource-IDs that indicate a real "Send" button.
|
||||
def _is_send_button(node: dict) -> bool:
|
||||
"""Semantic verification: returns True if the node is identified as a Send button."""
|
||||
desc = (node.get("description") or node.get("desc", "")).lower()
|
||||
text = (node.get("text") or "").lower()
|
||||
"""
|
||||
Structural verification: returns True if the node is identified as a Send button.
|
||||
NO hardcoded UI strings (no localized 'send' or 'absenden').
|
||||
"""
|
||||
rid = (node.get("id") or node.get("resource_id", "")).lower()
|
||||
|
||||
# Accept if semantic markers indicate sending
|
||||
if any(m in rid for m in ["send", "composer_button"]):
|
||||
# 1. Structural Resource IDs (Language agnostic)
|
||||
if "send" in rid or "composer_button" in rid or "direct_send_button_text" in rid:
|
||||
return True
|
||||
if any(m in desc for m in ["send", "absenden"]):
|
||||
return True
|
||||
if text == "send" or text == "absenden":
|
||||
|
||||
# 2. Spatial Guard: The send button in a DM thread is ALWAYS on the far right side of the screen
|
||||
# next to the composer input.
|
||||
x = node.get("x", 0)
|
||||
if int(x) > (1080 * 0.75):
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
|
||||
@@ -220,8 +220,8 @@ class ActionMemory:
|
||||
)
|
||||
if is_toggle:
|
||||
prompt += (
|
||||
"If the intent was 'follow', does the button now indicate 'Following' or 'Requested'? "
|
||||
"If it was 'like', is the heart icon clearly active/red? "
|
||||
"If the intent was 'follow', did the button change its visual state to indicate an active subscription? "
|
||||
"If it was 'like', is the heart icon clearly active/filled? "
|
||||
"If the screen shifted completely to a profile when you just wanted to like/follow from a feed, it FAILED. "
|
||||
"If the tapped element does NOT sound like a like/follow button (e.g. it's a caption, comment field, or post content), it FAILED. "
|
||||
)
|
||||
|
||||
@@ -30,6 +30,11 @@ class IntentResolver:
|
||||
bounding boxes around clickable candidates, sends the annotated image to the VLM,
|
||||
and lets the VLM visually decide which box to tap.
|
||||
|
||||
CRITICAL ARCHITECTURE RULE: Language Agnosticism & Structural Determinism
|
||||
- NO hardcoded localized UI strings (e.g. "follow", "message", "like") are permitted.
|
||||
- All non-VLM resolution logic MUST rely strictly on spatial coordinates (e.g. center_x bounds)
|
||||
or structural Android resource IDs.
|
||||
|
||||
Architecture:
|
||||
1. Navigation tabs → structural zone guard (bottom 15%, resource-id)
|
||||
2. Everything else → Visual Discovery (screenshot + numbered boxes + VLM)
|
||||
@@ -137,8 +142,17 @@ class IntentResolver:
|
||||
# This prevents the weak VLM from hallucinating bounding box numbers that point to "Comment".
|
||||
interaction_suffixes = ["comment", "like", "share", "send", "save", "button_icon", "scrubber"]
|
||||
is_interaction = any(s in rid for s in interaction_suffixes) or any(s in desc for s in interaction_suffixes)
|
||||
node_text_lower = (node.text or "").lower()
|
||||
is_follow = "follow" in rid or "follow" in node_text_lower or "follow" in desc
|
||||
|
||||
# SPATIAL GUARD (NO HARDCODED STRINGS): The post author (avatar/name) is always strictly on the left side of the screen.
|
||||
# The "More options" (Report menu) button is strictly on the far right.
|
||||
# We enforce a mathematical boundary to prevent LLM hallucinations from clicking the report menu.
|
||||
if is_author_intent and node.center_x > (1080 * 0.75):
|
||||
logger.debug(
|
||||
f"🛡️ [Author Spatial Guard] Excluded far-right element '{node.resource_id}' "
|
||||
f"(x={node.center_x}) for author intent '{intent_description}'"
|
||||
)
|
||||
continue
|
||||
is_follow = "button_follow" in rid or "ufi_follow" in rid
|
||||
is_media_intent = "media content" in intent_lower or "image" in intent_lower or "video" in intent_lower
|
||||
|
||||
if (is_author_intent or is_media_intent) and (is_interaction or is_follow):
|
||||
@@ -309,10 +323,8 @@ class IntentResolver:
|
||||
if ("send" in intent_lower or "share" in intent_lower) and "post" in intent_lower and "button" in intent_lower:
|
||||
for node in candidates:
|
||||
rid = (node.resource_id or "").lower()
|
||||
desc = (node.content_desc or "").lower()
|
||||
# Feed uses row_feed_button_share, POST_DETAIL may use direct_share_button or just desc "Send"
|
||||
if "row_feed_button_share" in rid or "direct_share_button" in rid or "send" in desc or "share" in desc:
|
||||
logger.info(f"🎯 [Structural Fast-Path] Found send/share post button: {rid or desc}")
|
||||
if "row_feed_button_share" in rid or "direct_share_button" in rid or "button_share" in rid:
|
||||
logger.info(f"🎯 [Structural Fast-Path] Found send/share post button: {rid}")
|
||||
return node
|
||||
|
||||
if "add to story" in intent_lower:
|
||||
@@ -708,14 +720,14 @@ class IntentResolver:
|
||||
f"CRITICAL RULES:\n"
|
||||
f"1. If the intent contains a word in quotes (e.g., 'Search', 'New Message'), you MUST look at the Box legend and pick the box that contains that word (case-insensitive). Do not pick anything else.\n"
|
||||
f"2. For icons without text:\n"
|
||||
f" - 'like button' = HEART-SHAPED ICON (♡/❤), usually has desc='Like'.\n"
|
||||
f" - 'comment button' = SPEECH BUBBLE ICON, usually has desc='Comment'.\n"
|
||||
f" - 'like button' = HEART-SHAPED ICON (♡/❤), look for id containing 'button_like' or 'ufi_heart'.\n"
|
||||
f" - 'comment button' = SPEECH BUBBLE ICON, look for id containing 'button_comment' or 'ufi_comment'.\n"
|
||||
f"3. Do NOT select text, captions, or view counts if looking for an icon.\n"
|
||||
f"4. Ignore numbers inside the text itself. Do not confuse the text '19' with Box [19].\n"
|
||||
f"5. If the intent contains 'following', you MUST pick the box containing 'following'. Do NOT pick 'followers' or 'Follow'.\n"
|
||||
f"5. If the intent contains 'following', you MUST pick the box containing id 'button_following' or 'profile_header_following'. Do NOT pick 'followers' or 'Follow'.\n"
|
||||
f"6. If the intent is to tap a 'post', 'first post', or 'grid item':\n"
|
||||
f" - Look for boxes with descriptions containing 'photos by', 'Reel by', or 'row 1, column 1'.\n"
|
||||
f" - Pick the FIRST matching box index (e.g. if [0] says '6 photos...', return 0, NOT 6).\n"
|
||||
f" - Look for boxes with ids like 'image_button' inside a grid, or visual grid thumbnails.\n"
|
||||
f" - Pick the FIRST matching box index.\n"
|
||||
f" - Do NOT pick navigation buttons like 'Search'.\n"
|
||||
f"7. If the intent is a bottom navigation tab (e.g. 'profile tab', 'home tab'):\n"
|
||||
f" - These are always at the BOTTOM edge of the screen.\n"
|
||||
@@ -724,17 +736,17 @@ class IntentResolver:
|
||||
f" - 'explore tab' is the magnifying glass.\n"
|
||||
f" - 'reels tab' is the video clapperboard.\n"
|
||||
f"8. If the intent involves 'author username' or 'author profile':\n"
|
||||
f" - Pick the profile picture (e.g. 'Profile picture of <username>') or the username text.\n"
|
||||
f" - NEVER pick a 'Follow' button. Do NOT pick 'Follow <username>'.\n"
|
||||
f" - Pick the profile picture or the username text.\n"
|
||||
f" - NEVER pick a 'Follow' button. Do NOT pick 'button_follow'.\n"
|
||||
f"9. If the intent is 'save post':\n"
|
||||
f" - The save icon is the bookmark icon on the bottom right of the post image/video.\n"
|
||||
f" - Usually has desc='Add to Saved' or 'Save'. Do NOT pick the post text or other action buttons.\n"
|
||||
f" - Look for id containing 'button_save' or 'ufi_save'. Do NOT pick the post text or other action buttons.\n"
|
||||
f"10. DISTINGUISHING BOTTOM TABS vs CONTENT BUTTONS:\n"
|
||||
f" - Bottom Navigation Tabs (Home, Search, Reels, Profile) are ALWAYS at the very bottom (y > 2100).\n"
|
||||
f" - Content Interaction Buttons (Like, Comment, Share, Reactions, Message Input) are attached to posts or threads, NOT the bottom nav bar.\n"
|
||||
f" - If looking for 'message input' or 'type message', do NOT select 'reactions' or emoji icons. Look for an empty text box or 'Message...'.\n"
|
||||
f" - If looking for 'message input' or 'type message', do NOT select 'reactions' or emoji icons. Look for an empty text box or id 'message_content'.\n"
|
||||
f"11. If the intent is 'feed post content' or 'post media content':\n"
|
||||
f" - Pick the largest box that contains the actual image or video, usually described as 'Photo', 'Video', or 'Carousel'.\n"
|
||||
f" - Pick the largest box that contains the actual image or video. Look for id 'zoomable_view_container' or 'media_frame'.\n"
|
||||
f"12. If the exact control is NOT visible, return null. Do NOT guess.\n\n"
|
||||
f'Reply ONLY with a valid JSON object: {{"box": <number>}} or {{"box": null}}'
|
||||
)
|
||||
|
||||
@@ -205,6 +205,12 @@ class ScreenIdentity:
|
||||
logger.info("🛡️ [ScreenIdentity] Critical obstacle detected → DANGER_ACTION_BLOCKED")
|
||||
return ScreenType.DANGER_ACTION_BLOCKED
|
||||
|
||||
# Menu Trap Detection (O(1) fast-path)
|
||||
# Overrides hallucinated LLM cache using STRICTLY STRUCTURAL IDs (no localized strings).
|
||||
if "bottom_sheet_container" in ids and "action_sheet_row_text_view" in ids:
|
||||
logger.info("🛡️ [ScreenIdentity] Options/Report menu trap detected → MODAL")
|
||||
return ScreenType.MODAL
|
||||
|
||||
# Priority 2: Structural Heuristics (100% Deterministic)
|
||||
if "unified_follow_list_tab_layout" in ids or "follow_list_container" in ids:
|
||||
return ScreenType.FOLLOW_LIST
|
||||
@@ -367,42 +373,30 @@ class ScreenIdentity:
|
||||
if tab_id in resource_ids:
|
||||
actions.append(action)
|
||||
|
||||
# Screen-specific actions
|
||||
desc_lower = " ".join(content_descs).lower()
|
||||
text_lower = " ".join(texts).lower()
|
||||
# Screen-specific actions (Purely structural, NO localized strings)
|
||||
ids_str = " ".join(resource_ids).lower()
|
||||
|
||||
if "like" in desc_lower:
|
||||
if "button_like" in ids_str or "ufi_heart" in ids_str:
|
||||
actions.append("tap like button")
|
||||
if "comment" in desc_lower:
|
||||
if "button_comment" in ids_str or "ufi_comment" in ids_str:
|
||||
actions.append("tap comment button")
|
||||
if "share" in desc_lower:
|
||||
if "button_share" in ids_str or "ufi_share" in ids_str or "direct_share" in ids_str:
|
||||
actions.append("tap share button")
|
||||
if "save" in desc_lower or "bookmark" in desc_lower:
|
||||
if "button_save" in ids_str or "ufi_save" in ids_str:
|
||||
actions.append("tap save button")
|
||||
if "back" in desc_lower:
|
||||
if "back" in ids_str or "action_bar_button_back" in ids_str:
|
||||
actions.append("tap back button")
|
||||
has_following = any(
|
||||
"following" in e.get("text", "").lower() or "following" in e.get("desc", "").lower()
|
||||
for e in clickable_elements
|
||||
)
|
||||
|
||||
has_following = "button_following" in ids_str or "profile_header_following" in ids_str
|
||||
if has_following:
|
||||
actions.append("tap following button")
|
||||
elif any(
|
||||
"follow" in e.get("text", "").lower()
|
||||
or "follow" in e.get("desc", "").lower()
|
||||
or "follow" in e.get("id", "").lower()
|
||||
for e in clickable_elements
|
||||
):
|
||||
elif "button_follow" in ids_str:
|
||||
actions.append("tap follow button")
|
||||
|
||||
if screen_type == ScreenType.OWN_PROFILE or screen_type == ScreenType.OTHER_PROFILE:
|
||||
if "message" in desc_lower:
|
||||
if "button_message" in ids_str or "direct_message" in ids_str:
|
||||
actions.append("tap message button")
|
||||
if (
|
||||
"following" in desc_lower
|
||||
or "following" in text_lower
|
||||
or "profile_header_following" in " ".join(resource_ids).lower()
|
||||
):
|
||||
if "profile_header_following" in ids_str:
|
||||
actions.append("tap following list")
|
||||
|
||||
# Grid items
|
||||
|
||||
@@ -8,6 +8,11 @@ and learns from every episode — positive AND negative.
|
||||
|
||||
After initial learning, 95%+ of situations are handled from memory
|
||||
alone with ZERO LLM calls. This is "Tesla fleet learning" for bots.
|
||||
|
||||
CRITICAL ARCHITECTURE RULE: Language Agnosticism & Structural Determinism
|
||||
NO hardcoded localized UI strings (e.g. "Report", "OK", "Deny") are permitted.
|
||||
All structural heuristic fallbacks must use O(1) Android `resource-id` bounds
|
||||
to ensure the bot functions flawlessly regardless of the device language.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
@@ -536,12 +541,12 @@ class SituationalAwarenessEngine:
|
||||
"- If you see a dismiss/close/cancel/skip/not now button, click it\n"
|
||||
"- If the Situation type is obstacle_locked_screen, action must be 'unlock'\n"
|
||||
"- If the Situation type is obstacle_foreign_app, action must be 'kill_foreign_apps'\n"
|
||||
"- If the Situation type is obstacle_system, you MUST look for 'Deny', 'Don't allow', or 'Cancel' and click it. \n"
|
||||
" NEVER click 'Allow', 'OK', or 'Confirm' on system permissions.\n"
|
||||
"- If the Situation type is obstacle_system, you MUST look for the negative action (e.g. deny, block, do not allow, cancel) and click it. \n"
|
||||
" NEVER click the positive action (allow, accept, confirm) on system permissions.\n"
|
||||
" If no negative action button exists, action must be 'back'\n"
|
||||
"- If there is NO obstacle and the screen is a normal Instagram view (false positive), action must be 'false_positive'\n"
|
||||
"- If nothing else works, suggest 'app_start' to force-reopen Instagram\n"
|
||||
"- NEVER click 'OK'/'Confirm'/'Accept' on surveys or prompts\n"
|
||||
"- NEVER click the positive/accept button on surveys or prompts\n"
|
||||
"- When you choose to click, you MUST use the EXACT coordinates provided in `center=(x,y)` for that element in the XML\n"
|
||||
'- Return ONLY valid JSON: {"action": "click"|"back"|"app_start"|"unlock"|"kill_foreign_apps"|"false_positive", "x": N, "y": N, "reason": "..."}'
|
||||
)
|
||||
|
||||
@@ -93,6 +93,7 @@ class TelepathicEngine:
|
||||
if not any(k in intent_lower for k in typing_keywords):
|
||||
logger.warning("⌨️ [TelepathicEngine] Keyboard detected during non-typing intent! Auto-dismissing.")
|
||||
import time
|
||||
|
||||
device.back()
|
||||
time.sleep(1.0)
|
||||
# Re-fetch UI state
|
||||
@@ -129,8 +130,8 @@ class TelepathicEngine:
|
||||
(best_node.text or "") + " " + (best_node.content_desc or "") + " " + (best_node.resource_id or "")
|
||||
)
|
||||
semantic = semantic.lower()
|
||||
# Zero-Maintenance: Only English UI states. resource_id never changes with locale.
|
||||
if "following" in semantic or "requested" in semantic:
|
||||
# Zero-Maintenance: Strictly structural IDs, no localized strings.
|
||||
if "button_following" in semantic or "profile_header_following" in semantic:
|
||||
return {"skip": True, "semantic": "already_followed"}
|
||||
|
||||
# 4. Track action
|
||||
|
||||
@@ -127,19 +127,23 @@ def _run_zero_latency_unfollow_loop(
|
||||
extra={"color": Fore.YELLOW},
|
||||
)
|
||||
|
||||
# Find 'Following' button on their profile
|
||||
# Find the button that indicates active subscription (Following)
|
||||
following_nodes = telepathic._extract_semantic_nodes(
|
||||
profile_xml, "find 'Following' button", threshold=0.7
|
||||
profile_xml,
|
||||
"find the button indicating active subscription, look for id 'button_following' or 'profile_header_following'",
|
||||
threshold=0.7,
|
||||
)
|
||||
if following_nodes and not following_nodes[0].get("skip"):
|
||||
f_node = following_nodes[0]
|
||||
_humanized_click(device, f_node["x"], f_node["y"])
|
||||
random_sleep(1.0, 2.0)
|
||||
|
||||
# Find 'Unfollow' confirm
|
||||
# Find the confirmation button
|
||||
confirm_xml = device.dump_hierarchy()
|
||||
confirm_nodes = telepathic._extract_semantic_nodes(
|
||||
confirm_xml, "find 'Unfollow' confirmation button", threshold=0.8
|
||||
confirm_xml,
|
||||
"find the confirmation button to stop following, look for id 'follow_sheet_unfollow_row' or red warning text",
|
||||
threshold=0.8,
|
||||
)
|
||||
|
||||
if confirm_nodes and not confirm_nodes[0].get("skip"):
|
||||
|
||||
13
README.md
13
README.md
@@ -63,3 +63,16 @@ The engine has undergone a massive stabilization refactor to achieve **100% TDD
|
||||
|
||||
> [!NOTE]
|
||||
> Unlike legacy bots, GramPilot requires zero maintenance. It will automatically re-learn the UI over time using its integrated Qdrant memory vectors.
|
||||
|
||||
---
|
||||
|
||||
## 🏛️ Core Architecture Rules: Language Agnosticism & Structural Determinism
|
||||
|
||||
**CRITICAL: Hardcoding localized UI strings (e.g., "Follow", "Like", "Send", "Report") is STRICTLY FORBIDDEN across the entire codebase.**
|
||||
|
||||
GramPilot operates on a globally language-agnostic plane. If the UI is switched to German, Arabic, or Japanese, the bot must not fail. To achieve this, all perception and navigation logic must adhere to the following strict rules:
|
||||
|
||||
1. **Spatial Geometry over Text:** Use coordinates and boundaries to identify elements. (e.g., The "Send" button in DMs is *always* on the far right `center_x > width * 0.75`).
|
||||
2. **Structural Resource IDs over Descriptions:** Use Android UI resource IDs (`action_sheet_row_text_view`, `button_following`) instead of English `desc` or `text` properties. IDs are not localized and are universally stable.
|
||||
3. **Semantic LLM Prompts:** When querying the Vision-Language Model (VLM), **do not give exact string examples** that assume an English UI. For example, do not say `find the 'Unfollow' button`. Instead, say `find the button to stop following, look for a red warning text or id 'follow_sheet_unfollow_row'`.
|
||||
4. **Zero Magie:** Any text-based matching logic is considered a legacy flaw and will be mercilessly purged. If you find a text match, replace it with a structural O(1) fast-path or spatial guard.
|
||||
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user