feat(navigation): Implement 100% autonomous Blank Start architecture; purge heuristics

This commit is contained in:
2026-04-22 00:05:03 +02:00
parent fff9ec5b0a
commit 75009d91a2
41 changed files with 1875 additions and 231 deletions

View File

@@ -76,7 +76,7 @@ def start_bot(**kwargs):
username = getattr(configs.args, "username", "") or "unknown_user"
# Parse persona interests from config (comma-separated string → list)
persona_raw = getattr(configs.args, "ai_target_audience", getattr(configs.args, "persona_interests", ""))
persona_raw = getattr(configs.args, "ai_target_audience", getattr(configs.args, "persona_interests", getattr(configs.args, "target_audience", "")))
persona_interests = [p.strip() for p in persona_raw.split(",") if p.strip()] if persona_raw else []
dopamine = DopamineEngine()
@@ -96,7 +96,20 @@ def start_bot(**kwargs):
darwin = DarwinEngine(username)
from GramAddict.core.telepathic_engine import TelepathicEngine
telepathic = TelepathicEngine()
telepathic = TelepathicEngine.get_instance()
# ── Stage 0: Blank Start (Scorched Earth) ──
if getattr(configs.args, "blank_start", False):
logger.warning(f"⚠️ [Blank Start] Wiping ALL persistent AI memories for '{username}'...")
telepathic.wipe()
# Wipe navigation paths too
try:
from GramAddict.core.goap import PathMemory
path_mem = PathMemory(username)
path_mem.wipe()
logger.info("🗑️ Wiped PathMemory collection.")
except Exception as e:
logger.warning(f"⚠️ Failed to wipe PathMemory: {e}")
cognitive_stack = {
"active_inference": active_inference,
@@ -196,22 +209,23 @@ def start_bot(**kwargs):
# Tap first grid post to learn from actual captions
if nav_graph.do("tap first image post in profile grid"):
logger.info("📸 [Identity Boot] Reading recent posts to analyze actual content vibe...", extra={"color": f"{Fore.CYAN}"})
sleep(2.0)
for _ in range(3):
post_xml = device.dump_hierarchy()
if isinstance(post_xml, str):
post_data = _extract_post_content(post_xml)
if post_data.get("caption"):
raw_bio_text.append(post_data["caption"])
elif post_data.get("description"):
raw_bio_text.append(post_data["description"])
post_loaded = _wait_for_post_loaded(device, timeout=5)
if post_loaded:
logger.info("📸 [Identity Boot] Reading recent posts to analyze actual content vibe...", extra={"color": f"{Fore.CYAN}"})
for _ in range(3):
post_xml = device.dump_hierarchy()
if isinstance(post_xml, str):
post_data = _extract_post_content(post_xml)
if post_data.get("caption"):
raw_bio_text.append(post_data["caption"])
elif post_data.get("description"):
raw_bio_text.append(post_data["description"])
_humanized_scroll(device, is_skip=False)
sleep(2.0)
_humanized_scroll(device, is_skip=False)
sleep(2.0)
device.press("back")
sleep(1.5)
device.press("back")
sleep(1.5)
# Deduplicate while preserving order
unique_texts = list(dict.fromkeys(raw_bio_text))
@@ -294,11 +308,17 @@ def start_bot(**kwargs):
nav_graph.do("tap first image in explore grid")
# Wait for post to actually load (poll for feed markers)
_wait_for_post_loaded(device, timeout=5)
post_loaded = _wait_for_post_loaded(device, nav_graph=nav_graph, timeout=5)
if not post_loaded:
logger.warning("❌ Post failed to open from grid. Retrying next loop.")
continue
elif current_target == "StoriesFeed":
logger.info("📱 Locating story tray on HomeFeed...")
nav_graph.do("tap story ring avatar")
_wait_for_post_loaded(device, timeout=5)
post_loaded = _wait_for_story_loaded(device, timeout=5)
if not post_loaded:
logger.warning("❌ Stories failed to open from HomeFeed. Retrying next loop.")
continue
if current_target == "StoriesFeed":
result = _run_zero_latency_stories_loop(device, configs, session_state, cognitive_stack)
@@ -361,9 +381,10 @@ FEED_MARKERS = [
def _wait_for_post_loaded(device, timeout=5):
def _wait_for_post_loaded(device, timeout=5, nav_graph=None):
"""Polls the UI hierarchy until feed markers appear, confirming a post is on screen."""
start = time.time()
xml = ""
while time.time() - start < timeout:
try:
xml = device.dump_hierarchy()
@@ -373,8 +394,56 @@ def _wait_for_post_loaded(device, timeout=5):
except Exception:
pass
sleep(0.5)
logger.warning("⚠️ Post did not load within timeout. Proceeding anyway.")
logger.warning("⚠️ Post did not load within timeout. Attempting Adaptive Snap.")
dump_ui_state(device, "post_load_timeout", {"timeout_sec": timeout})
try:
xml_lower = xml.lower()
# 1. Trapped in a Story or Reel viewer? Press back.
if "reel_viewer_root" in xml_lower or "clips_viewer" in xml_lower:
logger.warning("🧗 [Adaptive Snap] Trapped in Story/Reel viewer. Pressing BACK.")
device.press("back")
sleep(1.5)
# Give it one more chance to load the feed
xml = device.dump_hierarchy()
if any(marker in xml for marker in FEED_MARKERS):
logger.info("✅ Recovered to Feed.")
return True
# 2. Trapped in Profile?
if "profile_header" in xml_lower and "row_feed_photo_profile_name" not in xml_lower:
logger.warning("🧗 [Adaptive Snap] Trapped in Profile. Pressing BACK.")
device.press("back")
sleep(1.5)
# 3. Stuck between posts (Feed markers not fully visible)? Try to align or wobble.
# Fallback micro-wobble
info = device.get_info()
w, h = info.get("displayWidth", 1080), info.get("displayHeight", 2400)
logger.warning("🧗 [Adaptive Snap] Wobbling to force render.")
device.swipe(int(w/2), int(h/2), int(w/2), int(h/2) - 100, 100)
sleep(0.5)
device.swipe(int(w/2), int(h/2) - 100, int(w/2), int(h/2), 100)
except Exception as e:
logger.error(f"❌ [Adaptive Snap] Failed: {e}")
return False
def _wait_for_story_loaded(device, timeout=5):
"""Polls the UI hierarchy until story markers appear, confirming a story is on screen."""
start = time.time()
while time.time() - start < timeout:
try:
xml_lower = device.dump_hierarchy().lower()
if "reel_viewer_root" in xml_lower or "story_viewer" in xml_lower:
logger.debug("📱 Story loaded successfully.")
return True
except Exception:
pass
sleep(0.5)
logger.warning("⚠️ Story did not load within timeout.")
return False
def _humanized_scroll(device, is_skip=False, resonance_score=None):
@@ -651,6 +720,11 @@ def _interact_with_profile(device, configs, username, session_state, sleep_mod,
xml_dump = device.dump_hierarchy()
has_story = "reel_ring" in xml_dump or "'s unseen story" in xml_dump.lower() or "has a new story" in xml_dump.lower() or "story von" in xml_dump.lower()
if has_story and nav_graph.do("tap story ring avatar"):
post_loaded = _wait_for_story_loaded(device, timeout=5)
if not post_loaded:
logger.warning(f"❌ Story failed to open for @{username}.")
return
logger.info(f"📸 [Story] Viewing @{username}'s story ({count} times)...")
for i in range(count):
sleep(random.uniform(2.0, 5.0) * sleep_mod)
@@ -700,6 +774,11 @@ def _interact_with_profile(device, configs, username, session_state, sleep_mod,
nav_graph = QNavGraph(device)
if nav_graph.do("tap first image post in profile grid"):
post_loaded = _wait_for_post_loaded(device, timeout=5)
if not post_loaded:
logger.warning(f"❌ Post failed to open from profile grid of @{username}.")
return
logger.info(f"❤️ [Deep Interaction] Opening grid to drop {count} likes on @{username}...")
for i in range(count):
@@ -1308,7 +1387,7 @@ def _run_zero_latency_feed_loop(device, zero_engine, nav_graph, configs, session
# Return to feed
logger.info("🔙 [Profile Learning] Returning to main feed.")
device.press("back")
_wait_for_post_loaded(device)
_wait_for_post_loaded(device, nav_graph=nav_graph)
sleep(random.uniform(1.0, 1.5) * sleep_mod)
rnd_interact = random.random()
@@ -1687,7 +1766,8 @@ def _run_zero_latency_feed_loop(device, zero_engine, nav_graph, configs, session
# ── Active Inference: Evaluate prediction (after action) ──
if ai:
_wait_for_post_loaded(device, timeout=3)
# Wait for content to settle
_wait_for_post_loaded(device, timeout=3, nav_graph=nav_graph)
post_action_xml = device.dump_hierarchy()
ai.evaluate_prediction(post_action_xml)

View File

@@ -141,6 +141,7 @@ class Config:
self.parser.add_argument("--time-delta-session", help="Time delta between sessions", default=None)
self.parser.add_argument("--restart-atx-agent", action="store_true", help="Restart atx agent")
self.parser.add_argument("--allow-untested-ig-version", action="store_true", help="Allow untested IG version")
self.parser.add_argument("--blank-start", action="store_true", help="Wipe all learned navigation and telepathic memories on boot to start 100% blank.")
# Interaction settings
self.parser.add_argument("--likes-count", help="Likes count", default="2-3")
@@ -174,6 +175,7 @@ class Config:
# Persona & Resonance (drives ALL content evaluation and interaction decisions)
self.parser.add_argument("--persona-interests", help="Comma-separated niche interests for content matching", default="")
self.parser.add_argument("--ai-target-audience", help="Target audience used interchangeably with persona interests", default="")
self.parser.add_argument("--target-audience", help="Target audience used interchangeably with persona interests", default="")
self.parser.add_argument("--interact-percentage", help="Overall interaction probability percentage", default="80")
self.parser.add_argument("--comment-percentage", help="Comment probability percentage", default="0")
self.parser.add_argument("--follow-percentage", help="Follow probability percentage", default="0")

View File

@@ -44,6 +44,10 @@ def get_device_info(device):
logger.debug(f"Device Info: {info.get('productName')} | SDK: {info.get('sdkInt')}")
class DeviceFacade:
deviceV2 = None
app_id = None
device_id = None
def __init__(self, device_id, app_id, args):
self.device_id = device_id
self.app_id = app_id
@@ -94,6 +98,11 @@ class DeviceFacade:
self.deviceV2.press("home")
sleep(1)
@adb_retry()
def unlock(self):
self.deviceV2.unlock()
@property
def info(self):
return self.deviceV2.info

View File

@@ -22,6 +22,7 @@ from typing import Optional, List, Dict, Any
from enum import Enum
from GramAddict.core.utils import random_sleep
from GramAddict.core.qdrant_memory import QdrantBase
logger = logging.getLogger(__name__)
@@ -154,7 +155,7 @@ class ScreenIdentity:
# ── Extract available actions from clickable elements ──
available_actions = self._extract_available_actions(
clickable_elements, resource_ids, content_descs, screen_type
clickable_elements, resource_ids, content_descs, texts, screen_type
)
# ── Extract context ──
@@ -181,6 +182,9 @@ class ScreenIdentity:
pass
# Priority 2: Structural Heuristics (Instant, for core tabs)
if 'unified_follow_list_tab_layout' in ids or 'follow_list_container' in ids:
return ScreenType.FOLLOW_LIST
if selected_tab == 'feed_tab': return ScreenType.HOME_FEED
if selected_tab == 'clips_tab': return ScreenType.REELS_FEED
if selected_tab == 'search_tab': return ScreenType.EXPLORE_GRID
@@ -223,7 +227,7 @@ class ScreenIdentity:
return ScreenType.UNKNOWN
def _extract_available_actions(self, clickable_elements, resource_ids, content_descs, screen_type):
def _extract_available_actions(self, clickable_elements, resource_ids, content_descs, texts, screen_type):
"""Discover what actions are possible on this screen."""
actions = []
@@ -241,6 +245,7 @@ class ScreenIdentity:
# Screen-specific actions
desc_lower = ' '.join(content_descs).lower()
text_lower = ' '.join(texts).lower()
if 'like' in desc_lower:
actions.append('tap like button')
@@ -254,6 +259,12 @@ class ScreenIdentity:
actions.append('tap back button')
if any('follow' in e.get('text', '').lower() for e in clickable_elements):
actions.append('tap follow button')
if screen_type == ScreenType.OWN_PROFILE or screen_type == ScreenType.OTHER_PROFILE:
if 'message' in desc_lower or 'nachricht' in desc_lower:
actions.append('tap message button')
if 'following' in desc_lower or 'abonniert' in desc_lower or 'following' in text_lower:
actions.append('tap following list')
# Grid items
if screen_type == ScreenType.EXPLORE_GRID:
@@ -321,13 +332,23 @@ class PathMemory:
Enables instant recall for known goals.
"""
def __init__(self):
def __init__(self, username: str = ""):
self.username = username
try:
from GramAddict.core.qdrant_memory import QdrantBase
self._db = QdrantBase("goap_paths_v1", vector_size=768)
suffix = f"_{username}" if username else ""
self._db = QdrantBase(f"goap_paths_v1{suffix}", vector_size=768)
except Exception:
self._db = None
def wipe(self):
"""Wipe all learned navigation paths from Qdrant."""
if self._db and self._db.is_connected:
try:
self._db.client.delete_collection(self._db.collection_name)
logger.info(f"🗑️ [PathMemory] Wiped Qdrant collection: {self._db.collection_name}")
except Exception as e:
logger.warning(f"⚠️ [PathMemory] Could not wipe collection: {e}")
def recall_path(self, goal: str, current_screen_type: str) -> Optional[List[Dict]]:
"""
Recall a previously successful path for this goal from this screen type.
@@ -342,9 +363,18 @@ class PathMemory:
return None
try:
from qdrant_client.models import Filter, FieldCondition, MatchValue
results = self._db.client.query_points(
collection_name=self._db.collection_name,
query=vec,
query_filter=Filter(
must=[
FieldCondition(
key="start_screen",
match=MatchValue(value=current_screen_type)
)
]
),
limit=3,
score_threshold=0.85,
).points
@@ -373,7 +403,7 @@ class PathMemory:
if not vec:
return
seed = f"{goal}|{start_screen}|{len(steps)}|{success}"
seed = f"{goal}|{start_screen}"
payload = {
"goal": goal,
"start_screen": start_screen,
@@ -390,6 +420,22 @@ class PathMemory:
log_success=f"🧠 [GOAP Learn] {outcome} Path for '{goal}': {len(steps)} steps from {start_screen}"
)
def forget_path(self, goal: str, start_screen: str):
"""Remove a cached path to force re-discovery."""
if not self._db or not self._db.is_connected:
return
seed = f"{goal}|{start_screen}"
try:
from qdrant_client import models
point_id = self._db._get_id(seed)
self._db.client.delete(
collection_name=self._db.collection_name,
points_selector=models.PointIdsList(points=[point_id])
)
except Exception as e:
logger.debug(f"Failed to forget path: {e}")
# ══════════════════════════════════════════════════════
# 3. GOAL PLANNER — "What should I do next?"
@@ -405,44 +451,204 @@ class GoalPlanner:
3. LLM planning (slow, for truly unknown situations)
"""
# ── Navigation knowledge: Screen → Tab mapping ──
# This is NOT hardcoded navigation. It's the bot's understanding of
# WHERE things live (like knowing a GPS needs street addresses).
# The bot can discover these itself, but we seed them for speed.
SCREEN_TAB_MAP = {
ScreenType.HOME_FEED: 'feed_tab',
ScreenType.EXPLORE_GRID: 'search_tab',
ScreenType.REELS_FEED: 'clips_tab',
ScreenType.OWN_PROFILE: 'profile_tab',
ScreenType.DM_INBOX: 'direct_tab',
}
class NavigationKnowledge:
"""
Manages the bot's learned understanding of the Instagram UI.
Discovered dynamically through exploration and success.
"""
def __init__(self, username: str):
self.username = username
try:
self._db = QdrantBase("navigation_knowledge", vector_size=768)
except Exception:
self._db = None
# In-memory cache for rapidly avoiding traps during exploration
# In-memory cache for rapidly avoiding traps during exploration
self._learned_screen_mappings = {}
# ── Goal → Required screen type mapping ──
# "To achieve X, I first need to be on screen Y"
GOAL_SCREEN_REQUIREMENTS = {
'like a post': [ScreenType.HOME_FEED, ScreenType.POST_DETAIL],
'like this post': [ScreenType.POST_DETAIL, ScreenType.HOME_FEED],
'follow this user': [ScreenType.OTHER_PROFILE],
'open explore': [ScreenType.EXPLORE_GRID],
'open explore feed': [ScreenType.EXPLORE_GRID],
'open home feed': [ScreenType.HOME_FEED],
'open reels': [ScreenType.REELS_FEED],
'open profile': [ScreenType.OWN_PROFILE],
'learn own profile': [ScreenType.OWN_PROFILE],
'open messages': [ScreenType.DM_INBOX],
'tap first grid item': [ScreenType.EXPLORE_GRID],
'view a post from explore': [ScreenType.EXPLORE_GRID],
'visit profile': [ScreenType.OTHER_PROFILE],
'go back': [ScreenType.UNKNOWN],
}
def wipe(self):
"""Wipe all learned knowledge from Qdrant."""
if self._db and self._db.is_connected:
try:
self._db.client.delete_collection(self._db.collection_name)
logger.info(f"🗑️ [NavigationKnowledge] Wiped Qdrant collection: {self._db.collection_name}")
except Exception as e:
logger.warning(f"⚠️ [NavigationKnowledge] Could not wipe knowledge: {e}")
def plan_next_step(self, goal: str, screen: Dict[str, Any]) -> Optional[str]:
"""
Plans the NEXT single action to take toward the goal.
def update_username(self, username: str):
"""Update username and reconnect DB if needed."""
if self.username != username:
self.username = username
try:
self._db = QdrantBase("navigation_knowledge", vector_size=768)
except Exception:
self._db = None
def get_requirements(self, goal: str) -> List[ScreenType]:
"""Get required screens for a goal. Returns known requirements or empty list."""
if not self._db or not self._db.is_connected:
return []
try:
from qdrant_client.models import Filter, FieldCondition, MatchValue
results = self._db.client.scroll(
collection_name=self._db.collection_name,
scroll_filter=Filter(
must=[
FieldCondition(
key="goal",
match=MatchValue(value=goal)
)
]
),
limit=1
)[0]
if results:
screen_name = results[0].payload.get("required_screen")
logger.debug(f"🧠 [Nav Knowledge] Found requirement for '{goal}': {screen_name}")
if screen_name:
return [ScreenType[screen_name]]
except Exception as e:
logger.warning(f"⚠️ [Nav Knowledge] Search error: {e}")
return []
def learn_goal_requirement(self, goal: str, screen_type: ScreenType):
"""Learn that achieving 'goal' lands us on 'screen_type'."""
if not self._db or not self._db.is_connected:
logger.warning("⚠️ [Nav Knowledge] Cannot learn: DB not connected")
return
seed = f"req_{goal}"
vec = self._db._get_embedding(f"goal_requirement: {goal}")
payload = {
"goal": goal,
"required_screen": screen_type.name,
"timestamp": time.time()
}
self._db.upsert_point(seed, payload, vector=vec)
logger.info(f"🧠 [Nav Knowledge] Learned: '{goal}'{screen_type.name}")
def get_action_for_screen(self, target_screen: ScreenType) -> Optional[str]:
"""Find which action leads to this screen."""
for action, screen in self._learned_screen_mappings.items():
if screen == target_screen:
return action
if not self._db or not self._db.is_connected:
return None
try:
from qdrant_client.models import Filter, FieldCondition, MatchValue
results = self._db.client.scroll(
collection_name=self._db.collection_name,
scroll_filter=Filter(
must=[
FieldCondition(
key="result_screen",
match=MatchValue(value=target_screen.name)
)
]
),
limit=1
)[0]
if results:
return results[0].payload.get("action")
except Exception:
pass
return None
def get_screen_for_action(self, action: str) -> Optional[ScreenType]:
"""Find where this action leads to to avoid looping traps."""
if action in self._learned_screen_mappings:
return self._learned_screen_mappings[action]
Returns a natural language action string like 'tap explore tab'
or None if the goal is already achieved.
"""
if not self._db or not self._db.is_connected:
return None
try:
from qdrant_client.models import Filter, FieldCondition, MatchValue
results = self._db.client.scroll(
collection_name=self._db.collection_name,
scroll_filter=Filter(
must=[
FieldCondition(
key="action",
match=MatchValue(value=action)
)
]
),
limit=1
)[0]
if results:
screen_name = results[0].payload.get("result_screen")
if screen_name:
return ScreenType[screen_name]
except Exception:
pass
return None
def learn_screen_mapping(self, action: str, result_screen: ScreenType):
"""Learn that taking 'action' leads to 'result_screen'."""
if not self._db or not self._db.is_connected:
return
seed = f"map_{action}"
vec = self._db._get_embedding(f"screen_mapping: {result_screen.name}")
payload = {
"action": action,
"result_screen": result_screen.name,
"timestamp": time.time()
}
self._learned_screen_mappings[action] = result_screen
self._db.upsert_point(seed, payload, vector=vec)
logger.info(f"🧠 [Nav Knowledge] Learned Mapping: '{action}'{result_screen.name}")
def get_screen_for_tab(self, tab_id: str) -> Optional[ScreenType]:
"""Find where this tab leads to to avoid looping traps."""
if tab_id in self._learned_screen_mappings:
return self._learned_screen_mappings[tab_id]
if not self._db or not self._db.is_connected:
return None
try:
from qdrant_client.models import Filter, FieldCondition, MatchValue
results = self._db.client.scroll(
collection_name=self._db.collection_name,
scroll_filter=Filter(
must=[
FieldCondition(
key="tab_id",
match=MatchValue(value=tab_id)
)
]
),
limit=1
)[0]
if results:
s_name = results[0].payload.get("result_screen")
if s_name:
return ScreenType[s_name]
except Exception:
pass
return None
class GoalPlanner:
"""
Given a goal and current screen state, plans the next action.
Uses Dynamic Discovery to navigate without hardcoded maps.
"""
def __init__(self, username: str):
self.knowledge = NavigationKnowledge(username)
def plan_next_step(self, goal: str, screen: Dict[str, Any], explored_nav_actions: set = None) -> Optional[str]:
"""Plans the NEXT single action to take toward the goal."""
screen_type = screen['screen_type']
available = screen.get('available_actions', [])
context = screen.get('context', {})
@@ -454,19 +660,16 @@ class GoalPlanner:
return None
# ── 2. Am I on the right screen? If not, navigate there ──
nav_action = self._plan_navigation(goal_lower, screen_type, available)
selected_tab = screen.get('selected_tab')
nav_action = self._plan_navigation(goal_lower, screen_type, available, selected_tab, explored_nav_actions)
if nav_action:
return nav_action
# ── 3. I'm on the right screen — execute the goal action ──
# ── 3. Execute the goal action ──
goal_action = self._plan_goal_action(goal_lower, screen_type, available, context)
if goal_action:
return goal_action
# ── 4. Fallback: try scroll or back ──
if 'scroll down' in available:
return 'scroll down'
return 'press back'
def _is_goal_achieved(self, goal: str, screen_type: ScreenType, context: dict) -> bool:
@@ -485,53 +688,69 @@ class GoalPlanner:
return True
if 'open messages' in goal and screen_type == ScreenType.DM_INBOX:
return True
if ('following list' in goal or 'followers list' in goal) and screen_type == ScreenType.FOLLOW_LIST:
return True
if 'open explore' in goal and screen_type == ScreenType.EXPLORE_GRID:
return True
if 'view profile' in goal and (screen_type == ScreenType.OWN_PROFILE or screen_type == ScreenType.OTHER_PROFILE):
return True
return False
def _plan_navigation(self, goal: str, screen_type: ScreenType, available: List[str]) -> Optional[str]:
def _plan_navigation(self, goal: str, screen_type: ScreenType, available: List[str], selected_tab: Optional[str] = None, explored_nav_actions: set = None) -> Optional[str]:
"""If we're on the wrong screen, figure out how to navigate."""
# Find what screen(s) we need for this goal
required_screens = None
for goal_pattern, screens in self.GOAL_SCREEN_REQUIREMENTS.items():
if goal_pattern in goal:
required_screens = screens
break
# 1. Get required screens for this goal from knowledge
required_screens = self.knowledge.get_requirements(goal)
# 2. Blank Start Discovery (if knowledge is empty)
if not required_screens:
return None
logger.info(f"🧠 [Nav Discovery] No known requirements for '{goal}'. Will attempt autonomous discovery.")
# We don't know the required screen or path. Let the TelepathicEngine figure out
# what button to press based on the pure goal text!
if explored_nav_actions and goal in explored_nav_actions:
logger.info(f"🛑 [Nav Discovery] Autonomous intent '{goal}' already tried and failed/trapped. Yielding to back-tracking.")
pass # Don't return goal again, let it press back if possible
else:
return goal
# If we're already on an acceptable screen, no navigation needed
# 3. If we're already on an acceptable screen, no navigation needed
if screen_type in required_screens:
return None
# Find the tab we need to tap
# 4. Find the action we need to take
for target_screen in required_screens:
target_tab = self.SCREEN_TAB_MAP.get(target_screen)
if target_tab:
# Map tab to action
tab_actions = {
'feed_tab': 'tap home tab',
'search_tab': 'tap explore tab',
'clips_tab': 'tap reels tab',
'profile_tab': 'tap profile tab',
'direct_tab': 'tap messages tab',
}
action = tab_actions.get(target_tab)
if action and action in available:
logger.info(f"🧭 [GOAP Navigate] Need {target_screen.value} for '{goal}'{action}")
return action
known_action = self.knowledge.get_action_for_screen(target_screen)
if not known_action:
logger.info(f"🧭 [Nav Discovery] Don't know action to reach {target_screen.name}. Asking VLM...")
# Check ALL available actions if one linguistically aligns with the target screen
# Or just let VLM figure it out by returning an intention.
screen_friendly_name = target_screen.name.replace('_', ' ').lower()
# Semantic Heuristic Match on dynamically perceived actions
goal_words = [w.rstrip('s') for w in screen_friendly_name.split() if len(w) > 3]
for action in available:
if any(w in action.lower() for w in goal_words):
# Verify we don't know this leads somewhere else
known_target = self.knowledge.get_screen_for_action(action)
if known_target and known_target != target_screen:
continue # Trap prevention
logger.info(f"🎯 [Nav Discovery] Linguistic match on available action! '{action}' aligns with '{screen_friendly_name}'")
return action
# If no perceived action matches, return intent for TelepathicEngine
return f"navigate to {screen_friendly_name}"
else:
if known_action in available:
logger.info(f"🧭 [Nav Knowledge] Navigating to {target_screen.name} via '{known_action}'")
return known_action
# If no tab navigation works, try going back first
# If no targeted navigation works, try going back first
if 'press back' in available:
logger.info("🧭 [GOAP Navigate] Can't reach required screen directly. Pressing back...")
return 'press back'
# Heuristic Fallback: If we are on an UNKNOWN screen and have NO tab buttons visible,
# we are likely in a deep view (like a DM thread or nested settings).
# Suggesting 'press back' even if not explicitly found in available_actions
# as a generic escape mechanism.
if screen_type == ScreenType.UNKNOWN and not any(tab in available for tab in tab_actions.values()):
logger.info("🧠 [GOAP Heuristic] Stuck on UNKNOWN screen with no tabs. Suggesting 'press back' fallback.")
# Only press back if we aren't currently on the required screen (handled in step 3)
return 'press back'
return None
@@ -539,10 +758,14 @@ class GoalPlanner:
def _plan_goal_action(self, goal: str, screen_type: ScreenType, available: List[str], context: dict) -> Optional[str]:
"""Plan the specific action to achieve the goal on the current screen."""
import re
if 'like' in goal and 'tap like button' in available:
return 'tap like button'
if 'following list' in goal and 'tap following list' in available:
return 'tap following list'
if 'follow' in goal and 'tap follow button' in available:
if re.search(r'\bfollow\b', goal) and 'tap follow button' in available:
return 'tap follow button'
if 'comment' in goal and 'tap comment button' in available:
@@ -593,11 +816,13 @@ class GoalExecutor:
def __init__(self, device, bot_username: str = ""):
self.device = device
self.username = bot_username
self.screen_id = ScreenIdentity(bot_username)
self.planner = GoalPlanner()
self.path_memory = PathMemory()
self.planner = GoalPlanner(bot_username)
self.path_memory = PathMemory(bot_username)
self.max_steps = 15 # Safety: never execute more than 15 steps
self._sae = None # Lazy-loaded, injectable for tests
self.action_failures = {} # Tracking for failed actions in current goal session
def _get_sae(self):
"""Get or create the SAE instance. Injectable for tests."""
@@ -627,6 +852,7 @@ class GoalExecutor:
max_steps = self.max_steps
logger.info(f"🎯 [GOAP] Pursuing goal: '{goal}'")
self.action_failures.clear()
# ── Try recalled path first ──
screen = self.perceive()
@@ -642,10 +868,24 @@ class GoalExecutor:
# ── Live planning ──
steps_taken = []
last_action = None
explored_nav_actions = set()
for step_num in range(max_steps):
# PERCEIVE
screen = self.perceive()
screen_type = screen['screen_type']
# ── Loop Prevention: Mask Failed Actions ──
MAX_RETRIES = 2
original_available = screen.get('available_actions', []).copy()
masked_available = []
for act in original_available:
fail_count = self.action_failures.get(act, 0)
if fail_count >= MAX_RETRIES:
logger.warning(f"🚫 [GOAP] Masking action '{act}' due to {fail_count} consecutive failures to prevent loops.")
else:
masked_available.append(act)
screen['available_actions'] = masked_available
logger.debug(
f"📍 [GOAP Step {step_num + 1}] On: {screen_type.value} | "
@@ -653,48 +893,64 @@ class GoalExecutor:
)
# Handle obstacles
if screen_type == ScreenType.FOREIGN_APP:
logger.warning("🚨 [GOAP] Foreign app detected. Using SAE to recover...")
if screen_type == ScreenType.FOREIGN_APP or screen_type == ScreenType.MODAL:
obstacle_name = "Foreign app" if screen_type == ScreenType.FOREIGN_APP else "Modal"
logger.warning(f"🚨 [GOAP] {obstacle_name} detected. Using SAE to clear...")
# SAE Feedback Loop!
# If we hit this, the LAST action caused an obstacle! Mask it!
if last_action:
self.action_failures[last_action] = self.action_failures.get(last_action, 0) + MAX_RETRIES # Instantly mask it
logger.warning(f"🛡️ [SAE Feedback] Action '{last_action}' caused an obstacle. Masking aggressively.")
if not self._get_sae().ensure_clear_screen():
self.path_memory.learn_path(goal, start_screen, steps_taken, False)
return False
continue
if screen_type == ScreenType.MODAL:
logger.warning("🚨 [GOAP] Modal detected. Using SAE to clear...")
self._get_sae().ensure_clear_screen()
if screen_type == ScreenType.FOREIGN_APP:
self.path_memory.learn_path(goal, start_screen, steps_taken, False)
return False
continue
# PLAN
action = self.planner.plan_next_step(goal, screen)
action = self.planner.plan_next_step(goal, screen, explored_nav_actions=explored_nav_actions)
if action is None:
# Goal achieved!
logger.info(f"✅ [GOAP] Goal '{goal}' achieved in {step_num} steps!")
self.path_memory.learn_path(goal, start_screen, steps_taken, True)
# Record dynamic knowledge: This goal lands us on THIS screen
self.planner.knowledge.learn_goal_requirement(goal, screen_type)
return True
logger.info(f"🧭 [GOAP Step {step_num + 1}] Action: '{action}'")
last_action = action
# EXECUTE
success = self._execute_action(action)
success = self._execute_action(action, goal=goal)
steps_taken.append({
'screen': screen_type.value,
'action': action,
'success': success,
})
if not success:
if success:
steps_taken.append({"action": action})
# Check if it was a navigation action (vs a goal action). If we are not on the required screen,
# any action taken is essentially a navigation attempt.
explored_nav_actions.add(action)
# Reset failures for this action since it eventually succeeded
self.action_failures[action] = 0
else:
self.action_failures[action] = self.action_failures.get(action, 0) + 1
logger.warning(f"⚠️ [GOAP] Action '{action}' failed. Continuing with replanning...")
random_sleep(0.5, 1.5)
logger.error(f" [GOAP] Failed to achieve '{goal}' in {max_steps} steps")
logger.warning(f"⚠️ [GOAP] Goal '{goal}' failed after {max_steps} steps.")
self.path_memory.learn_path(goal, start_screen, steps_taken, False)
# Memory Purge Logic: Wipe the path memory for this start_screen/goal combo
# so it doesn't get stuck in a broken loop in future sessions!
self.path_memory.forget_path(goal, start_screen)
logger.warning(f"🧹 [Memory Purge] Wiped PathMemory cache for '{goal}' starting at '{start_screen}' to force re-discovery.")
return False
def _execute_action(self, action: str) -> bool:
def _execute_action(self, action: str, goal: str = None) -> bool:
"""Execute a single natural-language action using the TelepathicEngine."""
if action == 'press back':
@@ -738,9 +994,51 @@ class GoalExecutor:
import random
time.sleep(random.uniform(1.6, 2.8))
# Verify UI changed
# Verify success via Goal Context + Screen Feedback
post_xml = self.device.dump_hierarchy()
if post_xml != xml_dump:
post_screen = self.perceive(post_xml)
post_screen_type = post_screen['screen_type']
# Determine if this was a navigation or an interaction
is_navigation = any(k in action.lower() for k in ["tab", "open", "go to", "navigate"])
action_success = False
goal_met = False
ui_changed = post_xml != xml_dump
if is_navigation:
if ui_changed:
action_success = True
logger.info(f"✅ [GOAP Step] Navigation '{action}' successful -> {post_screen_type.name}.")
# Record dynamic knowledge: This action leads to THIS screen
self.planner.knowledge.learn_screen_mapping(action, post_screen_type)
else:
logger.warning(f"❌ [GOAP Step] No UI change detected after '{action}'.")
action_success = False
else:
# For interactions (like, follow) or unknown goals, use XML delta + semantic verify
if ui_changed:
if engine.verify_success(action, post_xml):
action_success = True
logger.info(f"✅ [GOAP Step] Interaction '{action}' successful.")
else:
logger.warning(f"❌ [GOAP Verify] Semantic verification failed for '{action}'.")
else:
logger.warning(f"❌ [GOAP Verify] No UI change detected after interaction '{action}'.")
# Optional: Log if the overarching goal was miraculously met early
if goal and action_success:
required_screens = self.planner.knowledge.get_requirements(goal)
if not required_screens:
if 'messages' in goal and post_screen_type == ScreenType.DM_INBOX: goal_met = True
if 'explore' in goal and post_screen_type == ScreenType.EXPLORE_GRID: goal_met = True
if 'home' in goal and post_screen_type == ScreenType.HOME_FEED: goal_met = True
if 'profile' in goal and post_screen_type == ScreenType.OWN_PROFILE: goal_met = True
if 'reels' in goal and post_screen_type == ScreenType.REELS_FEED: goal_met = True
if goal_met or (required_screens and post_screen_type in required_screens):
logger.info(f"🎉 [GOAP Verify] OVERARCHING Goal '{goal}' achieved during step '{action}'.")
if action_success:
engine.confirm_click(action)
return True
else:
@@ -753,7 +1051,7 @@ class GoalExecutor:
action = step.get('action', '')
logger.info(f"🧠 [GOAP Recall Step {i + 1}/{len(steps)}] '{action}'")
success = self._execute_action(action)
success = self._execute_action(action, goal=goal)
if not success:
logger.warning(f"⚠️ [GOAP Recall] Step '{action}' failed. Path may be stale.")
return False

View File

@@ -196,6 +196,15 @@ class QdrantBase:
except Exception as e:
self._handle_error(e, f"Search failed")
return []
def get_collection_size(self) -> int:
"""Returns the number of points in the collection."""
if not self.is_connected:
return 0
try:
count_result = self.client.count(collection_name=self.collection_name)
return count_result.count
except Exception:
return 0
class HeuristicMemoryDB(QdrantBase):
@@ -1062,6 +1071,13 @@ class ParasocialCRMDB(QdrantBase):
interactions = current.get("interactions", [])
import time
now = time.time()
if interactions:
last_interaction = interactions[-1]
if last_interaction.get("type") == intent_type and (now - last_interaction.get("timestamp", 0)) < 300:
logger.debug(f"🧠 [ParasocialCRM] Skipping redundant '{intent_type}' write for @{username}.")
return
interactions.append({
"type": intent_type,
"timestamp": now

View File

@@ -271,33 +271,54 @@ class SituationalAwarenessEngine:
logger.info("📱 [SAE Perceive] Screen is physically OFF.")
return SituationType.OBSTACLE_LOCKED_SCREEN
# ── Foreign App / System Dialog Detection (package-based) ──
# ── Foreign Environment Detection (package-based) ──
# Extract ALL packages present in the dump
# Support both single and double quotes for package detection (robustness across uia2 versions and tests)
packages = set(re.findall(r'package=["\']([^"\']+)["\']', xml_dump))
app_id = getattr(self.device, 'app_id', 'com.instagram.android')
# ── System Dialog Detection (BEFORE foreign app — system dialogs overlay Instagram) ──
system_packages = {'com.android.permissioncontroller', 'com.android.settings',
'com.google.android.packageinstaller'}
if system_packages & packages:
return SituationType.OBSTACLE_SYSTEM
# If Instagram package is not present AT ALL, we're in a foreign app
# If Instagram package is not present AT ALL, we are outside the app.
if app_id not in packages:
# Exception: system UI overlay is normal (status bar)
non_system = packages - {'com.android.systemui', 'android'}
if non_system:
logger.info(f"🔍 [SAE Perceive] Foreign app detected: {non_system}")
# We explicitly ask the TelepathicEngine to classify this to avoid writing brittle substring hacks
# for Android System UI variations across different device manufacturers.
try:
from GramAddict.core.llm_provider import query_telepathic_llm
from GramAddict.core.config import Config
screen_off = not getattr(self.device.deviceV2, 'info', {}).get("screenOn", True)
prompt = (
"You are a Situation Classifier for a mobile automation agent.\n"
"Analyze the given Android UI XML dump. Is this a physical DEVICE_LOCK_SCREEN, "
"a system PERMISSION_DIALOG, or a NOTIFICATION_SHADE / FOREIGN_APP?\n"
f"Hardware Screen Status: {'OFF (Locked)' if screen_off else 'ON'}.\n"
"Respond ONLY with a valid JSON object strictly matching this schema: "
"{\"situation\": \"OBSTACLE_LOCKED_SCREEN\" | \"OBSTACLE_SYSTEM\" | \"OBSTACLE_FOREIGN_APP\"}\n\n"
f"XML:\n{self._compress_xml(xml_dump)[:2500]}"
)
args = {}
try: args = Config().args
except Exception: pass
model = getattr(args, "ai_telepathic_model", "qwen3.5:latest")
url = getattr(args, "ai_telepathic_url", "http://localhost:11434/api/generate")
res = query_telepathic_llm(model=model, url=url, system_prompt="Strict JSON classifier.", user_prompt=prompt, use_local_edge=True)
import json
data = json.loads(res)
situ_str = data.get("situation", "")
if situ_str == "OBSTACLE_LOCKED_SCREEN":
logger.info("🧠 [Smart Perceive] SystemUI definitively classified as: LOCKED_SCREEN.")
return SituationType.OBSTACLE_LOCKED_SCREEN
elif situ_str == "OBSTACLE_SYSTEM":
logger.info("🧠 [Smart Perceive] SystemUI definitively classified as: SYSTEM_DIALOG.")
return SituationType.OBSTACLE_SYSTEM
else:
logger.info("🧠 [Smart Perceive] SystemUI classified as: FOREIGN_APP / NOTIFICATION.")
return SituationType.OBSTACLE_FOREIGN_APP
except Exception as e:
logger.warning(f"⚠️ [Smart Perceive] LLM Classification failed ({e}). Defaulting to FOREIGN_APP.")
return SituationType.OBSTACLE_FOREIGN_APP
# If only systemui is present, we might be on a lock screen, notification shade, or recent apps
lock_markers = ['keyguard', 'lock_icon', 'emergency_call_button', 'kg_']
if any(m in xml_lower for m in lock_markers):
logger.info("📱 [SAE Perceive] Lock screen markers detected in SystemUI.")
return SituationType.OBSTACLE_LOCKED_SCREEN
return SituationType.OBSTACLE_FOREIGN_APP
# ── Modal/Obstacle Detection (structural, not ID-based) ──
# Instead of checking specific IDs, we check STRUCTURAL patterns:

View File

@@ -58,7 +58,42 @@ class TelepathicEngine:
cls._instance = None
cls._last_click_context = None
def wipe(self):
"""Perform a full 'Blank Start' wipe of all AI caches and persistent memories."""
logger.warning("🔥 [TelepathicEngine] Wiping all AI caches and persistent records for BLANK START.")
# 1. Clear JSON files
for f in [MEMORY_FILE, BLACKLIST_FILE]:
if os.path.exists(f):
try:
os.remove(f)
logger.info(f"🗑️ Deleted persistent file: {f}")
except Exception as e:
logger.error(f"❌ Failed to delete {f}: {e}")
# 2. Clear Qdrant collections
try:
self.embedding_helper.client.delete_collection("telepathic_engine_cache")
logger.info("🗑️ Wiped Qdrant collection: telepathic_engine_cache")
except Exception as e:
logger.warning(f"⚠️ Could not wipe Qdrant collection (likely doesn't exist): {e}")
try:
if hasattr(self, 'ui_memory') and self.ui_memory and self.ui_memory.is_connected:
self.ui_memory.client.delete_collection(self.ui_memory.collection_name)
logger.info(f"🗑️ Wiped Qdrant collection: {self.ui_memory.collection_name}")
except Exception as e:
logger.warning(f"⚠️ Could not wipe UIMemoryDB collection: {e}")
# 3. Clear In-memory state
self._memory = {}
self._blacklist = {}
self._embedding_cache = {}
self._intent_cache = {}
def __init__(self):
from GramAddict.core.qdrant_memory import UIMemoryDB
self.ui_memory = UIMemoryDB()
self.embedding_helper = QdrantBase("telepathic_engine_cache")
self._embedding_cache: Dict[str, list] = {}
self._intent_cache: Dict[str, list] = {}
@@ -526,9 +561,15 @@ class TelepathicEngine:
score *= 0.3 # Heavy penalty
# Thresholding:
# - Short intents (1-2 words like 'tap home'): Require at least 50% hit (0.45)
# - Navigation intents: Require 100% exact match to avoid feed-cross-talk
# - Short intents (1-2 words): Require at least 50% hit (0.45)
# - Longer intents: Require 75% to avoid false matches on noisy screens.
threshold = 0.45 if len(intent_words) <= 2 else 0.75
is_nav_intent = any(k in intent_lower for k in ["tab", "navigation", "reels tab", "profile tab", "home tab", "explore tab", "message tab"])
if is_nav_intent:
threshold = 1.0
else:
threshold = 0.45 if len(intent_words) <= 2 else 0.75
if score >= threshold:
scored.append((node, score))
@@ -1015,18 +1056,21 @@ class TelepathicEngine:
clean_json = extract_json(resp_dict["response"])
if clean_json:
data = json.loads(clean_json)
idx = data.get("best_index")
if idx is not None and 0 <= idx < len(grid_nodes):
chosen = grid_nodes[idx]
logger.info(f"✅ [Vision Match] Cell {idx} chosen: {data.get('reason')}", extra={"color": f"{Fore.GREEN}"})
self._track_click(f"Visual Grid Selection ({idx})", chosen)
return {
"x": chosen["x"],
"y": chosen["y"],
"score": 0.99,
"semantic": f"Visual match {idx}: {data.get('reason')}",
"source": "vlm_grid"
}
if isinstance(data, list) and len(data) > 0:
data = data[0]
if isinstance(data, dict):
idx = data.get("best_index")
if idx is not None and 0 <= idx < len(grid_nodes):
chosen = grid_nodes[idx]
logger.info(f"✅ [Vision Match] Cell {idx} chosen: {data.get('reason')}", extra={"color": f"{Fore.GREEN}"})
self._track_click(f"Visual Grid Selection ({idx})", chosen)
return {
"x": chosen["x"],
"y": chosen["y"],
"score": 0.99,
"semantic": f"Visual match {idx}: {data.get('reason')}",
"source": "vlm_grid"
}
except Exception as e:
logger.error(f"👁️ [Vision Core] Grid evaluation failed: {e}")
@@ -1143,11 +1187,38 @@ class TelepathicEngine:
def _core_navigation_fast_path(self, intent_description: str, viable_nodes: list) -> Optional[dict]:
"""
[Phase 2] Hardened Fast Path with Qdrant Self-Learning.
Absolutely deterministic resource-ID targeting for core application navigation
(like direct messages or post usernames) to prevent VLM hallucination.
(like direct messages or post usernames). We query Qdrant first. If empty,
we seed it with legacy fallbacks.
"""
low_intent = intent_description.lower()
low_intent = intent_description.lower().strip()
# 0. Query Qdrant Memory first!
mem = self.ui_memory.retrieve_memory(intent_description, "", similarity_threshold=0.9)
if mem and isinstance(mem, dict):
if mem.get("action") == "tap" and mem.get("resource_id"):
learned_res_id = mem.get("resource_id")
for n in viable_nodes:
if learned_res_id in n.get("resource_id", "").lower():
logger.info(f"🧠 [TelepathicEngine] Fast-Path resolved '{intent_description}' via Qdrant -> '{learned_res_id}'", extra={"color": f"\\033[36m"})
self._track_click(intent_description, n)
return {
"x": n["x"],
"y": n["y"],
"score": 1.0,
"semantic": n["semantic_string"],
"source": "qdrant_nav"
}
# 0.5 Enforce Bootstrapper Lifecycle
# If Qdrant memory is sufficiently populated, ignore hardcoded seeds to enforce 100% self-learning
mem_size = self.ui_memory.get_collection_size()
if isinstance(mem_size, int) and mem_size > 25:
logger.debug(f"🌱 [Bootstrapper] Skipping hardcoded seeds for '{intent_description}' (Qdrant memory populated).")
return None
# 1. Post Username (Feed Profile)
if low_intent in ["tap_post_username", "tap post username"]:
for n in viable_nodes:
@@ -1195,6 +1266,8 @@ class TelepathicEngine:
"tap explore tab": "search_tab",
"tap_reels_tab": "clips_tab",
"tap reels tab": "clips_tab",
"tap_messages_tab": "direct_tab",
"tap messages tab": "direct_tab",
"tap_profile_tab": "profile_tab",
"tap profile tab": "profile_tab"
}
@@ -1202,7 +1275,15 @@ class TelepathicEngine:
target_res_id = tab_mappings[low_intent]
for n in viable_nodes:
if target_res_id in n.get("resource_id", "").lower():
logger.info(f" [Core Nav Fast Path] Found explicit Main Tab mapping for '{intent_description}' -> '{target_res_id}'")
logger.warning(f"🌱 [TelepathicEngine] Seeding Qdrant Memory with legacy fast-path: '{intent_description}' -> '{target_res_id}'")
self.ui_memory.store_memory(
intent_description,
"",
{
"resource_id": target_res_id,
"action": "tap"
}
)
self._track_click(intent_description, n)
return {
"x": n["x"],