feat(navigation): Implement 100% autonomous Blank Start architecture; purge heuristics
This commit is contained in:
@@ -76,7 +76,7 @@ def start_bot(**kwargs):
|
||||
username = getattr(configs.args, "username", "") or "unknown_user"
|
||||
|
||||
# Parse persona interests from config (comma-separated string → list)
|
||||
persona_raw = getattr(configs.args, "ai_target_audience", getattr(configs.args, "persona_interests", ""))
|
||||
persona_raw = getattr(configs.args, "ai_target_audience", getattr(configs.args, "persona_interests", getattr(configs.args, "target_audience", "")))
|
||||
persona_interests = [p.strip() for p in persona_raw.split(",") if p.strip()] if persona_raw else []
|
||||
|
||||
dopamine = DopamineEngine()
|
||||
@@ -96,7 +96,20 @@ def start_bot(**kwargs):
|
||||
darwin = DarwinEngine(username)
|
||||
|
||||
from GramAddict.core.telepathic_engine import TelepathicEngine
|
||||
telepathic = TelepathicEngine()
|
||||
telepathic = TelepathicEngine.get_instance()
|
||||
|
||||
# ── Stage 0: Blank Start (Scorched Earth) ──
|
||||
if getattr(configs.args, "blank_start", False):
|
||||
logger.warning(f"⚠️ [Blank Start] Wiping ALL persistent AI memories for '{username}'...")
|
||||
telepathic.wipe()
|
||||
# Wipe navigation paths too
|
||||
try:
|
||||
from GramAddict.core.goap import PathMemory
|
||||
path_mem = PathMemory(username)
|
||||
path_mem.wipe()
|
||||
logger.info("🗑️ Wiped PathMemory collection.")
|
||||
except Exception as e:
|
||||
logger.warning(f"⚠️ Failed to wipe PathMemory: {e}")
|
||||
|
||||
cognitive_stack = {
|
||||
"active_inference": active_inference,
|
||||
@@ -196,22 +209,23 @@ def start_bot(**kwargs):
|
||||
|
||||
# Tap first grid post to learn from actual captions
|
||||
if nav_graph.do("tap first image post in profile grid"):
|
||||
logger.info("📸 [Identity Boot] Reading recent posts to analyze actual content vibe...", extra={"color": f"{Fore.CYAN}"})
|
||||
sleep(2.0)
|
||||
for _ in range(3):
|
||||
post_xml = device.dump_hierarchy()
|
||||
if isinstance(post_xml, str):
|
||||
post_data = _extract_post_content(post_xml)
|
||||
if post_data.get("caption"):
|
||||
raw_bio_text.append(post_data["caption"])
|
||||
elif post_data.get("description"):
|
||||
raw_bio_text.append(post_data["description"])
|
||||
post_loaded = _wait_for_post_loaded(device, timeout=5)
|
||||
if post_loaded:
|
||||
logger.info("📸 [Identity Boot] Reading recent posts to analyze actual content vibe...", extra={"color": f"{Fore.CYAN}"})
|
||||
for _ in range(3):
|
||||
post_xml = device.dump_hierarchy()
|
||||
if isinstance(post_xml, str):
|
||||
post_data = _extract_post_content(post_xml)
|
||||
if post_data.get("caption"):
|
||||
raw_bio_text.append(post_data["caption"])
|
||||
elif post_data.get("description"):
|
||||
raw_bio_text.append(post_data["description"])
|
||||
|
||||
_humanized_scroll(device, is_skip=False)
|
||||
sleep(2.0)
|
||||
_humanized_scroll(device, is_skip=False)
|
||||
sleep(2.0)
|
||||
|
||||
device.press("back")
|
||||
sleep(1.5)
|
||||
device.press("back")
|
||||
sleep(1.5)
|
||||
|
||||
# Deduplicate while preserving order
|
||||
unique_texts = list(dict.fromkeys(raw_bio_text))
|
||||
@@ -294,11 +308,17 @@ def start_bot(**kwargs):
|
||||
nav_graph.do("tap first image in explore grid")
|
||||
|
||||
# Wait for post to actually load (poll for feed markers)
|
||||
_wait_for_post_loaded(device, timeout=5)
|
||||
post_loaded = _wait_for_post_loaded(device, nav_graph=nav_graph, timeout=5)
|
||||
if not post_loaded:
|
||||
logger.warning("❌ Post failed to open from grid. Retrying next loop.")
|
||||
continue
|
||||
elif current_target == "StoriesFeed":
|
||||
logger.info("📱 Locating story tray on HomeFeed...")
|
||||
nav_graph.do("tap story ring avatar")
|
||||
_wait_for_post_loaded(device, timeout=5)
|
||||
post_loaded = _wait_for_story_loaded(device, timeout=5)
|
||||
if not post_loaded:
|
||||
logger.warning("❌ Stories failed to open from HomeFeed. Retrying next loop.")
|
||||
continue
|
||||
|
||||
if current_target == "StoriesFeed":
|
||||
result = _run_zero_latency_stories_loop(device, configs, session_state, cognitive_stack)
|
||||
@@ -361,9 +381,10 @@ FEED_MARKERS = [
|
||||
|
||||
|
||||
|
||||
def _wait_for_post_loaded(device, timeout=5):
|
||||
def _wait_for_post_loaded(device, timeout=5, nav_graph=None):
|
||||
"""Polls the UI hierarchy until feed markers appear, confirming a post is on screen."""
|
||||
start = time.time()
|
||||
xml = ""
|
||||
while time.time() - start < timeout:
|
||||
try:
|
||||
xml = device.dump_hierarchy()
|
||||
@@ -373,8 +394,56 @@ def _wait_for_post_loaded(device, timeout=5):
|
||||
except Exception:
|
||||
pass
|
||||
sleep(0.5)
|
||||
logger.warning("⚠️ Post did not load within timeout. Proceeding anyway.")
|
||||
|
||||
logger.warning("⚠️ Post did not load within timeout. Attempting Adaptive Snap.")
|
||||
dump_ui_state(device, "post_load_timeout", {"timeout_sec": timeout})
|
||||
|
||||
try:
|
||||
xml_lower = xml.lower()
|
||||
# 1. Trapped in a Story or Reel viewer? Press back.
|
||||
if "reel_viewer_root" in xml_lower or "clips_viewer" in xml_lower:
|
||||
logger.warning("🧗 [Adaptive Snap] Trapped in Story/Reel viewer. Pressing BACK.")
|
||||
device.press("back")
|
||||
sleep(1.5)
|
||||
# Give it one more chance to load the feed
|
||||
xml = device.dump_hierarchy()
|
||||
if any(marker in xml for marker in FEED_MARKERS):
|
||||
logger.info("✅ Recovered to Feed.")
|
||||
return True
|
||||
|
||||
# 2. Trapped in Profile?
|
||||
if "profile_header" in xml_lower and "row_feed_photo_profile_name" not in xml_lower:
|
||||
logger.warning("🧗 [Adaptive Snap] Trapped in Profile. Pressing BACK.")
|
||||
device.press("back")
|
||||
sleep(1.5)
|
||||
|
||||
# 3. Stuck between posts (Feed markers not fully visible)? Try to align or wobble.
|
||||
# Fallback micro-wobble
|
||||
info = device.get_info()
|
||||
w, h = info.get("displayWidth", 1080), info.get("displayHeight", 2400)
|
||||
logger.warning("🧗 [Adaptive Snap] Wobbling to force render.")
|
||||
device.swipe(int(w/2), int(h/2), int(w/2), int(h/2) - 100, 100)
|
||||
sleep(0.5)
|
||||
device.swipe(int(w/2), int(h/2) - 100, int(w/2), int(h/2), 100)
|
||||
except Exception as e:
|
||||
logger.error(f"❌ [Adaptive Snap] Failed: {e}")
|
||||
|
||||
return False
|
||||
|
||||
def _wait_for_story_loaded(device, timeout=5):
|
||||
"""Polls the UI hierarchy until story markers appear, confirming a story is on screen."""
|
||||
start = time.time()
|
||||
while time.time() - start < timeout:
|
||||
try:
|
||||
xml_lower = device.dump_hierarchy().lower()
|
||||
if "reel_viewer_root" in xml_lower or "story_viewer" in xml_lower:
|
||||
logger.debug("📱 Story loaded successfully.")
|
||||
return True
|
||||
except Exception:
|
||||
pass
|
||||
sleep(0.5)
|
||||
|
||||
logger.warning("⚠️ Story did not load within timeout.")
|
||||
return False
|
||||
|
||||
def _humanized_scroll(device, is_skip=False, resonance_score=None):
|
||||
@@ -651,6 +720,11 @@ def _interact_with_profile(device, configs, username, session_state, sleep_mod,
|
||||
xml_dump = device.dump_hierarchy()
|
||||
has_story = "reel_ring" in xml_dump or "'s unseen story" in xml_dump.lower() or "has a new story" in xml_dump.lower() or "story von" in xml_dump.lower()
|
||||
if has_story and nav_graph.do("tap story ring avatar"):
|
||||
post_loaded = _wait_for_story_loaded(device, timeout=5)
|
||||
if not post_loaded:
|
||||
logger.warning(f"❌ Story failed to open for @{username}.")
|
||||
return
|
||||
|
||||
logger.info(f"📸 [Story] Viewing @{username}'s story ({count} times)...")
|
||||
for i in range(count):
|
||||
sleep(random.uniform(2.0, 5.0) * sleep_mod)
|
||||
@@ -700,6 +774,11 @@ def _interact_with_profile(device, configs, username, session_state, sleep_mod,
|
||||
nav_graph = QNavGraph(device)
|
||||
|
||||
if nav_graph.do("tap first image post in profile grid"):
|
||||
post_loaded = _wait_for_post_loaded(device, timeout=5)
|
||||
if not post_loaded:
|
||||
logger.warning(f"❌ Post failed to open from profile grid of @{username}.")
|
||||
return
|
||||
|
||||
logger.info(f"❤️ [Deep Interaction] Opening grid to drop {count} likes on @{username}...")
|
||||
|
||||
for i in range(count):
|
||||
@@ -1308,7 +1387,7 @@ def _run_zero_latency_feed_loop(device, zero_engine, nav_graph, configs, session
|
||||
# Return to feed
|
||||
logger.info("🔙 [Profile Learning] Returning to main feed.")
|
||||
device.press("back")
|
||||
_wait_for_post_loaded(device)
|
||||
_wait_for_post_loaded(device, nav_graph=nav_graph)
|
||||
sleep(random.uniform(1.0, 1.5) * sleep_mod)
|
||||
|
||||
rnd_interact = random.random()
|
||||
@@ -1687,7 +1766,8 @@ def _run_zero_latency_feed_loop(device, zero_engine, nav_graph, configs, session
|
||||
|
||||
# ── Active Inference: Evaluate prediction (after action) ──
|
||||
if ai:
|
||||
_wait_for_post_loaded(device, timeout=3)
|
||||
# Wait for content to settle
|
||||
_wait_for_post_loaded(device, timeout=3, nav_graph=nav_graph)
|
||||
post_action_xml = device.dump_hierarchy()
|
||||
ai.evaluate_prediction(post_action_xml)
|
||||
|
||||
|
||||
@@ -141,6 +141,7 @@ class Config:
|
||||
self.parser.add_argument("--time-delta-session", help="Time delta between sessions", default=None)
|
||||
self.parser.add_argument("--restart-atx-agent", action="store_true", help="Restart atx agent")
|
||||
self.parser.add_argument("--allow-untested-ig-version", action="store_true", help="Allow untested IG version")
|
||||
self.parser.add_argument("--blank-start", action="store_true", help="Wipe all learned navigation and telepathic memories on boot to start 100% blank.")
|
||||
|
||||
# Interaction settings
|
||||
self.parser.add_argument("--likes-count", help="Likes count", default="2-3")
|
||||
@@ -174,6 +175,7 @@ class Config:
|
||||
# Persona & Resonance (drives ALL content evaluation and interaction decisions)
|
||||
self.parser.add_argument("--persona-interests", help="Comma-separated niche interests for content matching", default="")
|
||||
self.parser.add_argument("--ai-target-audience", help="Target audience used interchangeably with persona interests", default="")
|
||||
self.parser.add_argument("--target-audience", help="Target audience used interchangeably with persona interests", default="")
|
||||
self.parser.add_argument("--interact-percentage", help="Overall interaction probability percentage", default="80")
|
||||
self.parser.add_argument("--comment-percentage", help="Comment probability percentage", default="0")
|
||||
self.parser.add_argument("--follow-percentage", help="Follow probability percentage", default="0")
|
||||
|
||||
@@ -44,6 +44,10 @@ def get_device_info(device):
|
||||
logger.debug(f"Device Info: {info.get('productName')} | SDK: {info.get('sdkInt')}")
|
||||
|
||||
class DeviceFacade:
|
||||
deviceV2 = None
|
||||
app_id = None
|
||||
device_id = None
|
||||
|
||||
def __init__(self, device_id, app_id, args):
|
||||
self.device_id = device_id
|
||||
self.app_id = app_id
|
||||
@@ -94,6 +98,11 @@ class DeviceFacade:
|
||||
self.deviceV2.press("home")
|
||||
sleep(1)
|
||||
|
||||
@adb_retry()
|
||||
def unlock(self):
|
||||
self.deviceV2.unlock()
|
||||
|
||||
|
||||
@property
|
||||
def info(self):
|
||||
return self.deviceV2.info
|
||||
|
||||
@@ -22,6 +22,7 @@ from typing import Optional, List, Dict, Any
|
||||
from enum import Enum
|
||||
|
||||
from GramAddict.core.utils import random_sleep
|
||||
from GramAddict.core.qdrant_memory import QdrantBase
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -154,7 +155,7 @@ class ScreenIdentity:
|
||||
|
||||
# ── Extract available actions from clickable elements ──
|
||||
available_actions = self._extract_available_actions(
|
||||
clickable_elements, resource_ids, content_descs, screen_type
|
||||
clickable_elements, resource_ids, content_descs, texts, screen_type
|
||||
)
|
||||
|
||||
# ── Extract context ──
|
||||
@@ -181,6 +182,9 @@ class ScreenIdentity:
|
||||
pass
|
||||
|
||||
# Priority 2: Structural Heuristics (Instant, for core tabs)
|
||||
if 'unified_follow_list_tab_layout' in ids or 'follow_list_container' in ids:
|
||||
return ScreenType.FOLLOW_LIST
|
||||
|
||||
if selected_tab == 'feed_tab': return ScreenType.HOME_FEED
|
||||
if selected_tab == 'clips_tab': return ScreenType.REELS_FEED
|
||||
if selected_tab == 'search_tab': return ScreenType.EXPLORE_GRID
|
||||
@@ -223,7 +227,7 @@ class ScreenIdentity:
|
||||
|
||||
return ScreenType.UNKNOWN
|
||||
|
||||
def _extract_available_actions(self, clickable_elements, resource_ids, content_descs, screen_type):
|
||||
def _extract_available_actions(self, clickable_elements, resource_ids, content_descs, texts, screen_type):
|
||||
"""Discover what actions are possible on this screen."""
|
||||
actions = []
|
||||
|
||||
@@ -241,6 +245,7 @@ class ScreenIdentity:
|
||||
|
||||
# Screen-specific actions
|
||||
desc_lower = ' '.join(content_descs).lower()
|
||||
text_lower = ' '.join(texts).lower()
|
||||
|
||||
if 'like' in desc_lower:
|
||||
actions.append('tap like button')
|
||||
@@ -254,6 +259,12 @@ class ScreenIdentity:
|
||||
actions.append('tap back button')
|
||||
if any('follow' in e.get('text', '').lower() for e in clickable_elements):
|
||||
actions.append('tap follow button')
|
||||
|
||||
if screen_type == ScreenType.OWN_PROFILE or screen_type == ScreenType.OTHER_PROFILE:
|
||||
if 'message' in desc_lower or 'nachricht' in desc_lower:
|
||||
actions.append('tap message button')
|
||||
if 'following' in desc_lower or 'abonniert' in desc_lower or 'following' in text_lower:
|
||||
actions.append('tap following list')
|
||||
|
||||
# Grid items
|
||||
if screen_type == ScreenType.EXPLORE_GRID:
|
||||
@@ -321,13 +332,23 @@ class PathMemory:
|
||||
Enables instant recall for known goals.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
def __init__(self, username: str = ""):
|
||||
self.username = username
|
||||
try:
|
||||
from GramAddict.core.qdrant_memory import QdrantBase
|
||||
self._db = QdrantBase("goap_paths_v1", vector_size=768)
|
||||
suffix = f"_{username}" if username else ""
|
||||
self._db = QdrantBase(f"goap_paths_v1{suffix}", vector_size=768)
|
||||
except Exception:
|
||||
self._db = None
|
||||
|
||||
def wipe(self):
|
||||
"""Wipe all learned navigation paths from Qdrant."""
|
||||
if self._db and self._db.is_connected:
|
||||
try:
|
||||
self._db.client.delete_collection(self._db.collection_name)
|
||||
logger.info(f"🗑️ [PathMemory] Wiped Qdrant collection: {self._db.collection_name}")
|
||||
except Exception as e:
|
||||
logger.warning(f"⚠️ [PathMemory] Could not wipe collection: {e}")
|
||||
|
||||
def recall_path(self, goal: str, current_screen_type: str) -> Optional[List[Dict]]:
|
||||
"""
|
||||
Recall a previously successful path for this goal from this screen type.
|
||||
@@ -342,9 +363,18 @@ class PathMemory:
|
||||
return None
|
||||
|
||||
try:
|
||||
from qdrant_client.models import Filter, FieldCondition, MatchValue
|
||||
results = self._db.client.query_points(
|
||||
collection_name=self._db.collection_name,
|
||||
query=vec,
|
||||
query_filter=Filter(
|
||||
must=[
|
||||
FieldCondition(
|
||||
key="start_screen",
|
||||
match=MatchValue(value=current_screen_type)
|
||||
)
|
||||
]
|
||||
),
|
||||
limit=3,
|
||||
score_threshold=0.85,
|
||||
).points
|
||||
@@ -373,7 +403,7 @@ class PathMemory:
|
||||
if not vec:
|
||||
return
|
||||
|
||||
seed = f"{goal}|{start_screen}|{len(steps)}|{success}"
|
||||
seed = f"{goal}|{start_screen}"
|
||||
payload = {
|
||||
"goal": goal,
|
||||
"start_screen": start_screen,
|
||||
@@ -390,6 +420,22 @@ class PathMemory:
|
||||
log_success=f"🧠 [GOAP Learn] {outcome} Path for '{goal}': {len(steps)} steps from {start_screen}"
|
||||
)
|
||||
|
||||
def forget_path(self, goal: str, start_screen: str):
|
||||
"""Remove a cached path to force re-discovery."""
|
||||
if not self._db or not self._db.is_connected:
|
||||
return
|
||||
|
||||
seed = f"{goal}|{start_screen}"
|
||||
try:
|
||||
from qdrant_client import models
|
||||
point_id = self._db._get_id(seed)
|
||||
self._db.client.delete(
|
||||
collection_name=self._db.collection_name,
|
||||
points_selector=models.PointIdsList(points=[point_id])
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug(f"Failed to forget path: {e}")
|
||||
|
||||
|
||||
# ══════════════════════════════════════════════════════
|
||||
# 3. GOAL PLANNER — "What should I do next?"
|
||||
@@ -405,44 +451,204 @@ class GoalPlanner:
|
||||
3. LLM planning (slow, for truly unknown situations)
|
||||
"""
|
||||
|
||||
# ── Navigation knowledge: Screen → Tab mapping ──
|
||||
# This is NOT hardcoded navigation. It's the bot's understanding of
|
||||
# WHERE things live (like knowing a GPS needs street addresses).
|
||||
# The bot can discover these itself, but we seed them for speed.
|
||||
SCREEN_TAB_MAP = {
|
||||
ScreenType.HOME_FEED: 'feed_tab',
|
||||
ScreenType.EXPLORE_GRID: 'search_tab',
|
||||
ScreenType.REELS_FEED: 'clips_tab',
|
||||
ScreenType.OWN_PROFILE: 'profile_tab',
|
||||
ScreenType.DM_INBOX: 'direct_tab',
|
||||
}
|
||||
class NavigationKnowledge:
|
||||
"""
|
||||
Manages the bot's learned understanding of the Instagram UI.
|
||||
Discovered dynamically through exploration and success.
|
||||
"""
|
||||
def __init__(self, username: str):
|
||||
self.username = username
|
||||
try:
|
||||
self._db = QdrantBase("navigation_knowledge", vector_size=768)
|
||||
except Exception:
|
||||
self._db = None
|
||||
|
||||
# In-memory cache for rapidly avoiding traps during exploration
|
||||
# In-memory cache for rapidly avoiding traps during exploration
|
||||
self._learned_screen_mappings = {}
|
||||
|
||||
# ── Goal → Required screen type mapping ──
|
||||
# "To achieve X, I first need to be on screen Y"
|
||||
GOAL_SCREEN_REQUIREMENTS = {
|
||||
'like a post': [ScreenType.HOME_FEED, ScreenType.POST_DETAIL],
|
||||
'like this post': [ScreenType.POST_DETAIL, ScreenType.HOME_FEED],
|
||||
'follow this user': [ScreenType.OTHER_PROFILE],
|
||||
'open explore': [ScreenType.EXPLORE_GRID],
|
||||
'open explore feed': [ScreenType.EXPLORE_GRID],
|
||||
'open home feed': [ScreenType.HOME_FEED],
|
||||
'open reels': [ScreenType.REELS_FEED],
|
||||
'open profile': [ScreenType.OWN_PROFILE],
|
||||
'learn own profile': [ScreenType.OWN_PROFILE],
|
||||
'open messages': [ScreenType.DM_INBOX],
|
||||
'tap first grid item': [ScreenType.EXPLORE_GRID],
|
||||
'view a post from explore': [ScreenType.EXPLORE_GRID],
|
||||
'visit profile': [ScreenType.OTHER_PROFILE],
|
||||
'go back': [ScreenType.UNKNOWN],
|
||||
}
|
||||
def wipe(self):
|
||||
"""Wipe all learned knowledge from Qdrant."""
|
||||
if self._db and self._db.is_connected:
|
||||
try:
|
||||
self._db.client.delete_collection(self._db.collection_name)
|
||||
logger.info(f"🗑️ [NavigationKnowledge] Wiped Qdrant collection: {self._db.collection_name}")
|
||||
except Exception as e:
|
||||
logger.warning(f"⚠️ [NavigationKnowledge] Could not wipe knowledge: {e}")
|
||||
|
||||
def plan_next_step(self, goal: str, screen: Dict[str, Any]) -> Optional[str]:
|
||||
"""
|
||||
Plans the NEXT single action to take toward the goal.
|
||||
def update_username(self, username: str):
|
||||
"""Update username and reconnect DB if needed."""
|
||||
if self.username != username:
|
||||
self.username = username
|
||||
try:
|
||||
self._db = QdrantBase("navigation_knowledge", vector_size=768)
|
||||
except Exception:
|
||||
self._db = None
|
||||
|
||||
def get_requirements(self, goal: str) -> List[ScreenType]:
|
||||
"""Get required screens for a goal. Returns known requirements or empty list."""
|
||||
if not self._db or not self._db.is_connected:
|
||||
return []
|
||||
|
||||
try:
|
||||
from qdrant_client.models import Filter, FieldCondition, MatchValue
|
||||
results = self._db.client.scroll(
|
||||
collection_name=self._db.collection_name,
|
||||
scroll_filter=Filter(
|
||||
must=[
|
||||
FieldCondition(
|
||||
key="goal",
|
||||
match=MatchValue(value=goal)
|
||||
)
|
||||
]
|
||||
),
|
||||
limit=1
|
||||
)[0]
|
||||
if results:
|
||||
screen_name = results[0].payload.get("required_screen")
|
||||
logger.debug(f"🧠 [Nav Knowledge] Found requirement for '{goal}': {screen_name}")
|
||||
if screen_name:
|
||||
return [ScreenType[screen_name]]
|
||||
except Exception as e:
|
||||
logger.warning(f"⚠️ [Nav Knowledge] Search error: {e}")
|
||||
return []
|
||||
|
||||
def learn_goal_requirement(self, goal: str, screen_type: ScreenType):
|
||||
"""Learn that achieving 'goal' lands us on 'screen_type'."""
|
||||
if not self._db or not self._db.is_connected:
|
||||
logger.warning("⚠️ [Nav Knowledge] Cannot learn: DB not connected")
|
||||
return
|
||||
|
||||
seed = f"req_{goal}"
|
||||
vec = self._db._get_embedding(f"goal_requirement: {goal}")
|
||||
payload = {
|
||||
"goal": goal,
|
||||
"required_screen": screen_type.name,
|
||||
"timestamp": time.time()
|
||||
}
|
||||
self._db.upsert_point(seed, payload, vector=vec)
|
||||
logger.info(f"🧠 [Nav Knowledge] Learned: '{goal}' → {screen_type.name}")
|
||||
|
||||
def get_action_for_screen(self, target_screen: ScreenType) -> Optional[str]:
|
||||
"""Find which action leads to this screen."""
|
||||
for action, screen in self._learned_screen_mappings.items():
|
||||
if screen == target_screen:
|
||||
return action
|
||||
|
||||
if not self._db or not self._db.is_connected:
|
||||
return None
|
||||
|
||||
try:
|
||||
from qdrant_client.models import Filter, FieldCondition, MatchValue
|
||||
results = self._db.client.scroll(
|
||||
collection_name=self._db.collection_name,
|
||||
scroll_filter=Filter(
|
||||
must=[
|
||||
FieldCondition(
|
||||
key="result_screen",
|
||||
match=MatchValue(value=target_screen.name)
|
||||
)
|
||||
]
|
||||
),
|
||||
limit=1
|
||||
)[0]
|
||||
if results:
|
||||
return results[0].payload.get("action")
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
def get_screen_for_action(self, action: str) -> Optional[ScreenType]:
|
||||
"""Find where this action leads to to avoid looping traps."""
|
||||
if action in self._learned_screen_mappings:
|
||||
return self._learned_screen_mappings[action]
|
||||
|
||||
Returns a natural language action string like 'tap explore tab'
|
||||
or None if the goal is already achieved.
|
||||
"""
|
||||
if not self._db or not self._db.is_connected:
|
||||
return None
|
||||
|
||||
try:
|
||||
from qdrant_client.models import Filter, FieldCondition, MatchValue
|
||||
results = self._db.client.scroll(
|
||||
collection_name=self._db.collection_name,
|
||||
scroll_filter=Filter(
|
||||
must=[
|
||||
FieldCondition(
|
||||
key="action",
|
||||
match=MatchValue(value=action)
|
||||
)
|
||||
]
|
||||
),
|
||||
limit=1
|
||||
)[0]
|
||||
if results:
|
||||
screen_name = results[0].payload.get("result_screen")
|
||||
if screen_name:
|
||||
return ScreenType[screen_name]
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
def learn_screen_mapping(self, action: str, result_screen: ScreenType):
|
||||
"""Learn that taking 'action' leads to 'result_screen'."""
|
||||
if not self._db or not self._db.is_connected:
|
||||
return
|
||||
|
||||
seed = f"map_{action}"
|
||||
vec = self._db._get_embedding(f"screen_mapping: {result_screen.name}")
|
||||
payload = {
|
||||
"action": action,
|
||||
"result_screen": result_screen.name,
|
||||
"timestamp": time.time()
|
||||
}
|
||||
|
||||
self._learned_screen_mappings[action] = result_screen
|
||||
|
||||
self._db.upsert_point(seed, payload, vector=vec)
|
||||
logger.info(f"🧠 [Nav Knowledge] Learned Mapping: '{action}' → {result_screen.name}")
|
||||
|
||||
def get_screen_for_tab(self, tab_id: str) -> Optional[ScreenType]:
|
||||
"""Find where this tab leads to to avoid looping traps."""
|
||||
if tab_id in self._learned_screen_mappings:
|
||||
return self._learned_screen_mappings[tab_id]
|
||||
|
||||
if not self._db or not self._db.is_connected:
|
||||
return None
|
||||
|
||||
try:
|
||||
from qdrant_client.models import Filter, FieldCondition, MatchValue
|
||||
results = self._db.client.scroll(
|
||||
collection_name=self._db.collection_name,
|
||||
scroll_filter=Filter(
|
||||
must=[
|
||||
FieldCondition(
|
||||
key="tab_id",
|
||||
match=MatchValue(value=tab_id)
|
||||
)
|
||||
]
|
||||
),
|
||||
limit=1
|
||||
)[0]
|
||||
if results:
|
||||
s_name = results[0].payload.get("result_screen")
|
||||
if s_name:
|
||||
return ScreenType[s_name]
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
class GoalPlanner:
|
||||
"""
|
||||
Given a goal and current screen state, plans the next action.
|
||||
|
||||
Uses Dynamic Discovery to navigate without hardcoded maps.
|
||||
"""
|
||||
|
||||
def __init__(self, username: str):
|
||||
self.knowledge = NavigationKnowledge(username)
|
||||
|
||||
def plan_next_step(self, goal: str, screen: Dict[str, Any], explored_nav_actions: set = None) -> Optional[str]:
|
||||
"""Plans the NEXT single action to take toward the goal."""
|
||||
screen_type = screen['screen_type']
|
||||
available = screen.get('available_actions', [])
|
||||
context = screen.get('context', {})
|
||||
@@ -454,19 +660,16 @@ class GoalPlanner:
|
||||
return None
|
||||
|
||||
# ── 2. Am I on the right screen? If not, navigate there ──
|
||||
nav_action = self._plan_navigation(goal_lower, screen_type, available)
|
||||
selected_tab = screen.get('selected_tab')
|
||||
nav_action = self._plan_navigation(goal_lower, screen_type, available, selected_tab, explored_nav_actions)
|
||||
if nav_action:
|
||||
return nav_action
|
||||
|
||||
# ── 3. I'm on the right screen — execute the goal action ──
|
||||
# ── 3. Execute the goal action ──
|
||||
goal_action = self._plan_goal_action(goal_lower, screen_type, available, context)
|
||||
if goal_action:
|
||||
return goal_action
|
||||
|
||||
# ── 4. Fallback: try scroll or back ──
|
||||
if 'scroll down' in available:
|
||||
return 'scroll down'
|
||||
|
||||
return 'press back'
|
||||
|
||||
def _is_goal_achieved(self, goal: str, screen_type: ScreenType, context: dict) -> bool:
|
||||
@@ -485,53 +688,69 @@ class GoalPlanner:
|
||||
return True
|
||||
if 'open messages' in goal and screen_type == ScreenType.DM_INBOX:
|
||||
return True
|
||||
if ('following list' in goal or 'followers list' in goal) and screen_type == ScreenType.FOLLOW_LIST:
|
||||
return True
|
||||
if 'open explore' in goal and screen_type == ScreenType.EXPLORE_GRID:
|
||||
return True
|
||||
if 'view profile' in goal and (screen_type == ScreenType.OWN_PROFILE or screen_type == ScreenType.OTHER_PROFILE):
|
||||
return True
|
||||
return False
|
||||
|
||||
def _plan_navigation(self, goal: str, screen_type: ScreenType, available: List[str]) -> Optional[str]:
|
||||
def _plan_navigation(self, goal: str, screen_type: ScreenType, available: List[str], selected_tab: Optional[str] = None, explored_nav_actions: set = None) -> Optional[str]:
|
||||
"""If we're on the wrong screen, figure out how to navigate."""
|
||||
|
||||
# Find what screen(s) we need for this goal
|
||||
required_screens = None
|
||||
for goal_pattern, screens in self.GOAL_SCREEN_REQUIREMENTS.items():
|
||||
if goal_pattern in goal:
|
||||
required_screens = screens
|
||||
break
|
||||
# 1. Get required screens for this goal from knowledge
|
||||
required_screens = self.knowledge.get_requirements(goal)
|
||||
|
||||
# 2. Blank Start Discovery (if knowledge is empty)
|
||||
if not required_screens:
|
||||
return None
|
||||
logger.info(f"🧠 [Nav Discovery] No known requirements for '{goal}'. Will attempt autonomous discovery.")
|
||||
# We don't know the required screen or path. Let the TelepathicEngine figure out
|
||||
# what button to press based on the pure goal text!
|
||||
if explored_nav_actions and goal in explored_nav_actions:
|
||||
logger.info(f"🛑 [Nav Discovery] Autonomous intent '{goal}' already tried and failed/trapped. Yielding to back-tracking.")
|
||||
pass # Don't return goal again, let it press back if possible
|
||||
else:
|
||||
return goal
|
||||
|
||||
# If we're already on an acceptable screen, no navigation needed
|
||||
# 3. If we're already on an acceptable screen, no navigation needed
|
||||
if screen_type in required_screens:
|
||||
return None
|
||||
|
||||
# Find the tab we need to tap
|
||||
# 4. Find the action we need to take
|
||||
for target_screen in required_screens:
|
||||
target_tab = self.SCREEN_TAB_MAP.get(target_screen)
|
||||
if target_tab:
|
||||
# Map tab to action
|
||||
tab_actions = {
|
||||
'feed_tab': 'tap home tab',
|
||||
'search_tab': 'tap explore tab',
|
||||
'clips_tab': 'tap reels tab',
|
||||
'profile_tab': 'tap profile tab',
|
||||
'direct_tab': 'tap messages tab',
|
||||
}
|
||||
action = tab_actions.get(target_tab)
|
||||
if action and action in available:
|
||||
logger.info(f"🧭 [GOAP Navigate] Need {target_screen.value} for '{goal}' → {action}")
|
||||
return action
|
||||
known_action = self.knowledge.get_action_for_screen(target_screen)
|
||||
|
||||
if not known_action:
|
||||
logger.info(f"🧭 [Nav Discovery] Don't know action to reach {target_screen.name}. Asking VLM...")
|
||||
|
||||
# Check ALL available actions if one linguistically aligns with the target screen
|
||||
# Or just let VLM figure it out by returning an intention.
|
||||
screen_friendly_name = target_screen.name.replace('_', ' ').lower()
|
||||
|
||||
# Semantic Heuristic Match on dynamically perceived actions
|
||||
goal_words = [w.rstrip('s') for w in screen_friendly_name.split() if len(w) > 3]
|
||||
|
||||
for action in available:
|
||||
if any(w in action.lower() for w in goal_words):
|
||||
# Verify we don't know this leads somewhere else
|
||||
known_target = self.knowledge.get_screen_for_action(action)
|
||||
if known_target and known_target != target_screen:
|
||||
continue # Trap prevention
|
||||
|
||||
logger.info(f"🎯 [Nav Discovery] Linguistic match on available action! '{action}' aligns with '{screen_friendly_name}'")
|
||||
return action
|
||||
|
||||
# If no perceived action matches, return intent for TelepathicEngine
|
||||
return f"navigate to {screen_friendly_name}"
|
||||
else:
|
||||
if known_action in available:
|
||||
logger.info(f"🧭 [Nav Knowledge] Navigating to {target_screen.name} via '{known_action}'")
|
||||
return known_action
|
||||
|
||||
# If no tab navigation works, try going back first
|
||||
# If no targeted navigation works, try going back first
|
||||
if 'press back' in available:
|
||||
logger.info("🧭 [GOAP Navigate] Can't reach required screen directly. Pressing back...")
|
||||
return 'press back'
|
||||
|
||||
# Heuristic Fallback: If we are on an UNKNOWN screen and have NO tab buttons visible,
|
||||
# we are likely in a deep view (like a DM thread or nested settings).
|
||||
# Suggesting 'press back' even if not explicitly found in available_actions
|
||||
# as a generic escape mechanism.
|
||||
if screen_type == ScreenType.UNKNOWN and not any(tab in available for tab in tab_actions.values()):
|
||||
logger.info("🧠 [GOAP Heuristic] Stuck on UNKNOWN screen with no tabs. Suggesting 'press back' fallback.")
|
||||
# Only press back if we aren't currently on the required screen (handled in step 3)
|
||||
return 'press back'
|
||||
|
||||
return None
|
||||
@@ -539,10 +758,14 @@ class GoalPlanner:
|
||||
def _plan_goal_action(self, goal: str, screen_type: ScreenType, available: List[str], context: dict) -> Optional[str]:
|
||||
"""Plan the specific action to achieve the goal on the current screen."""
|
||||
|
||||
import re
|
||||
if 'like' in goal and 'tap like button' in available:
|
||||
return 'tap like button'
|
||||
|
||||
if 'following list' in goal and 'tap following list' in available:
|
||||
return 'tap following list'
|
||||
|
||||
if 'follow' in goal and 'tap follow button' in available:
|
||||
if re.search(r'\bfollow\b', goal) and 'tap follow button' in available:
|
||||
return 'tap follow button'
|
||||
|
||||
if 'comment' in goal and 'tap comment button' in available:
|
||||
@@ -593,11 +816,13 @@ class GoalExecutor:
|
||||
|
||||
def __init__(self, device, bot_username: str = ""):
|
||||
self.device = device
|
||||
self.username = bot_username
|
||||
self.screen_id = ScreenIdentity(bot_username)
|
||||
self.planner = GoalPlanner()
|
||||
self.path_memory = PathMemory()
|
||||
self.planner = GoalPlanner(bot_username)
|
||||
self.path_memory = PathMemory(bot_username)
|
||||
self.max_steps = 15 # Safety: never execute more than 15 steps
|
||||
self._sae = None # Lazy-loaded, injectable for tests
|
||||
self.action_failures = {} # Tracking for failed actions in current goal session
|
||||
|
||||
def _get_sae(self):
|
||||
"""Get or create the SAE instance. Injectable for tests."""
|
||||
@@ -627,6 +852,7 @@ class GoalExecutor:
|
||||
max_steps = self.max_steps
|
||||
|
||||
logger.info(f"🎯 [GOAP] Pursuing goal: '{goal}'")
|
||||
self.action_failures.clear()
|
||||
|
||||
# ── Try recalled path first ──
|
||||
screen = self.perceive()
|
||||
@@ -642,10 +868,24 @@ class GoalExecutor:
|
||||
|
||||
# ── Live planning ──
|
||||
steps_taken = []
|
||||
last_action = None
|
||||
explored_nav_actions = set()
|
||||
for step_num in range(max_steps):
|
||||
# PERCEIVE
|
||||
screen = self.perceive()
|
||||
screen_type = screen['screen_type']
|
||||
|
||||
# ── Loop Prevention: Mask Failed Actions ──
|
||||
MAX_RETRIES = 2
|
||||
original_available = screen.get('available_actions', []).copy()
|
||||
masked_available = []
|
||||
for act in original_available:
|
||||
fail_count = self.action_failures.get(act, 0)
|
||||
if fail_count >= MAX_RETRIES:
|
||||
logger.warning(f"🚫 [GOAP] Masking action '{act}' due to {fail_count} consecutive failures to prevent loops.")
|
||||
else:
|
||||
masked_available.append(act)
|
||||
screen['available_actions'] = masked_available
|
||||
|
||||
logger.debug(
|
||||
f"📍 [GOAP Step {step_num + 1}] On: {screen_type.value} | "
|
||||
@@ -653,48 +893,64 @@ class GoalExecutor:
|
||||
)
|
||||
|
||||
# Handle obstacles
|
||||
if screen_type == ScreenType.FOREIGN_APP:
|
||||
logger.warning("🚨 [GOAP] Foreign app detected. Using SAE to recover...")
|
||||
if screen_type == ScreenType.FOREIGN_APP or screen_type == ScreenType.MODAL:
|
||||
obstacle_name = "Foreign app" if screen_type == ScreenType.FOREIGN_APP else "Modal"
|
||||
logger.warning(f"🚨 [GOAP] {obstacle_name} detected. Using SAE to clear...")
|
||||
|
||||
# SAE Feedback Loop!
|
||||
# If we hit this, the LAST action caused an obstacle! Mask it!
|
||||
if last_action:
|
||||
self.action_failures[last_action] = self.action_failures.get(last_action, 0) + MAX_RETRIES # Instantly mask it
|
||||
logger.warning(f"🛡️ [SAE Feedback] Action '{last_action}' caused an obstacle. Masking aggressively.")
|
||||
|
||||
if not self._get_sae().ensure_clear_screen():
|
||||
self.path_memory.learn_path(goal, start_screen, steps_taken, False)
|
||||
return False
|
||||
continue
|
||||
|
||||
if screen_type == ScreenType.MODAL:
|
||||
logger.warning("🚨 [GOAP] Modal detected. Using SAE to clear...")
|
||||
self._get_sae().ensure_clear_screen()
|
||||
if screen_type == ScreenType.FOREIGN_APP:
|
||||
self.path_memory.learn_path(goal, start_screen, steps_taken, False)
|
||||
return False
|
||||
continue
|
||||
|
||||
# PLAN
|
||||
action = self.planner.plan_next_step(goal, screen)
|
||||
action = self.planner.plan_next_step(goal, screen, explored_nav_actions=explored_nav_actions)
|
||||
|
||||
if action is None:
|
||||
# Goal achieved!
|
||||
logger.info(f"✅ [GOAP] Goal '{goal}' achieved in {step_num} steps!")
|
||||
self.path_memory.learn_path(goal, start_screen, steps_taken, True)
|
||||
|
||||
# Record dynamic knowledge: This goal lands us on THIS screen
|
||||
self.planner.knowledge.learn_goal_requirement(goal, screen_type)
|
||||
return True
|
||||
|
||||
logger.info(f"🧭 [GOAP Step {step_num + 1}] Action: '{action}'")
|
||||
last_action = action
|
||||
|
||||
# EXECUTE
|
||||
success = self._execute_action(action)
|
||||
success = self._execute_action(action, goal=goal)
|
||||
|
||||
steps_taken.append({
|
||||
'screen': screen_type.value,
|
||||
'action': action,
|
||||
'success': success,
|
||||
})
|
||||
|
||||
if not success:
|
||||
if success:
|
||||
steps_taken.append({"action": action})
|
||||
# Check if it was a navigation action (vs a goal action). If we are not on the required screen,
|
||||
# any action taken is essentially a navigation attempt.
|
||||
explored_nav_actions.add(action)
|
||||
# Reset failures for this action since it eventually succeeded
|
||||
self.action_failures[action] = 0
|
||||
else:
|
||||
self.action_failures[action] = self.action_failures.get(action, 0) + 1
|
||||
logger.warning(f"⚠️ [GOAP] Action '{action}' failed. Continuing with replanning...")
|
||||
|
||||
random_sleep(0.5, 1.5)
|
||||
|
||||
logger.error(f"❌ [GOAP] Failed to achieve '{goal}' in {max_steps} steps")
|
||||
logger.warning(f"⚠️ [GOAP] Goal '{goal}' failed after {max_steps} steps.")
|
||||
self.path_memory.learn_path(goal, start_screen, steps_taken, False)
|
||||
|
||||
# Memory Purge Logic: Wipe the path memory for this start_screen/goal combo
|
||||
# so it doesn't get stuck in a broken loop in future sessions!
|
||||
self.path_memory.forget_path(goal, start_screen)
|
||||
logger.warning(f"🧹 [Memory Purge] Wiped PathMemory cache for '{goal}' starting at '{start_screen}' to force re-discovery.")
|
||||
|
||||
return False
|
||||
|
||||
def _execute_action(self, action: str) -> bool:
|
||||
def _execute_action(self, action: str, goal: str = None) -> bool:
|
||||
"""Execute a single natural-language action using the TelepathicEngine."""
|
||||
|
||||
if action == 'press back':
|
||||
@@ -738,9 +994,51 @@ class GoalExecutor:
|
||||
import random
|
||||
time.sleep(random.uniform(1.6, 2.8))
|
||||
|
||||
# Verify UI changed
|
||||
# Verify success via Goal Context + Screen Feedback
|
||||
post_xml = self.device.dump_hierarchy()
|
||||
if post_xml != xml_dump:
|
||||
post_screen = self.perceive(post_xml)
|
||||
post_screen_type = post_screen['screen_type']
|
||||
|
||||
# Determine if this was a navigation or an interaction
|
||||
is_navigation = any(k in action.lower() for k in ["tab", "open", "go to", "navigate"])
|
||||
action_success = False
|
||||
goal_met = False
|
||||
ui_changed = post_xml != xml_dump
|
||||
|
||||
if is_navigation:
|
||||
if ui_changed:
|
||||
action_success = True
|
||||
logger.info(f"✅ [GOAP Step] Navigation '{action}' successful -> {post_screen_type.name}.")
|
||||
# Record dynamic knowledge: This action leads to THIS screen
|
||||
self.planner.knowledge.learn_screen_mapping(action, post_screen_type)
|
||||
else:
|
||||
logger.warning(f"❌ [GOAP Step] No UI change detected after '{action}'.")
|
||||
action_success = False
|
||||
else:
|
||||
# For interactions (like, follow) or unknown goals, use XML delta + semantic verify
|
||||
if ui_changed:
|
||||
if engine.verify_success(action, post_xml):
|
||||
action_success = True
|
||||
logger.info(f"✅ [GOAP Step] Interaction '{action}' successful.")
|
||||
else:
|
||||
logger.warning(f"❌ [GOAP Verify] Semantic verification failed for '{action}'.")
|
||||
else:
|
||||
logger.warning(f"❌ [GOAP Verify] No UI change detected after interaction '{action}'.")
|
||||
|
||||
# Optional: Log if the overarching goal was miraculously met early
|
||||
if goal and action_success:
|
||||
required_screens = self.planner.knowledge.get_requirements(goal)
|
||||
if not required_screens:
|
||||
if 'messages' in goal and post_screen_type == ScreenType.DM_INBOX: goal_met = True
|
||||
if 'explore' in goal and post_screen_type == ScreenType.EXPLORE_GRID: goal_met = True
|
||||
if 'home' in goal and post_screen_type == ScreenType.HOME_FEED: goal_met = True
|
||||
if 'profile' in goal and post_screen_type == ScreenType.OWN_PROFILE: goal_met = True
|
||||
if 'reels' in goal and post_screen_type == ScreenType.REELS_FEED: goal_met = True
|
||||
|
||||
if goal_met or (required_screens and post_screen_type in required_screens):
|
||||
logger.info(f"🎉 [GOAP Verify] OVERARCHING Goal '{goal}' achieved during step '{action}'.")
|
||||
|
||||
if action_success:
|
||||
engine.confirm_click(action)
|
||||
return True
|
||||
else:
|
||||
@@ -753,7 +1051,7 @@ class GoalExecutor:
|
||||
action = step.get('action', '')
|
||||
logger.info(f"🧠 [GOAP Recall Step {i + 1}/{len(steps)}] '{action}'")
|
||||
|
||||
success = self._execute_action(action)
|
||||
success = self._execute_action(action, goal=goal)
|
||||
if not success:
|
||||
logger.warning(f"⚠️ [GOAP Recall] Step '{action}' failed. Path may be stale.")
|
||||
return False
|
||||
|
||||
@@ -196,6 +196,15 @@ class QdrantBase:
|
||||
except Exception as e:
|
||||
self._handle_error(e, f"Search failed")
|
||||
return []
|
||||
def get_collection_size(self) -> int:
|
||||
"""Returns the number of points in the collection."""
|
||||
if not self.is_connected:
|
||||
return 0
|
||||
try:
|
||||
count_result = self.client.count(collection_name=self.collection_name)
|
||||
return count_result.count
|
||||
except Exception:
|
||||
return 0
|
||||
|
||||
|
||||
class HeuristicMemoryDB(QdrantBase):
|
||||
@@ -1062,6 +1071,13 @@ class ParasocialCRMDB(QdrantBase):
|
||||
interactions = current.get("interactions", [])
|
||||
import time
|
||||
now = time.time()
|
||||
|
||||
if interactions:
|
||||
last_interaction = interactions[-1]
|
||||
if last_interaction.get("type") == intent_type and (now - last_interaction.get("timestamp", 0)) < 300:
|
||||
logger.debug(f"🧠 [ParasocialCRM] Skipping redundant '{intent_type}' write for @{username}.")
|
||||
return
|
||||
|
||||
interactions.append({
|
||||
"type": intent_type,
|
||||
"timestamp": now
|
||||
|
||||
@@ -271,33 +271,54 @@ class SituationalAwarenessEngine:
|
||||
logger.info("📱 [SAE Perceive] Screen is physically OFF.")
|
||||
return SituationType.OBSTACLE_LOCKED_SCREEN
|
||||
|
||||
# ── Foreign App / System Dialog Detection (package-based) ──
|
||||
# ── Foreign Environment Detection (package-based) ──
|
||||
# Extract ALL packages present in the dump
|
||||
# Support both single and double quotes for package detection (robustness across uia2 versions and tests)
|
||||
packages = set(re.findall(r'package=["\']([^"\']+)["\']', xml_dump))
|
||||
app_id = getattr(self.device, 'app_id', 'com.instagram.android')
|
||||
|
||||
# ── System Dialog Detection (BEFORE foreign app — system dialogs overlay Instagram) ──
|
||||
system_packages = {'com.android.permissioncontroller', 'com.android.settings',
|
||||
'com.google.android.packageinstaller'}
|
||||
if system_packages & packages:
|
||||
return SituationType.OBSTACLE_SYSTEM
|
||||
|
||||
# If Instagram package is not present AT ALL, we're in a foreign app
|
||||
# If Instagram package is not present AT ALL, we are outside the app.
|
||||
if app_id not in packages:
|
||||
# Exception: system UI overlay is normal (status bar)
|
||||
non_system = packages - {'com.android.systemui', 'android'}
|
||||
if non_system:
|
||||
logger.info(f"🔍 [SAE Perceive] Foreign app detected: {non_system}")
|
||||
# We explicitly ask the TelepathicEngine to classify this to avoid writing brittle substring hacks
|
||||
# for Android System UI variations across different device manufacturers.
|
||||
try:
|
||||
from GramAddict.core.llm_provider import query_telepathic_llm
|
||||
from GramAddict.core.config import Config
|
||||
|
||||
screen_off = not getattr(self.device.deviceV2, 'info', {}).get("screenOn", True)
|
||||
|
||||
prompt = (
|
||||
"You are a Situation Classifier for a mobile automation agent.\n"
|
||||
"Analyze the given Android UI XML dump. Is this a physical DEVICE_LOCK_SCREEN, "
|
||||
"a system PERMISSION_DIALOG, or a NOTIFICATION_SHADE / FOREIGN_APP?\n"
|
||||
f"Hardware Screen Status: {'OFF (Locked)' if screen_off else 'ON'}.\n"
|
||||
"Respond ONLY with a valid JSON object strictly matching this schema: "
|
||||
"{\"situation\": \"OBSTACLE_LOCKED_SCREEN\" | \"OBSTACLE_SYSTEM\" | \"OBSTACLE_FOREIGN_APP\"}\n\n"
|
||||
f"XML:\n{self._compress_xml(xml_dump)[:2500]}"
|
||||
)
|
||||
|
||||
args = {}
|
||||
try: args = Config().args
|
||||
except Exception: pass
|
||||
model = getattr(args, "ai_telepathic_model", "qwen3.5:latest")
|
||||
url = getattr(args, "ai_telepathic_url", "http://localhost:11434/api/generate")
|
||||
|
||||
res = query_telepathic_llm(model=model, url=url, system_prompt="Strict JSON classifier.", user_prompt=prompt, use_local_edge=True)
|
||||
import json
|
||||
data = json.loads(res)
|
||||
situ_str = data.get("situation", "")
|
||||
|
||||
if situ_str == "OBSTACLE_LOCKED_SCREEN":
|
||||
logger.info("🧠 [Smart Perceive] SystemUI definitively classified as: LOCKED_SCREEN.")
|
||||
return SituationType.OBSTACLE_LOCKED_SCREEN
|
||||
elif situ_str == "OBSTACLE_SYSTEM":
|
||||
logger.info("🧠 [Smart Perceive] SystemUI definitively classified as: SYSTEM_DIALOG.")
|
||||
return SituationType.OBSTACLE_SYSTEM
|
||||
else:
|
||||
logger.info("🧠 [Smart Perceive] SystemUI classified as: FOREIGN_APP / NOTIFICATION.")
|
||||
return SituationType.OBSTACLE_FOREIGN_APP
|
||||
except Exception as e:
|
||||
logger.warning(f"⚠️ [Smart Perceive] LLM Classification failed ({e}). Defaulting to FOREIGN_APP.")
|
||||
return SituationType.OBSTACLE_FOREIGN_APP
|
||||
|
||||
# If only systemui is present, we might be on a lock screen, notification shade, or recent apps
|
||||
lock_markers = ['keyguard', 'lock_icon', 'emergency_call_button', 'kg_']
|
||||
if any(m in xml_lower for m in lock_markers):
|
||||
logger.info("📱 [SAE Perceive] Lock screen markers detected in SystemUI.")
|
||||
return SituationType.OBSTACLE_LOCKED_SCREEN
|
||||
|
||||
return SituationType.OBSTACLE_FOREIGN_APP
|
||||
|
||||
# ── Modal/Obstacle Detection (structural, not ID-based) ──
|
||||
# Instead of checking specific IDs, we check STRUCTURAL patterns:
|
||||
|
||||
@@ -58,7 +58,42 @@ class TelepathicEngine:
|
||||
cls._instance = None
|
||||
cls._last_click_context = None
|
||||
|
||||
def wipe(self):
|
||||
"""Perform a full 'Blank Start' wipe of all AI caches and persistent memories."""
|
||||
logger.warning("🔥 [TelepathicEngine] Wiping all AI caches and persistent records for BLANK START.")
|
||||
|
||||
# 1. Clear JSON files
|
||||
for f in [MEMORY_FILE, BLACKLIST_FILE]:
|
||||
if os.path.exists(f):
|
||||
try:
|
||||
os.remove(f)
|
||||
logger.info(f"🗑️ Deleted persistent file: {f}")
|
||||
except Exception as e:
|
||||
logger.error(f"❌ Failed to delete {f}: {e}")
|
||||
|
||||
# 2. Clear Qdrant collections
|
||||
try:
|
||||
self.embedding_helper.client.delete_collection("telepathic_engine_cache")
|
||||
logger.info("🗑️ Wiped Qdrant collection: telepathic_engine_cache")
|
||||
except Exception as e:
|
||||
logger.warning(f"⚠️ Could not wipe Qdrant collection (likely doesn't exist): {e}")
|
||||
|
||||
try:
|
||||
if hasattr(self, 'ui_memory') and self.ui_memory and self.ui_memory.is_connected:
|
||||
self.ui_memory.client.delete_collection(self.ui_memory.collection_name)
|
||||
logger.info(f"🗑️ Wiped Qdrant collection: {self.ui_memory.collection_name}")
|
||||
except Exception as e:
|
||||
logger.warning(f"⚠️ Could not wipe UIMemoryDB collection: {e}")
|
||||
|
||||
# 3. Clear In-memory state
|
||||
self._memory = {}
|
||||
self._blacklist = {}
|
||||
self._embedding_cache = {}
|
||||
self._intent_cache = {}
|
||||
|
||||
def __init__(self):
|
||||
from GramAddict.core.qdrant_memory import UIMemoryDB
|
||||
self.ui_memory = UIMemoryDB()
|
||||
self.embedding_helper = QdrantBase("telepathic_engine_cache")
|
||||
self._embedding_cache: Dict[str, list] = {}
|
||||
self._intent_cache: Dict[str, list] = {}
|
||||
@@ -526,9 +561,15 @@ class TelepathicEngine:
|
||||
score *= 0.3 # Heavy penalty
|
||||
|
||||
# Thresholding:
|
||||
# - Short intents (1-2 words like 'tap home'): Require at least 50% hit (0.45)
|
||||
# - Navigation intents: Require 100% exact match to avoid feed-cross-talk
|
||||
# - Short intents (1-2 words): Require at least 50% hit (0.45)
|
||||
# - Longer intents: Require 75% to avoid false matches on noisy screens.
|
||||
threshold = 0.45 if len(intent_words) <= 2 else 0.75
|
||||
is_nav_intent = any(k in intent_lower for k in ["tab", "navigation", "reels tab", "profile tab", "home tab", "explore tab", "message tab"])
|
||||
if is_nav_intent:
|
||||
threshold = 1.0
|
||||
else:
|
||||
threshold = 0.45 if len(intent_words) <= 2 else 0.75
|
||||
|
||||
if score >= threshold:
|
||||
scored.append((node, score))
|
||||
|
||||
@@ -1015,18 +1056,21 @@ class TelepathicEngine:
|
||||
clean_json = extract_json(resp_dict["response"])
|
||||
if clean_json:
|
||||
data = json.loads(clean_json)
|
||||
idx = data.get("best_index")
|
||||
if idx is not None and 0 <= idx < len(grid_nodes):
|
||||
chosen = grid_nodes[idx]
|
||||
logger.info(f"✅ [Vision Match] Cell {idx} chosen: {data.get('reason')}", extra={"color": f"{Fore.GREEN}"})
|
||||
self._track_click(f"Visual Grid Selection ({idx})", chosen)
|
||||
return {
|
||||
"x": chosen["x"],
|
||||
"y": chosen["y"],
|
||||
"score": 0.99,
|
||||
"semantic": f"Visual match {idx}: {data.get('reason')}",
|
||||
"source": "vlm_grid"
|
||||
}
|
||||
if isinstance(data, list) and len(data) > 0:
|
||||
data = data[0]
|
||||
if isinstance(data, dict):
|
||||
idx = data.get("best_index")
|
||||
if idx is not None and 0 <= idx < len(grid_nodes):
|
||||
chosen = grid_nodes[idx]
|
||||
logger.info(f"✅ [Vision Match] Cell {idx} chosen: {data.get('reason')}", extra={"color": f"{Fore.GREEN}"})
|
||||
self._track_click(f"Visual Grid Selection ({idx})", chosen)
|
||||
return {
|
||||
"x": chosen["x"],
|
||||
"y": chosen["y"],
|
||||
"score": 0.99,
|
||||
"semantic": f"Visual match {idx}: {data.get('reason')}",
|
||||
"source": "vlm_grid"
|
||||
}
|
||||
except Exception as e:
|
||||
logger.error(f"👁️ [Vision Core] Grid evaluation failed: {e}")
|
||||
|
||||
@@ -1143,11 +1187,38 @@ class TelepathicEngine:
|
||||
|
||||
def _core_navigation_fast_path(self, intent_description: str, viable_nodes: list) -> Optional[dict]:
|
||||
"""
|
||||
[Phase 2] Hardened Fast Path with Qdrant Self-Learning.
|
||||
Absolutely deterministic resource-ID targeting for core application navigation
|
||||
(like direct messages or post usernames) to prevent VLM hallucination.
|
||||
(like direct messages or post usernames). We query Qdrant first. If empty,
|
||||
we seed it with legacy fallbacks.
|
||||
"""
|
||||
low_intent = intent_description.lower()
|
||||
low_intent = intent_description.lower().strip()
|
||||
|
||||
# 0. Query Qdrant Memory first!
|
||||
mem = self.ui_memory.retrieve_memory(intent_description, "", similarity_threshold=0.9)
|
||||
if mem and isinstance(mem, dict):
|
||||
if mem.get("action") == "tap" and mem.get("resource_id"):
|
||||
learned_res_id = mem.get("resource_id")
|
||||
for n in viable_nodes:
|
||||
if learned_res_id in n.get("resource_id", "").lower():
|
||||
logger.info(f"🧠 [TelepathicEngine] Fast-Path resolved '{intent_description}' via Qdrant -> '{learned_res_id}'", extra={"color": f"\\033[36m"})
|
||||
self._track_click(intent_description, n)
|
||||
return {
|
||||
"x": n["x"],
|
||||
"y": n["y"],
|
||||
"score": 1.0,
|
||||
"semantic": n["semantic_string"],
|
||||
"source": "qdrant_nav"
|
||||
}
|
||||
|
||||
# 0.5 Enforce Bootstrapper Lifecycle
|
||||
# If Qdrant memory is sufficiently populated, ignore hardcoded seeds to enforce 100% self-learning
|
||||
mem_size = self.ui_memory.get_collection_size()
|
||||
if isinstance(mem_size, int) and mem_size > 25:
|
||||
logger.debug(f"🌱 [Bootstrapper] Skipping hardcoded seeds for '{intent_description}' (Qdrant memory populated).")
|
||||
return None
|
||||
|
||||
|
||||
# 1. Post Username (Feed Profile)
|
||||
if low_intent in ["tap_post_username", "tap post username"]:
|
||||
for n in viable_nodes:
|
||||
@@ -1195,6 +1266,8 @@ class TelepathicEngine:
|
||||
"tap explore tab": "search_tab",
|
||||
"tap_reels_tab": "clips_tab",
|
||||
"tap reels tab": "clips_tab",
|
||||
"tap_messages_tab": "direct_tab",
|
||||
"tap messages tab": "direct_tab",
|
||||
"tap_profile_tab": "profile_tab",
|
||||
"tap profile tab": "profile_tab"
|
||||
}
|
||||
@@ -1202,7 +1275,15 @@ class TelepathicEngine:
|
||||
target_res_id = tab_mappings[low_intent]
|
||||
for n in viable_nodes:
|
||||
if target_res_id in n.get("resource_id", "").lower():
|
||||
logger.info(f"⚡ [Core Nav Fast Path] Found explicit Main Tab mapping for '{intent_description}' -> '{target_res_id}'")
|
||||
logger.warning(f"🌱 [TelepathicEngine] Seeding Qdrant Memory with legacy fast-path: '{intent_description}' -> '{target_res_id}'")
|
||||
self.ui_memory.store_memory(
|
||||
intent_description,
|
||||
"",
|
||||
{
|
||||
"resource_id": target_res_id,
|
||||
"action": "tap"
|
||||
}
|
||||
)
|
||||
self._track_click(intent_description, n)
|
||||
return {
|
||||
"x": n["x"],
|
||||
|
||||
Reference in New Issue
Block a user